torvalds/linux
torvalds/linux
2
0
Fork 0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-09-04 20:19:47 +08:00
Code

KVM/arm64 changes for 6.17, round #1

Browse Source 
- Host driver for GICv5, the next generation interrupt controller for
    arm64, including support for interrupt routing, MSIs, interrupt
    translation and wired interrupts.
 
  - Use FEAT_GCIE_LEGACY on GICv5 systems to virtualize GICv3 VMs on
    GICv5 hardware, leveraging the legacy VGIC interface.
 
  - Userspace control of the 'nASSGIcap' GICv3 feature, allowing
    userspace to disable support for SGIs w/o an active state on hardware
    that previously advertised it unconditionally.
 
  - Map supporting endpoints with cacheable memory attributes on systems
    with FEAT_S2FWB and DIC where KVM no longer needs to perform cache
    maintenance on the address range.
 
  - Nested support for FEAT_RAS and FEAT_DoubleFault2, allowing the guest
    hypervisor to inject external aborts into an L2 VM and take traps of
    masked external aborts to the hypervisor.
 
  - Convert more system register sanitization to the config-driven
    implementation.
 
  - Fixes to the visibility of EL2 registers, namely making VGICv3 system
    registers accessible through the VGIC device instead of the ONE_REG
    vCPU ioctls.
 
  - Various cleanups and minor fixes.
 -----BEGIN PGP SIGNATURE-----
 
 iI0EABYIADUWIQSNXHjWXuzMZutrKNKivnWIJHzdFgUCaIezbRccb2xpdmVyLnVw
 dG9uQGxpbnV4LmRldgAKCRCivnWIJHzdFr/eAQDY5NIG5cR6ZcAWnPQLmGWpz2ou
 pq4Jhn9E/mGR3n5L1AEAsJpfLLpOsmnLBdwfbjmW59gGsa8k3i5tjWEOJ6yzAwk=
 =r+sp
 -----END PGP SIGNATURE-----

Merge tag 'kvmarm-6.17' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm into HEAD

KVM/arm64 changes for 6.17, round #1

 - Host driver for GICv5, the next generation interrupt controller for
   arm64, including support for interrupt routing, MSIs, interrupt
   translation and wired interrupts.

 - Use FEAT_GCIE_LEGACY on GICv5 systems to virtualize GICv3 VMs on
   GICv5 hardware, leveraging the legacy VGIC interface.

 - Userspace control of the 'nASSGIcap' GICv3 feature, allowing
   userspace to disable support for SGIs w/o an active state on hardware
   that previously advertised it unconditionally.

 - Map supporting endpoints with cacheable memory attributes on systems
   with FEAT_S2FWB and DIC where KVM no longer needs to perform cache
   maintenance on the address range.

 - Nested support for FEAT_RAS and FEAT_DoubleFault2, allowing the guest
   hypervisor to inject external aborts into an L2 VM and take traps of
   masked external aborts to the hypervisor.

 - Convert more system register sanitization to the config-driven
   implementation.

 - Fixes to the visibility of EL2 registers, namely making VGICv3 system
   registers accessible through the VGIC device instead of the ONE_REG
   vCPU ioctls.

 - Various cleanups and minor fixes.
This commit is contained in:
Paolo Bonzini 2025-07-29 12:27:40 -04:00
commit 314b40b3b6
80 changed files with 7589 additions and 681 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
Documentation/arch/arm64/booting.rst
View File

@ -223,6 +223,47 @@ Before jumping into the kernel, the following conditions must be met:
- SCR_EL3.HCE (bit 8) must be initialised to 0b1.
For systems with a GICv5 interrupt controller to be used in v5 mode:
- If the kernel is entered at EL1 and EL2 is present:
- ICH_HFGRTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PPI_HMRn_EL1 (bit 16) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_IAFFIDR_EL1 (bit 7) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_HPPIR_EL1 (bit 4) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_HAPR_EL1 (bit 3) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_IDRn_EL1 (bit 1) must be initialised to 0b1.
- ICH_HFGRTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_ACTIVERn_EL1 (bit 20) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_PRIORITYRn_EL1 (bit 19) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_PENDRn_EL1 (bit 18) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PPI_ENABLERn_EL1 (bit 17) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_ICSR_EL1 (bit 6) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_PCR_EL1 (bit 5) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_CR0_EL1 (bit 2) must be initialised to 0b1.
- ICH_HFGWTR_EL2.ICC_APR_EL1 (bit 0) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICRCDNMIA (bit 10) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICRCDIA (bit 9) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDDI (bit 8) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDEOI (bit 7) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDHM (bit 6) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDRCFG (bit 5) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDPEND (bit 4) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDAFF (bit 3) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDPRI (bit 2) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDDIS (bit 1) must be initialised to 0b1.
- ICH_HFGITR_EL2.GICCDEN (bit 0) must be initialised to 0b1.
- The DT or ACPI tables must describe a GICv5 interrupt controller.
For systems with a GICv3 interrupt controller to be used in v3 mode:
- If EL3 is present:

78
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5-iwb.yaml Normal file
View File

@ -0,0 +1,78 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5-iwb.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM Generic Interrupt Controller, version 5 Interrupt Wire Bridge (IWB)
maintainers:
- Lorenzo Pieralisi <lpieralisi@kernel.org>
- Marc Zyngier <maz@kernel.org>
description: |
The GICv5 architecture defines the guidelines to implement GICv5
compliant interrupt controllers for AArch64 systems.
The GICv5 specification can be found at
https://developer.arm.com/documentation/aes0070
GICv5 has zero or more Interrupt Wire Bridges (IWB) that are responsible
for translating wire signals into interrupt messages to the GICv5 ITS.
allOf:
- $ref: /schemas/interrupt-controller.yaml#
properties:
compatible:
const: arm,gic-v5-iwb
reg:
items:
- description: IWB control frame
"#address-cells":
const: 0
"#interrupt-cells":
description: |
The 1st cell corresponds to the IWB wire.
The 2nd cell is the flags, encoded as follows:
bits[3:0] trigger type and level flags.
1 = low-to-high edge triggered
2 = high-to-low edge triggered
4 = active high level-sensitive
8 = active low level-sensitive
const: 2
interrupt-controller: true
msi-parent:
maxItems: 1
required:
- compatible
- reg
- "#interrupt-cells"
- interrupt-controller
- msi-parent
additionalProperties: false
examples:
- |
interrupt-controller@2f000000 {
compatible = "arm,gic-v5-iwb";
reg = <0x2f000000 0x10000>;
#address-cells = <0>;
#interrupt-cells = <2>;
interrupt-controller;
msi-parent = <&its0 64>;
};
...

267
Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5.yaml Normal file
View File

@ -0,0 +1,267 @@
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
%YAML 1.2
---
$id: http://devicetree.org/schemas/interrupt-controller/arm,gic-v5.yaml#
$schema: http://devicetree.org/meta-schemas/core.yaml#
title: ARM Generic Interrupt Controller, version 5
maintainers:
- Lorenzo Pieralisi <lpieralisi@kernel.org>
- Marc Zyngier <maz@kernel.org>
description: |
The GICv5 architecture defines the guidelines to implement GICv5
compliant interrupt controllers for AArch64 systems.
The GICv5 specification can be found at
https://developer.arm.com/documentation/aes0070
The GICv5 architecture is composed of multiple components:
- one or more IRS (Interrupt Routing Service)
- zero or more ITS (Interrupt Translation Service)
The architecture defines:
- PE-Private Peripheral Interrupts (PPI)
- Shared Peripheral Interrupts (SPI)
- Logical Peripheral Interrupts (LPI)
allOf:
- $ref: /schemas/interrupt-controller.yaml#
properties:
compatible:
const: arm,gic-v5
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
"#interrupt-cells":
description: |
The 1st cell corresponds to the INTID.Type field in the INTID; 1 for PPI,
3 for SPI. LPI interrupts must not be described in the bindings since
they are allocated dynamically by the software component managing them.
The 2nd cell contains the interrupt INTID.ID field.
The 3rd cell is the flags, encoded as follows:
bits[3:0] trigger type and level flags.
1 = low-to-high edge triggered
2 = high-to-low edge triggered
4 = active high level-sensitive
8 = active low level-sensitive
const: 3
interrupt-controller: true
interrupts:
description:
The VGIC maintenance interrupt.
maxItems: 1
required:
- compatible
- "#address-cells"
- "#size-cells"
- ranges
- "#interrupt-cells"
- interrupt-controller
patternProperties:
"^irs@[0-9a-f]+$":
type: object
description:
GICv5 has one or more Interrupt Routing Services (IRS) that are
responsible for handling IRQ state and routing.
additionalProperties: false
properties:
compatible:
const: arm,gic-v5-irs
reg:
minItems: 1
items:
- description: IRS config frames
- description: IRS setlpi frames
reg-names:
description:
Describe config and setlpi frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 8
items:
enum: [ ns-config, s-config, realm-config, el3-config, ns-setlpi,
s-setlpi, realm-setlpi, el3-setlpi ]
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
dma-noncoherent:
description:
Present if the GIC IRS permits programming shareability and
cacheability attributes but is connected to a non-coherent
downstream interconnect.
cpus:
description:
CPUs managed by the IRS.
arm,iaffids:
$ref: /schemas/types.yaml#/definitions/uint16-array
description:
Interrupt AFFinity ID (IAFFID) associated with the CPU whose
CPU node phandle is at the same index in the cpus array.
patternProperties:
"^its@[0-9a-f]+$":
type: object
description:
GICv5 has zero or more Interrupt Translation Services (ITS) that are
used to route Message Signalled Interrupts (MSI) to the CPUs. Each
ITS is connected to an IRS.
additionalProperties: false
properties:
compatible:
const: arm,gic-v5-its
reg:
items:
- description: ITS config frames
reg-names:
description:
Describe config frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 4
items:
enum: [ ns-config, s-config, realm-config, el3-config ]
"#address-cells":
enum: [ 1, 2 ]
"#size-cells":
enum: [ 1, 2 ]
ranges: true
dma-noncoherent:
description:
Present if the GIC ITS permits programming shareability and
cacheability attributes but is connected to a non-coherent
downstream interconnect.
patternProperties:
"^msi-controller@[0-9a-f]+$":
type: object
description:
GICv5 ITS has one or more translate register frames.
additionalProperties: false
properties:
reg:
items:
- description: ITS translate frames
reg-names:
description:
Describe translate frames that are present.
"ns-" stands for non-secure, "s-" for secure, "realm-" for realm
and "el3-" for EL3.
minItems: 1
maxItems: 4
items:
enum: [ ns-translate, s-translate, realm-translate, el3-translate ]
"#msi-cells":
description:
The single msi-cell is the DeviceID of the device which will
generate the MSI.
const: 1
msi-controller: true
required:
- reg
- reg-names
- "#msi-cells"
- msi-controller
required:
- compatible
- reg
- reg-names
required:
- compatible
- reg
- reg-names
- cpus
- arm,iaffids
additionalProperties: false
examples:
- |
interrupt-controller {
compatible = "arm,gic-v5";
#interrupt-cells = <3>;
interrupt-controller;
#address-cells = <1>;
#size-cells = <1>;
ranges;
interrupts = <1 25 4>;
irs@2f1a0000 {
compatible = "arm,gic-v5-irs";
reg = <0x2f1a0000 0x10000>; // IRS_CONFIG_FRAME
reg-names = "ns-config";
#address-cells = <1>;
#size-cells = <1>;
ranges;
cpus = <&cpu0>, <&cpu1>, <&cpu2>, <&cpu3>, <&cpu4>, <&cpu5>, <&cpu6>, <&cpu7>;
arm,iaffids = /bits/ 16 <0 1 2 3 4 5 6 7>;
its@2f120000 {
compatible = "arm,gic-v5-its";
reg = <0x2f120000 0x10000>; // ITS_CONFIG_FRAME
reg-names = "ns-config";
#address-cells = <1>;
#size-cells = <1>;
ranges;
msi-controller@2f130000 {
reg = <0x2f130000 0x10000>; // ITS_TRANSLATE_FRAME
reg-names = "ns-translate";
#msi-cells = <1>;
msi-controller;
};
};
};
};
...

13
Documentation/virt/kvm/api.rst
View File

@ -8622,7 +8622,7 @@ ENOSYS for the others.
When enabled, KVM will exit to userspace with KVM_EXIT_SYSTEM_EVENT of
type KVM_SYSTEM_EVENT_SUSPEND to process the guest suspend request.
7.37 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
7.42 KVM_CAP_ARM_WRITABLE_IMP_ID_REGS
-------------------------------------
:Architectures: arm64
@ -8651,6 +8651,17 @@ given VM.
When this capability is enabled, KVM resets the VCPU when setting
MP_STATE_INIT_RECEIVED through IOCTL. The original MP_STATE is preserved.
7.43 KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED
-------------------------------------------
:Architectures: arm64
:Target: VM
:Parameters: None
This capability indicate to the userspace whether a PFNMAP memory region
can be safely mapped as cacheable. This relies on the presence of
force write back (FWB) feature support on the hardware.
8. Other capabilities.
======================

80
Documentation/virt/kvm/devices/arm-vgic-v3.rst
View File

@ -78,6 +78,8 @@ Groups:
-ENXIO The group or attribute is unknown/unsupported for this device
or hardware support is missing.
-EFAULT Invalid user pointer for attr->addr.
-EBUSY Attempt to write a register that is read-only after
initialization
======= =============================================================
@ -120,6 +122,15 @@ Groups:
Note that distributor fields are not banked, but return the same value
regardless of the mpidr used to access the register.
Userspace is allowed to write the following register fields prior to
initialization of the VGIC:
=====================
GICD_IIDR.Revision
GICD_TYPER2.nASSGIcap
=====================
GICD_IIDR.Revision is updated when the KVM implementation is changed in a
way directly observable by the guest or userspace. Userspace should read
GICD_IIDR from KVM and write back the read value to confirm its expected
@ -128,6 +139,12 @@ Groups:
behavior.
GICD_TYPER2.nASSGIcap allows userspace to control the support of SGIs
without an active state. At VGIC creation the field resets to the
maximum capability of the system. Userspace is expected to read the field
to determine the supported value(s) before writing to the field.
The GICD_STATUSR and GICR_STATUSR registers are architecturally defined such
that a write of a clear bit has no effect, whereas a write with a set bit
clears that value. To allow userspace to freely set the values of these two
@ -202,16 +219,69 @@ Groups:
KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS accesses the CPU interface registers for the
CPU specified by the mpidr field.
CPU interface registers access is not implemented for AArch32 mode.
Error -ENXIO is returned when accessed in AArch32 mode.
The available registers are:
=============== ====================================================
ICC_PMR_EL1
ICC_BPR0_EL1
ICC_AP0R0_EL1
ICC_AP0R1_EL1 when the host implements at least 6 bits of priority
ICC_AP0R2_EL1 when the host implements 7 bits of priority
ICC_AP0R3_EL1 when the host implements 7 bits of priority
ICC_AP1R0_EL1
ICC_AP1R1_EL1 when the host implements at least 6 bits of priority
ICC_AP1R2_EL1 when the host implements 7 bits of priority
ICC_AP1R3_EL1 when the host implements 7 bits of priority
ICC_BPR1_EL1
ICC_CTLR_EL1
ICC_SRE_EL1
ICC_IGRPEN0_EL1
ICC_IGRPEN1_EL1
=============== ====================================================
When EL2 is available for the guest, these registers are also available:
============= ====================================================
ICH_AP0R0_EL2
ICH_AP0R1_EL2 when the host implements at least 6 bits of priority
ICH_AP0R2_EL2 when the host implements 7 bits of priority
ICH_AP0R3_EL2 when the host implements 7 bits of priority
ICH_AP1R0_EL2
ICH_AP1R1_EL2 when the host implements at least 6 bits of priority
ICH_AP1R2_EL2 when the host implements 7 bits of priority
ICH_AP1R3_EL2 when the host implements 7 bits of priority
ICH_HCR_EL2
ICC_SRE_EL2
ICH_VTR_EL2
ICH_VMCR_EL2
ICH_LR0_EL2
ICH_LR1_EL2
ICH_LR2_EL2
ICH_LR3_EL2
ICH_LR4_EL2
ICH_LR5_EL2
ICH_LR6_EL2
ICH_LR7_EL2
ICH_LR8_EL2
ICH_LR9_EL2
ICH_LR10_EL2
ICH_LR11_EL2
ICH_LR12_EL2
ICH_LR13_EL2
ICH_LR14_EL2
ICH_LR15_EL2
============= ====================================================
CPU interface registers are only described using the AArch64
encoding.
Errors:
======= =====================================================
-ENXIO Getting or setting this register is not yet supported
======= =================================================
-ENXIO Getting or setting this register is not supported
-EBUSY VCPU is running
-EINVAL Invalid mpidr or register value supplied
======= =====================================================
======= =================================================
KVM_DEV_ARM_VGIC_GRP_NR_IRQS

10
MAINTAINERS
View File

@ -1964,6 +1964,16 @@ F: drivers/irqchip/irq-gic*.[ch]
F: include/linux/irqchip/arm-gic*.h
F: include/linux/irqchip/arm-vgic-info.h
ARM GENERIC INTERRUPT CONTROLLER V5 DRIVERS
M: Lorenzo Pieralisi <lpieralisi@kernel.org>
M: Marc Zyngier <maz@kernel.org>
L: linux-arm-kernel@lists.infradead.org (moderated for non-subscribers)
S: Maintained
F: Documentation/devicetree/bindings/interrupt-controller/arm,gic-v5*.yaml
F: drivers/irqchip/irq-gic-its-msi-parent.[ch]
F: drivers/irqchip/irq-gic-v5*.[ch]
F: include/linux/irqchip/arm-gic-v5.h
ARM HDLCD DRM DRIVER
M: Liviu Dudau <liviu.dudau@arm.com>
S: Supported

1
arch/arm64/Kconfig
View File

@ -129,6 +129,7 @@ config ARM64
select ARM_GIC_V2M if PCI
select ARM_GIC_V3
select ARM_GIC_V3_ITS if PCI
select ARM_GIC_V5
select ARM_PSCI_FW
select BUILDTIME_TABLE_SORT
select CLONE_BACKWARDS

3
arch/arm64/include/asm/barrier.h
View File

@ -44,6 +44,9 @@
SB_BARRIER_INSN"nop\n", \
ARM64_HAS_SB))
#define gsb_ack() asm volatile(GSB_ACK_BARRIER_INSN : : : "memory")
#define gsb_sys() asm volatile(GSB_SYS_BARRIER_INSN : : : "memory")
#ifdef CONFIG_ARM64_PSEUDO_NMI
#define pmr_sync() \
do { \

45
arch/arm64/include/asm/el2_setup.h
View File

@ -165,6 +165,50 @@
.Lskip_gicv3_\@:
.endm
/* GICv5 system register access */
.macro __init_el2_gicv5
mrs_s x0, SYS_ID_AA64PFR2_EL1
ubfx x0, x0, #ID_AA64PFR2_EL1_GCIE_SHIFT, #4
cbz x0, .Lskip_gicv5_\@
mov x0, #(ICH_HFGITR_EL2_GICRCDNMIA | \
ICH_HFGITR_EL2_GICRCDIA | \
ICH_HFGITR_EL2_GICCDDI | \
ICH_HFGITR_EL2_GICCDEOI | \
ICH_HFGITR_EL2_GICCDHM | \
ICH_HFGITR_EL2_GICCDRCFG | \
ICH_HFGITR_EL2_GICCDPEND | \
ICH_HFGITR_EL2_GICCDAFF | \
ICH_HFGITR_EL2_GICCDPRI | \
ICH_HFGITR_EL2_GICCDDIS | \
ICH_HFGITR_EL2_GICCDEN)
msr_s SYS_ICH_HFGITR_EL2, x0 // Disable instruction traps
mov_q x0, (ICH_HFGRTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_PENDRn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_ENABLERn_EL1 | \
ICH_HFGRTR_EL2_ICC_PPI_HMRn_EL1 | \
ICH_HFGRTR_EL2_ICC_IAFFIDR_EL1 | \
ICH_HFGRTR_EL2_ICC_ICSR_EL1 | \
ICH_HFGRTR_EL2_ICC_PCR_EL1 | \
ICH_HFGRTR_EL2_ICC_HPPIR_EL1 | \
ICH_HFGRTR_EL2_ICC_HAPR_EL1 | \
ICH_HFGRTR_EL2_ICC_CR0_EL1 | \
ICH_HFGRTR_EL2_ICC_IDRn_EL1 | \
ICH_HFGRTR_EL2_ICC_APR_EL1)
msr_s SYS_ICH_HFGRTR_EL2, x0 // Disable reg read traps
mov_q x0, (ICH_HFGWTR_EL2_ICC_PPI_ACTIVERn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_PRIORITYRn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_PENDRn_EL1 | \
ICH_HFGWTR_EL2_ICC_PPI_ENABLERn_EL1 | \
ICH_HFGWTR_EL2_ICC_ICSR_EL1 | \
ICH_HFGWTR_EL2_ICC_PCR_EL1 | \
ICH_HFGWTR_EL2_ICC_CR0_EL1 | \
ICH_HFGWTR_EL2_ICC_APR_EL1)
msr_s SYS_ICH_HFGWTR_EL2, x0 // Disable reg write traps
.Lskip_gicv5_\@:
.endm
.macro __init_el2_hstr
msr hstr_el2, xzr // Disable CP15 traps to EL2
.endm
@ -303,6 +347,7 @@
__init_el2_lor
__init_el2_stage2
__init_el2_gicv3
__init_el2_gicv5
__init_el2_hstr
__init_el2_nvhe_idregs
__init_el2_cptr

51
arch/arm64/include/asm/kvm_emulate.h
View File

@ -45,16 +45,39 @@ bool kvm_condition_valid32(const struct kvm_vcpu *vcpu);
void kvm_skip_instr32(struct kvm_vcpu *vcpu);
void kvm_inject_undefined(struct kvm_vcpu *vcpu);
void kvm_inject_vabt(struct kvm_vcpu *vcpu);
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr);
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr);
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr);
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
void kvm_inject_size_fault(struct kvm_vcpu *vcpu);
static inline int kvm_inject_sea_dabt(struct kvm_vcpu *vcpu, u64 addr)
{
return kvm_inject_sea(vcpu, false, addr);
}
static inline int kvm_inject_sea_iabt(struct kvm_vcpu *vcpu, u64 addr)
{
return kvm_inject_sea(vcpu, true, addr);
}
static inline int kvm_inject_serror(struct kvm_vcpu *vcpu)
{
/*
* ESR_ELx.ISV (later renamed to IDS) indicates whether or not
* ESR_ELx.ISS contains IMPLEMENTATION DEFINED syndrome information.
*
* Set the bit when injecting an SError w/o an ESR to indicate ISS
* does not follow the architected format.
*/
return kvm_inject_serror_esr(vcpu, ESR_ELx_ISV);
}
void kvm_vcpu_wfi(struct kvm_vcpu *vcpu);
void kvm_emulate_nested_eret(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sync(struct kvm_vcpu *vcpu, u64 esr_el2);
int kvm_inject_nested_irq(struct kvm_vcpu *vcpu);
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr);
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr);
static inline void kvm_inject_nested_sve_trap(struct kvm_vcpu *vcpu)
{
@ -195,6 +218,11 @@ static inline bool vcpu_el2_tge_is_set(const struct kvm_vcpu *vcpu)
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_TGE;
}
static inline bool vcpu_el2_amo_is_set(const struct kvm_vcpu *vcpu)
{
return ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2) & HCR_AMO;
}
static inline bool is_hyp_ctxt(const struct kvm_vcpu *vcpu)
{
bool e2h, tge;
@ -224,6 +252,20 @@ static inline bool vcpu_is_host_el0(const struct kvm_vcpu *vcpu)
return is_hyp_ctxt(vcpu) && !vcpu_is_el2(vcpu);
}
static inline bool is_nested_ctxt(struct kvm_vcpu *vcpu)
{
return vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
}
static inline bool vserror_state_is_nested(struct kvm_vcpu *vcpu)
{
if (!is_nested_ctxt(vcpu))
return false;
return vcpu_el2_amo_is_set(vcpu) ||
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
}
/*
* The layout of SPSR for an AArch32 state is different when observed from an
* AArch64 SPSR_ELx or an AArch32 SPSR_*. This function generates the AArch32
@ -627,6 +669,9 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
if (kvm_has_fpmr(kvm))
vcpu->arch.hcrx_el2 |= HCRX_EL2_EnFPM;
if (kvm_has_sctlr2(kvm))
vcpu->arch.hcrx_el2 |= HCRX_EL2_SCTLR2En;
}
}
#endif /* __ARM64_KVM_EMULATE_H__ */

36
arch/arm64/include/asm/kvm_host.h
View File

@ -523,6 +523,7 @@ enum vcpu_sysreg {
/* Anything from this can be RES0/RES1 sanitised */
MARKER(__SANITISED_REG_START__),
TCR2_EL2, /* Extended Translation Control Register (EL2) */
SCTLR2_EL2, /* System Control Register 2 (EL2) */
MDCR_EL2, /* Monitor Debug Configuration Register (EL2) */
CNTHCTL_EL2, /* Counter-timer Hypervisor Control register */
@ -537,6 +538,7 @@ enum vcpu_sysreg {
VNCR(TTBR1_EL1),/* Translation Table Base Register 1 */
VNCR(TCR_EL1), /* Translation Control Register */
VNCR(TCR2_EL1), /* Extended Translation Control Register */
VNCR(SCTLR2_EL1), /* System Control Register 2 */
VNCR(ESR_EL1), /* Exception Syndrome Register */
VNCR(AFSR0_EL1),/* Auxiliary Fault Status Register 0 */
VNCR(AFSR1_EL1),/* Auxiliary Fault Status Register 1 */
@ -565,6 +567,10 @@ enum vcpu_sysreg {
VNCR(POR_EL1), /* Permission Overlay Register 1 (EL1) */
/* FEAT_RAS registers */
VNCR(VDISR_EL2),
VNCR(VSESR_EL2),
VNCR(HFGRTR_EL2),
VNCR(HFGWTR_EL2),
VNCR(HFGITR_EL2),
@ -817,7 +823,7 @@ struct kvm_vcpu_arch {
u8 iflags;
/* State flags for kernel bookkeeping, unused by the hypervisor code */
u8 sflags;
u16 sflags;
/*
* Don't run the guest (internal implementation need).
@ -953,9 +959,21 @@ struct kvm_vcpu_arch {
__vcpu_flags_preempt_enable(); \
} while (0)
#define __vcpu_test_and_clear_flag(v, flagset, f, m) \
({ \
typeof(v->arch.flagset) set; \
\
set = __vcpu_get_flag(v, flagset, f, m); \
__vcpu_clear_flag(v, flagset, f, m); \
\
set; \
})
#define vcpu_get_flag(v, ...) __vcpu_get_flag((v), __VA_ARGS__)
#define vcpu_set_flag(v, ...) __vcpu_set_flag((v), __VA_ARGS__)
#define vcpu_clear_flag(v, ...) __vcpu_clear_flag((v), __VA_ARGS__)
#define vcpu_test_and_clear_flag(v, ...) \
__vcpu_test_and_clear_flag((v), __VA_ARGS__)
/* KVM_ARM_VCPU_INIT completed */
#define VCPU_INITIALIZED __vcpu_single_flag(cflags, BIT(0))
@ -1015,6 +1033,8 @@ struct kvm_vcpu_arch {
#define IN_WFI __vcpu_single_flag(sflags, BIT(6))
/* KVM is currently emulating a nested ERET */
#define IN_NESTED_ERET __vcpu_single_flag(sflags, BIT(7))
/* SError pending for nested guest */
#define NESTED_SERROR_PENDING __vcpu_single_flag(sflags, BIT(8))
/* Pointer to the vcpu's SVE FFR for sve_{save,load}_state() */
@ -1149,6 +1169,8 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
* System registers listed in the switch are not saved on every
* exit from the guest but are only saved on vcpu_put.
*
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
*
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
* should never be listed below, because the guest cannot modify its
* own MPIDR_EL1 and MPIDR_EL1 is accessed for VCPU A from VCPU B's
@ -1186,6 +1208,7 @@ static inline bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val)
case IFSR32_EL2: *val = read_sysreg_s(SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: *val = read_sysreg_s(SYS_DBGVCR32_EL2); break;
case ZCR_EL1: *val = read_sysreg_s(SYS_ZCR_EL12); break;
case SCTLR2_EL1: *val = read_sysreg_s(SYS_SCTLR2_EL12); break;
default: return false;
}
@ -1200,6 +1223,8 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
* System registers listed in the switch are not restored on every
* entry to the guest but are only restored on vcpu_load.
*
* SYSREGS_ON_CPU *MUST* be checked before using this helper.
*
* Note that MPIDR_EL1 for the guest is set by KVM via VMPIDR_EL2 but
* should never be listed below, because the MPIDR should only be set
* once, before running the VCPU, and never changed later.
@ -1236,6 +1261,7 @@ static inline bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg)
case IFSR32_EL2: write_sysreg_s(val, SYS_IFSR32_EL2); break;
case DBGVCR32_EL2: write_sysreg_s(val, SYS_DBGVCR32_EL2); break;
case ZCR_EL1: write_sysreg_s(val, SYS_ZCR_EL12); break;
case SCTLR2_EL1: write_sysreg_s(val, SYS_SCTLR2_EL12); break;
default: return false;
}
@ -1387,8 +1413,6 @@ static inline bool kvm_arm_is_pvtime_enabled(struct kvm_vcpu_arch *vcpu_arch)
return (vcpu_arch->steal.base != INVALID_GPA);
}
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 syndrome);
struct kvm_vcpu *kvm_mpidr_to_vcpu(struct kvm *kvm, unsigned long mpidr);
DECLARE_KVM_HYP_PER_CPU(struct kvm_host_data, kvm_host_data);
@ -1665,6 +1689,12 @@ void kvm_set_vm_id_reg(struct kvm *kvm, u32 reg, u64 val);
#define kvm_has_s1poe(k) \
(kvm_has_feat((k), ID_AA64MMFR3_EL1, S1POE, IMP))
#define kvm_has_ras(k) \
(kvm_has_feat((k), ID_AA64PFR0_EL1, RAS, IMP))
#define kvm_has_sctlr2(k) \
(kvm_has_feat((k), ID_AA64MMFR3_EL1, SCTLRX, IMP))
static inline bool kvm_arch_has_irq_bypass(void)
{
return true;

18
arch/arm64/include/asm/kvm_mmu.h
View File

@ -371,6 +371,24 @@ static inline void kvm_fault_unlock(struct kvm *kvm)
read_unlock(&kvm->mmu_lock);
}
/*
* ARM64 KVM relies on a simple conversion from physaddr to a kernel
* virtual address (KVA) when it does cache maintenance as the CMO
* instructions work on virtual addresses. This is incompatible with
* VM_PFNMAP VMAs which may not have a kernel direct mapping to a
* virtual address.
*
* With S2FWB and CACHE DIC features, KVM need not do cache flushing
* and CMOs are NOP'd. This has the effect of no longer requiring a
* KVA for addresses mapped into the S2. The presence of these features
* are thus necessary to support cacheable S2 mapping of VM_PFNMAP.
*/
static inline bool kvm_supports_cacheable_pfnmap(void)
{
return cpus_have_final_cap(ARM64_HAS_STAGE2_FWB) &&
cpus_have_final_cap(ARM64_HAS_CACHE_DIC);
}
#ifdef CONFIG_PTDUMP_STAGE2_DEBUGFS
void kvm_s2_ptdump_create_debugfs(struct kvm *kvm);
#else

2
arch/arm64/include/asm/kvm_nested.h
View File

@ -80,6 +80,8 @@ extern void kvm_vcpu_load_hw_mmu(struct kvm_vcpu *vcpu);
extern void kvm_vcpu_put_hw_mmu(struct kvm_vcpu *vcpu);
extern void check_nested_vcpu_requests(struct kvm_vcpu *vcpu);
extern void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu);
extern void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu);
struct kvm_s2_trans {
phys_addr_t output;

24
arch/arm64/include/asm/smp.h
View File

@ -50,10 +50,32 @@ struct seq_file;
*/
extern void smp_init_cpus(void);
enum ipi_msg_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
IPI_CPU_STOP_NMI,
IPI_TIMER,
IPI_IRQ_WORK,
NR_IPI,
/*
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
* with trace_ipi_*
*/
IPI_CPU_BACKTRACE = NR_IPI,
IPI_KGDB_ROUNDUP,
MAX_IPI
};
/*
* Register IPI interrupts with the arch SMP code
*/
extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
extern void set_smp_ipi_range_percpu(int ipi_base, int nr_ipi, int ncpus);
static inline void set_smp_ipi_range(int ipi_base, int n)
{
set_smp_ipi_range_percpu(ipi_base, n, 0);
}
/*
* Called from the secondary holding pen, this is the secondary CPU entry point.

71
arch/arm64/include/asm/sysreg.h
View File

@ -113,10 +113,14 @@
/* Register-based PAN access, for save/restore purposes */
#define SYS_PSTATE_PAN sys_reg(3, 0, 4, 2, 3)
#define __SYS_BARRIER_INSN(CRm, op2, Rt) \
__emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f))
#define __SYS_BARRIER_INSN(op0, op1, CRn, CRm, op2, Rt) \
__emit_inst(0xd5000000 | \
sys_insn((op0), (op1), (CRn), (CRm), (op2)) | \
((Rt) & 0x1f))
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 7, 31)
#define SB_BARRIER_INSN __SYS_BARRIER_INSN(0, 3, 3, 0, 7, 31)
#define GSB_SYS_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 0, 31)
#define GSB_ACK_BARRIER_INSN __SYS_BARRIER_INSN(1, 0, 12, 0, 1, 31)
/* Data cache zero operations */
#define SYS_DC_ISW sys_insn(1, 0, 7, 6, 2)
@ -1078,6 +1082,67 @@
#define GCS_CAP(x) ((((unsigned long)x) & GCS_CAP_ADDR_MASK) | \
GCS_CAP_VALID_TOKEN)
/*
* Definitions for GICv5 instructions
*/
#define GICV5_OP_GIC_CDAFF sys_insn(1, 0, 12, 1, 3)
#define GICV5_OP_GIC_CDDI sys_insn(1, 0, 12, 2, 0)
#define GICV5_OP_GIC_CDDIS sys_insn(1, 0, 12, 1, 0)
#define GICV5_OP_GIC_CDHM sys_insn(1, 0, 12, 2, 1)
#define GICV5_OP_GIC_CDEN sys_insn(1, 0, 12, 1, 1)
#define GICV5_OP_GIC_CDEOI sys_insn(1, 0, 12, 1, 7)
#define GICV5_OP_GIC_CDPEND sys_insn(1, 0, 12, 1, 4)
#define GICV5_OP_GIC_CDPRI sys_insn(1, 0, 12, 1, 2)
#define GICV5_OP_GIC_CDRCFG sys_insn(1, 0, 12, 1, 5)
#define GICV5_OP_GICR_CDIA sys_insn(1, 0, 12, 3, 0)
/* Definitions for GIC CDAFF */
#define GICV5_GIC_CDAFF_IAFFID_MASK GENMASK_ULL(47, 32)
#define GICV5_GIC_CDAFF_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDAFF_IRM_MASK BIT_ULL(28)
#define GICV5_GIC_CDAFF_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDDI */
#define GICV5_GIC_CDDI_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDDI_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDDIS */
#define GICV5_GIC_CDDIS_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDDIS_TYPE(r) FIELD_GET(GICV5_GIC_CDDIS_TYPE_MASK, r)
#define GICV5_GIC_CDDIS_ID_MASK GENMASK_ULL(23, 0)
#define GICV5_GIC_CDDIS_ID(r) FIELD_GET(GICV5_GIC_CDDIS_ID_MASK, r)
/* Definitions for GIC CDEN */
#define GICV5_GIC_CDEN_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDEN_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDHM */
#define GICV5_GIC_CDHM_HM_MASK BIT_ULL(32)
#define GICV5_GIC_CDHM_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDHM_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDPEND */
#define GICV5_GIC_CDPEND_PENDING_MASK BIT_ULL(32)
#define GICV5_GIC_CDPEND_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDPEND_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDPRI */
#define GICV5_GIC_CDPRI_PRIORITY_MASK GENMASK_ULL(39, 35)
#define GICV5_GIC_CDPRI_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDPRI_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GIC CDRCFG */
#define GICV5_GIC_CDRCFG_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDRCFG_ID_MASK GENMASK_ULL(23, 0)
/* Definitions for GICR CDIA */
#define GICV5_GIC_CDIA_VALID_MASK BIT_ULL(32)
#define GICV5_GICR_CDIA_VALID(r) FIELD_GET(GICV5_GIC_CDIA_VALID_MASK, r)
#define GICV5_GIC_CDIA_TYPE_MASK GENMASK_ULL(31, 29)
#define GICV5_GIC_CDIA_ID_MASK GENMASK_ULL(23, 0)
#define gicr_insn(insn) read_sysreg_s(GICV5_OP_GICR_##insn)
#define gic_insn(v, insn) write_sysreg_s(v, GICV5_OP_GIC_##insn)
#define ARM64_FEATURE_FIELD_BITS 4

2
arch/arm64/include/asm/vncr_mapping.h
View File

@ -51,6 +51,7 @@
#define VNCR_SP_EL1 0x240
#define VNCR_VBAR_EL1 0x250
#define VNCR_TCR2_EL1 0x270
#define VNCR_SCTLR2_EL1 0x278
#define VNCR_PIRE0_EL1 0x290
#define VNCR_PIR_EL1 0x2A0
#define VNCR_POR_EL1 0x2A8
@ -84,6 +85,7 @@
#define VNCR_ICH_HCR_EL2 0x4C0
#define VNCR_ICH_VMCR_EL2 0x4C8
#define VNCR_VDISR_EL2 0x500
#define VNCR_VSESR_EL2 0x508
#define VNCR_PMBLIMITR_EL1 0x800
#define VNCR_PMBPTR_EL1 0x810
#define VNCR_PMBSR_EL1 0x820

26
arch/arm64/kernel/cpufeature.c
View File

@ -303,6 +303,7 @@ static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
};
static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_DF2_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_GCS),
FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_GCS_SHIFT, 4, 0),
S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_frac_SHIFT, 4, 0),
@ -500,6 +501,7 @@ static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = {
ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_POE),
FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),
ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
ARM64_FTR_END,
};
@ -2296,11 +2298,11 @@ static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
int scope)
{
/*
* ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU
* ARM64_HAS_GICV3_CPUIF has a lower index, and is a boot CPU
* feature, so will be detected earlier.
*/
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GIC_CPUIF_SYSREGS);
if (!cpus_have_cap(ARM64_HAS_GIC_CPUIF_SYSREGS))
BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GICV3_CPUIF);
if (!cpus_have_cap(ARM64_HAS_GICV3_CPUIF))
return false;
return enable_pseudo_nmi;
@ -2496,8 +2498,8 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_always,
},
{
.desc = "GIC system register CPU interface",
.capability = ARM64_HAS_GIC_CPUIF_SYSREGS,
.desc = "GICv3 CPU interface",
.capability = ARM64_HAS_GICV3_CPUIF,
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
.matches = has_useable_gicv3_cpuif,
ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP)
@ -3061,6 +3063,20 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
.matches = has_pmuv3,
},
#endif
{
.desc = "SCTLR2",
.capability = ARM64_HAS_SCTLR2,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, SCTLRX, IMP)
},
{
.desc = "GICv5 CPU interface",
.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
.capability = ARM64_HAS_GICV5_CPUIF,
.matches = has_cpuid_feature,
ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP)
},
{},
};

144
arch/arm64/kernel/smp.c
View File

@ -64,26 +64,18 @@ struct secondary_data secondary_data;
/* Number of CPUs which aren't online, but looping in kernel text. */
static int cpus_stuck_in_kernel;
enum ipi_msg_type {
IPI_RESCHEDULE,
IPI_CALL_FUNC,
IPI_CPU_STOP,
IPI_CPU_STOP_NMI,
IPI_TIMER,
IPI_IRQ_WORK,
NR_IPI,
/*
* Any enum >= NR_IPI and < MAX_IPI is special and not tracable
* with trace_ipi_*
*/
IPI_CPU_BACKTRACE = NR_IPI,
IPI_KGDB_ROUNDUP,
MAX_IPI
};
static int ipi_irq_base __ro_after_init;
static int nr_ipi __ro_after_init = NR_IPI;
static struct irq_desc *ipi_desc[MAX_IPI] __ro_after_init;
struct ipi_descs {
struct irq_desc *descs[MAX_IPI];
};
static DEFINE_PER_CPU_READ_MOSTLY(struct ipi_descs, pcpu_ipi_desc);
#define get_ipi_desc(__cpu, __ipi) (per_cpu_ptr(&pcpu_ipi_desc, __cpu)->descs[__ipi])
static bool percpu_ipi_descs __ro_after_init;
static bool crash_stop;
@ -844,7 +836,7 @@ int arch_show_interrupts(struct seq_file *p, int prec)
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
prec >= 4 ? " " : "");
for_each_online_cpu(cpu)
seq_printf(p, "%10u ", irq_desc_kstat_cpu(ipi_desc[i], cpu));
seq_printf(p, "%10u ", irq_desc_kstat_cpu(get_ipi_desc(cpu, i), cpu));
seq_printf(p, " %s\n", ipi_types[i]);
}
@ -917,9 +909,20 @@ static void __noreturn ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs
#endif
}
static void arm64_send_ipi(const cpumask_t *mask, unsigned int nr)
{
unsigned int cpu;
if (!percpu_ipi_descs)
__ipi_send_mask(get_ipi_desc(0, nr), mask);
else
for_each_cpu(cpu, mask)
__ipi_send_single(get_ipi_desc(cpu, nr), cpu);
}
static void arm64_backtrace_ipi(cpumask_t *mask)
{
__ipi_send_mask(ipi_desc[IPI_CPU_BACKTRACE], mask);
arm64_send_ipi(mask, IPI_CPU_BACKTRACE);
}
void arch_trigger_cpumask_backtrace(const cpumask_t *mask, int exclude_cpu)
@ -944,7 +947,7 @@ void kgdb_roundup_cpus(void)
if (cpu == this_cpu)
continue;
__ipi_send_single(ipi_desc[IPI_KGDB_ROUNDUP], cpu);
__ipi_send_single(get_ipi_desc(cpu, IPI_KGDB_ROUNDUP), cpu);
}
}
#endif
@ -1013,14 +1016,16 @@ static void do_handle_IPI(int ipinr)
static irqreturn_t ipi_handler(int irq, void *data)
{
do_handle_IPI(irq - ipi_irq_base);
unsigned int ipi = (irq - ipi_irq_base) % nr_ipi;
do_handle_IPI(ipi);
return IRQ_HANDLED;
}
static void smp_cross_call(const struct cpumask *target, unsigned int ipinr)
{
trace_ipi_raise(target, ipi_types[ipinr]);
__ipi_send_mask(ipi_desc[ipinr], target);
arm64_send_ipi(target, ipinr);
}
static bool ipi_should_be_nmi(enum ipi_msg_type ipi)
@ -1046,11 +1051,15 @@ static void ipi_setup(int cpu)
return;
for (i = 0; i < nr_ipi; i++) {
if (ipi_should_be_nmi(i)) {
prepare_percpu_nmi(ipi_irq_base + i);
enable_percpu_nmi(ipi_irq_base + i, 0);
if (!percpu_ipi_descs) {
if (ipi_should_be_nmi(i)) {
prepare_percpu_nmi(ipi_irq_base + i);
enable_percpu_nmi(ipi_irq_base + i, 0);
} else {
enable_percpu_irq(ipi_irq_base + i, 0);
}
} else {
enable_percpu_irq(ipi_irq_base + i, 0);
enable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
}
}
}
@ -1064,44 +1073,77 @@ static void ipi_teardown(int cpu)
return;
for (i = 0; i < nr_ipi; i++) {
if (ipi_should_be_nmi(i)) {
disable_percpu_nmi(ipi_irq_base + i);
teardown_percpu_nmi(ipi_irq_base + i);
if (!percpu_ipi_descs) {
if (ipi_should_be_nmi(i)) {
disable_percpu_nmi(ipi_irq_base + i);
teardown_percpu_nmi(ipi_irq_base + i);
} else {
disable_percpu_irq(ipi_irq_base + i);
}
} else {
disable_percpu_irq(ipi_irq_base + i);
disable_irq(irq_desc_get_irq(get_ipi_desc(cpu, i)));
}
}
}
#endif
void __init set_smp_ipi_range(int ipi_base, int n)
static void ipi_setup_sgi(int ipi)
{
int err, irq, cpu;
irq = ipi_irq_base + ipi;
if (ipi_should_be_nmi(ipi)) {
err = request_percpu_nmi(irq, ipi_handler, "IPI", &irq_stat);
WARN(err, "Could not request IRQ %d as NMI, err=%d\n", irq, err);
} else {
err = request_percpu_irq(irq, ipi_handler, "IPI", &irq_stat);
WARN(err, "Could not request IRQ %d as IRQ, err=%d\n", irq, err);
}
for_each_possible_cpu(cpu)
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
irq_set_status_flags(irq, IRQ_HIDDEN);
}
static void ipi_setup_lpi(int ipi, int ncpus)
{
for (int cpu = 0; cpu < ncpus; cpu++) {
int err, irq;
irq = ipi_irq_base + (cpu * nr_ipi) + ipi;
err = irq_force_affinity(irq, cpumask_of(cpu));
WARN(err, "Could not force affinity IRQ %d, err=%d\n", irq, err);
err = request_irq(irq, ipi_handler, IRQF_NO_AUTOEN, "IPI",
NULL);
WARN(err, "Could not request IRQ %d, err=%d\n", irq, err);
irq_set_status_flags(irq, (IRQ_HIDDEN | IRQ_NO_BALANCING_MASK));
get_ipi_desc(cpu, ipi) = irq_to_desc(irq);
}
}
void __init set_smp_ipi_range_percpu(int ipi_base, int n, int ncpus)
{
int i;
WARN_ON(n < MAX_IPI);
nr_ipi = min(n, MAX_IPI);
for (i = 0; i < nr_ipi; i++) {
int err;
if (ipi_should_be_nmi(i)) {
err = request_percpu_nmi(ipi_base + i, ipi_handler,
"IPI", &irq_stat);
WARN(err, "Could not request IPI %d as NMI, err=%d\n",
i, err);
} else {
err = request_percpu_irq(ipi_base + i, ipi_handler,
"IPI", &irq_stat);
WARN(err, "Could not request IPI %d as IRQ, err=%d\n",
i, err);
}
ipi_desc[i] = irq_to_desc(ipi_base + i);
irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
}
percpu_ipi_descs = !!ncpus;
ipi_irq_base = ipi_base;
for (i = 0; i < nr_ipi; i++) {
if (!percpu_ipi_descs)
ipi_setup_sgi(i);
else
ipi_setup_lpi(i, ncpus);
}
/* Setup the boot CPU immediately */
ipi_setup(smp_processor_id());
}

3
arch/arm64/kvm/Makefile
View File

@ -23,7 +23,8 @@ kvm-y += arm.o mmu.o mmio.o psci.o hypercalls.o pvtime.o \
vgic/vgic-v3.o vgic/vgic-v4.o \
vgic/vgic-mmio.o vgic/vgic-mmio-v2.o \
vgic/vgic-mmio-v3.o vgic/vgic-kvm-device.o \
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o
vgic/vgic-its.o vgic/vgic-debug.o vgic/vgic-v3-nested.o \
vgic/vgic-v5.o
kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o
kvm-$(CONFIG_ARM64_PTR_AUTH) += pauth.o

2
arch/arm64/kvm/arch_timer.c
View File

@ -830,7 +830,7 @@ static void timer_set_traps(struct kvm_vcpu *vcpu, struct timer_map *map)
* by the guest (either FEAT_VHE or FEAT_E2H0 is implemented, but
* not both). This simplifies the handling of the EL1NV* bits.
*/
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
u64 val = __vcpu_sys_reg(vcpu, CNTHCTL_EL2);
/* Use the VHE format for mental sanity */

16
arch/arm64/kvm/arm.c
View File

@ -408,6 +408,13 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
case KVM_CAP_ARM_SUPPORTED_REG_MASK_RANGES:
r = BIT(0);
break;
case KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED:
if (!kvm)
r = -EINVAL;
else
r = kvm_supports_cacheable_pfnmap();
break;
default:
r = 0;
}
@ -521,7 +528,7 @@ static void vcpu_set_pauth_traps(struct kvm_vcpu *vcpu)
* Either we're running an L2 guest, and the API/APK bits come
* from L1's HCR_EL2, or API/APK are both set.
*/
if (unlikely(vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))) {
if (unlikely(is_nested_ctxt(vcpu))) {
u64 val;
val = __vcpu_sys_reg(vcpu, HCR_EL2);
@ -740,7 +747,8 @@ int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
*/
int kvm_arch_vcpu_runnable(struct kvm_vcpu *v)
{
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF);
bool irq_lines = *vcpu_hcr(v) & (HCR_VI | HCR_VF | HCR_VSE);
return ((irq_lines || kvm_vgic_vcpu_pending_irq(v))
&& !kvm_arm_vcpu_stopped(v) && !v->arch.pause);
}
@ -1183,6 +1191,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
*/
preempt_disable();
kvm_nested_flush_hwstate(vcpu);
if (kvm_vcpu_has_pmu(vcpu))
kvm_pmu_flush_hwstate(vcpu);
@ -1282,6 +1292,8 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
/* Exit types that need handling before we can be preempted */
handle_exit_early(vcpu, ret);
kvm_nested_sync_hwstate(vcpu);
preempt_enable();
/*

80
arch/arm64/kvm/at.c
View File

@ -1047,34 +1047,51 @@ static void compute_s1_overlay_permissions(struct kvm_vcpu *vcpu,
idx = FIELD_GET(PTE_PO_IDX_MASK, wr->desc);
switch (wi->regime) {
case TR_EL10:
pov_perms = perm_idx(vcpu, POR_EL1, idx);
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL20:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL2:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
uov_perms = 0;
break;
}
if (wr->pov) {
switch (wi->regime) {
case TR_EL10:
pov_perms = perm_idx(vcpu, POR_EL1, idx);
break;
case TR_EL20:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
break;
case TR_EL2:
pov_perms = perm_idx(vcpu, POR_EL2, idx);
break;
}
if (pov_perms & ~POE_RWX)
pov_perms = POE_NONE;
if (pov_perms & ~POE_RWX)
pov_perms = POE_NONE;
/* R_QXXPC, S1PrivOverflow enabled */
if (wr->pwxn && (pov_perms & POE_X))
pov_perms &= ~POE_W;
if (wi->poe && wr->pov) {
wr->pr &= pov_perms & POE_R;
wr->pw &= pov_perms & POE_W;
wr->px &= pov_perms & POE_X;
}
if (uov_perms & ~POE_RWX)
uov_perms = POE_NONE;
if (wr->uov) {
switch (wi->regime) {
case TR_EL10:
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL20:
uov_perms = perm_idx(vcpu, POR_EL0, idx);
break;
case TR_EL2:
uov_perms = 0;
break;
}
if (uov_perms & ~POE_RWX)
uov_perms = POE_NONE;
/* R_NPBXC, S1UnprivOverlay enabled */
if (wr->uwxn && (uov_perms & POE_X))
uov_perms &= ~POE_W;
if (wi->e0poe && wr->uov) {
wr->ur &= uov_perms & POE_R;
wr->uw &= uov_perms & POE_W;
wr->ux &= uov_perms & POE_X;
@ -1095,24 +1112,15 @@ static void compute_s1_permissions(struct kvm_vcpu *vcpu,
if (!wi->hpd)
compute_s1_hierarchical_permissions(vcpu, wi, wr);
if (wi->poe || wi->e0poe)
compute_s1_overlay_permissions(vcpu, wi, wr);
compute_s1_overlay_permissions(vcpu, wi, wr);
/* R_QXXPC */
if (wr->pwxn) {
if (!wr->pov && wr->pw)
wr->px = false;
if (wr->pov && wr->px)
wr->pw = false;
}
/* R_QXXPC, S1PrivOverlay disabled */
if (!wr->pov)
wr->px &= !(wr->pwxn && wr->pw);
/* R_NPBXC */
if (wr->uwxn) {
if (!wr->uov && wr->uw)
wr->ux = false;
if (wr->uov && wr->ux)
wr->uw = false;
}
/* R_NPBXC, S1UnprivOverlay disabled */
if (!wr->uov)
wr->ux &= !(wr->uwxn && wr->uw);
pan = wi->pan && (wr->ur || wr->uw ||
(pan3_enabled(vcpu, wi->regime) && wr->ux));

255
arch/arm64/kvm/config.c
View File

@ -66,7 +66,6 @@ struct reg_bits_to_feat_map {
#define FEAT_BRBE ID_AA64DFR0_EL1, BRBE, IMP
#define FEAT_TRC_SR ID_AA64DFR0_EL1, TraceVer, IMP
#define FEAT_PMUv3 ID_AA64DFR0_EL1, PMUVer, IMP
#define FEAT_PMUv3p9 ID_AA64DFR0_EL1, PMUVer, V3P9
#define FEAT_TRBE ID_AA64DFR0_EL1, TraceBuffer, IMP
#define FEAT_TRBEv1p1 ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1
#define FEAT_DoubleLock ID_AA64DFR0_EL1, DoubleLock, IMP
@ -89,6 +88,7 @@ struct reg_bits_to_feat_map {
#define FEAT_RASv2 ID_AA64PFR0_EL1, RAS, V2
#define FEAT_GICv3 ID_AA64PFR0_EL1, GIC, IMP
#define FEAT_LOR ID_AA64MMFR1_EL1, LO, IMP
#define FEAT_SPEv1p2 ID_AA64DFR0_EL1, PMSVer, V1P2
#define FEAT_SPEv1p4 ID_AA64DFR0_EL1, PMSVer, V1P4
#define FEAT_SPEv1p5 ID_AA64DFR0_EL1, PMSVer, V1P5
#define FEAT_ATS1A ID_AA64ISAR2_EL1, ATS1A, IMP
@ -131,6 +131,27 @@ struct reg_bits_to_feat_map {
#define FEAT_SPMU ID_AA64DFR1_EL1, SPMU, IMP
#define FEAT_SPE_nVM ID_AA64DFR2_EL1, SPE_nVM, IMP
#define FEAT_STEP2 ID_AA64DFR2_EL1, STEP, IMP
#define FEAT_SYSREG128 ID_AA64ISAR2_EL1, SYSREG_128, IMP
#define FEAT_CPA2 ID_AA64ISAR3_EL1, CPA, CPA2
#define FEAT_ASID2 ID_AA64MMFR4_EL1, ASID2, IMP
#define FEAT_MEC ID_AA64MMFR3_EL1, MEC, IMP
#define FEAT_HAFT ID_AA64MMFR1_EL1, HAFDBS, HAFT
#define FEAT_BTI ID_AA64PFR1_EL1, BT, IMP
#define FEAT_ExS ID_AA64MMFR0_EL1, EXS, IMP
#define FEAT_IESB ID_AA64MMFR2_EL1, IESB, IMP
#define FEAT_LSE2 ID_AA64MMFR2_EL1, AT, IMP
#define FEAT_LSMAOC ID_AA64MMFR2_EL1, LSM, IMP
#define FEAT_MixedEnd ID_AA64MMFR0_EL1, BIGEND, IMP
#define FEAT_MixedEndEL0 ID_AA64MMFR0_EL1, BIGENDEL0, IMP
#define FEAT_MTE2 ID_AA64PFR1_EL1, MTE, MTE2
#define FEAT_MTE_ASYNC ID_AA64PFR1_EL1, MTE_frac, ASYNC
#define FEAT_MTE_STORE_ONLY ID_AA64PFR2_EL1, MTESTOREONLY, IMP
#define FEAT_PAN ID_AA64MMFR1_EL1, PAN, IMP
#define FEAT_PAN3 ID_AA64MMFR1_EL1, PAN, PAN3
#define FEAT_SSBS ID_AA64PFR1_EL1, SSBS, IMP
#define FEAT_TIDCP1 ID_AA64MMFR1_EL1, TIDCP1, IMP
#define FEAT_FGT ID_AA64MMFR0_EL1, FGT, IMP
#define FEAT_MTPMU ID_AA64DFR0_EL1, MTPMU, IMP
static bool not_feat_aa64el3(struct kvm *kvm)
{
@ -218,11 +239,62 @@ static bool feat_trbe_mpam(struct kvm *kvm)
(read_sysreg_s(SYS_TRBIDR_EL1) & TRBIDR_EL1_MPAM));
}
static bool feat_asid2_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_ASID2) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_d128_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_D128) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_mec_e2h1(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MEC) && !kvm_has_feat(kvm, FEAT_E2H0);
}
static bool feat_ebep_pmuv3_ss(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_EBEP) || kvm_has_feat(kvm, FEAT_PMUv3_SS);
}
static bool feat_mixedendel0(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MixedEnd) || kvm_has_feat(kvm, FEAT_MixedEndEL0);
}
static bool feat_mte_async(struct kvm *kvm)
{
return kvm_has_feat(kvm, FEAT_MTE2) && kvm_has_feat_enum(kvm, FEAT_MTE_ASYNC);
}
#define check_pmu_revision(k, r) \
({ \
(kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, r) && \
!kvm_has_feat((k), ID_AA64DFR0_EL1, PMUVer, IMP_DEF)); \
})
static bool feat_pmuv3p1(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P1);
}
static bool feat_pmuv3p5(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P5);
}
static bool feat_pmuv3p7(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P7);
}
static bool feat_pmuv3p9(struct kvm *kvm)
{
return check_pmu_revision(kvm, V3P9);
}
static bool compute_hcr_rw(struct kvm *kvm, u64 *bits)
{
/* This is purely academic: AArch32 and NV are mutually exclusive */
@ -681,7 +753,7 @@ static const struct reg_bits_to_feat_map hdfgrtr2_feat_map[] = {
NEEDS_FEAT(HDFGRTR2_EL2_nPMICFILTR_EL0 |
HDFGRTR2_EL2_nPMICNTR_EL0,
FEAT_PMUv3_ICNTR),
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, FEAT_PMUv3p9),
NEEDS_FEAT(HDFGRTR2_EL2_nPMUACR_EL1, feat_pmuv3p9),
NEEDS_FEAT(HDFGRTR2_EL2_nPMSSCR_EL1 |
HDFGRTR2_EL2_nPMSSDATA,
FEAT_PMUv3_SS),
@ -713,7 +785,7 @@ static const struct reg_bits_to_feat_map hdfgwtr2_feat_map[] = {
FEAT_PMUv3_ICNTR),
NEEDS_FEAT(HDFGWTR2_EL2_nPMUACR_EL1 |
HDFGWTR2_EL2_nPMZR_EL0,
FEAT_PMUv3p9),
feat_pmuv3p9),
NEEDS_FEAT(HDFGWTR2_EL2_nPMSSCR_EL1, FEAT_PMUv3_SS),
NEEDS_FEAT(HDFGWTR2_EL2_nPMIAR_EL1, FEAT_SEBEP),
NEEDS_FEAT(HDFGWTR2_EL2_nPMSDSFR_EL1, feat_spe_fds),
@ -832,6 +904,150 @@ static const struct reg_bits_to_feat_map hcr_feat_map[] = {
NEEDS_FEAT_FIXED(HCR_EL2_E2H, compute_hcr_e2h),
};
static const struct reg_bits_to_feat_map sctlr2_feat_map[] = {
NEEDS_FEAT(SCTLR2_EL1_NMEA |
SCTLR2_EL1_EASE,
FEAT_DoubleFault2),
NEEDS_FEAT(SCTLR2_EL1_EnADERR, feat_aderr),
NEEDS_FEAT(SCTLR2_EL1_EnANERR, feat_anerr),
NEEDS_FEAT(SCTLR2_EL1_EnIDCP128, FEAT_SYSREG128),
NEEDS_FEAT(SCTLR2_EL1_EnPACM |
SCTLR2_EL1_EnPACM0,
feat_pauth_lr),
NEEDS_FEAT(SCTLR2_EL1_CPTA |
SCTLR2_EL1_CPTA0 |
SCTLR2_EL1_CPTM |
SCTLR2_EL1_CPTM0,
FEAT_CPA2),
};
static const struct reg_bits_to_feat_map tcr2_el2_feat_map[] = {
NEEDS_FEAT(TCR2_EL2_FNG1 |
TCR2_EL2_FNG0 |
TCR2_EL2_A2,
feat_asid2_e2h1),
NEEDS_FEAT(TCR2_EL2_DisCH1 |
TCR2_EL2_DisCH0 |
TCR2_EL2_D128,
feat_d128_e2h1),
NEEDS_FEAT(TCR2_EL2_AMEC1, feat_mec_e2h1),
NEEDS_FEAT(TCR2_EL2_AMEC0, FEAT_MEC),
NEEDS_FEAT(TCR2_EL2_HAFT, FEAT_HAFT),
NEEDS_FEAT(TCR2_EL2_PTTWI |
TCR2_EL2_PnCH,
FEAT_THE),
NEEDS_FEAT(TCR2_EL2_AIE, FEAT_AIE),
NEEDS_FEAT(TCR2_EL2_POE |
TCR2_EL2_E0POE,
FEAT_S1POE),
NEEDS_FEAT(TCR2_EL2_PIE, FEAT_S1PIE),
};
static const struct reg_bits_to_feat_map sctlr_el1_feat_map[] = {
NEEDS_FEAT(SCTLR_EL1_CP15BEN |
SCTLR_EL1_ITD |
SCTLR_EL1_SED,
FEAT_AA32EL0),
NEEDS_FEAT(SCTLR_EL1_BT0 |
SCTLR_EL1_BT1,
FEAT_BTI),
NEEDS_FEAT(SCTLR_EL1_CMOW, FEAT_CMOW),
NEEDS_FEAT(SCTLR_EL1_TSCXT, feat_csv2_2_csv2_1p2),
NEEDS_FEAT(SCTLR_EL1_EIS |
SCTLR_EL1_EOS,
FEAT_ExS),
NEEDS_FEAT(SCTLR_EL1_EnFPM, FEAT_FPMR),
NEEDS_FEAT(SCTLR_EL1_IESB, FEAT_IESB),
NEEDS_FEAT(SCTLR_EL1_EnALS, FEAT_LS64),
NEEDS_FEAT(SCTLR_EL1_EnAS0, FEAT_LS64_ACCDATA),
NEEDS_FEAT(SCTLR_EL1_EnASR, FEAT_LS64_V),
NEEDS_FEAT(SCTLR_EL1_nAA, FEAT_LSE2),
NEEDS_FEAT(SCTLR_EL1_LSMAOE |
SCTLR_EL1_nTLSMD,
FEAT_LSMAOC),
NEEDS_FEAT(SCTLR_EL1_EE, FEAT_MixedEnd),
NEEDS_FEAT(SCTLR_EL1_E0E, feat_mixedendel0),
NEEDS_FEAT(SCTLR_EL1_MSCEn, FEAT_MOPS),
NEEDS_FEAT(SCTLR_EL1_ATA0 |
SCTLR_EL1_ATA |
SCTLR_EL1_TCF0 |
SCTLR_EL1_TCF,
FEAT_MTE2),
NEEDS_FEAT(SCTLR_EL1_ITFSB, feat_mte_async),
NEEDS_FEAT(SCTLR_EL1_TCSO0 |
SCTLR_EL1_TCSO,
FEAT_MTE_STORE_ONLY),
NEEDS_FEAT(SCTLR_EL1_NMI |
SCTLR_EL1_SPINTMASK,
FEAT_NMI),
NEEDS_FEAT(SCTLR_EL1_SPAN, FEAT_PAN),
NEEDS_FEAT(SCTLR_EL1_EPAN, FEAT_PAN3),
NEEDS_FEAT(SCTLR_EL1_EnDA |
SCTLR_EL1_EnDB |
SCTLR_EL1_EnIA |
SCTLR_EL1_EnIB,
feat_pauth),
NEEDS_FEAT(SCTLR_EL1_EnTP2, FEAT_SME),
NEEDS_FEAT(SCTLR_EL1_EnRCTX, FEAT_SPECRES),
NEEDS_FEAT(SCTLR_EL1_DSSBS, FEAT_SSBS),
NEEDS_FEAT(SCTLR_EL1_TIDCP, FEAT_TIDCP1),
NEEDS_FEAT(SCTLR_EL1_TME0 |
SCTLR_EL1_TME |
SCTLR_EL1_TMT0 |
SCTLR_EL1_TMT,
FEAT_TME),
NEEDS_FEAT(SCTLR_EL1_TWEDEL |
SCTLR_EL1_TWEDEn,
FEAT_TWED),
NEEDS_FEAT(SCTLR_EL1_UCI |
SCTLR_EL1_EE |
SCTLR_EL1_E0E |
SCTLR_EL1_WXN |
SCTLR_EL1_nTWE |
SCTLR_EL1_nTWI |
SCTLR_EL1_UCT |
SCTLR_EL1_DZE |
SCTLR_EL1_I |
SCTLR_EL1_UMA |
SCTLR_EL1_SA0 |
SCTLR_EL1_SA |
SCTLR_EL1_C |
SCTLR_EL1_A |
SCTLR_EL1_M,
FEAT_AA64EL1),
};
static const struct reg_bits_to_feat_map mdcr_el2_feat_map[] = {
NEEDS_FEAT(MDCR_EL2_EBWE, FEAT_Debugv8p9),
NEEDS_FEAT(MDCR_EL2_TDOSA, FEAT_DoubleLock),
NEEDS_FEAT(MDCR_EL2_PMEE, FEAT_EBEP),
NEEDS_FEAT(MDCR_EL2_TDCC, FEAT_FGT),
NEEDS_FEAT(MDCR_EL2_MTPME, FEAT_MTPMU),
NEEDS_FEAT(MDCR_EL2_HPME |
MDCR_EL2_HPMN |
MDCR_EL2_TPMCR |
MDCR_EL2_TPM,
FEAT_PMUv3),
NEEDS_FEAT(MDCR_EL2_HPMD, feat_pmuv3p1),
NEEDS_FEAT(MDCR_EL2_HCCD |
MDCR_EL2_HLP,
feat_pmuv3p5),
NEEDS_FEAT(MDCR_EL2_HPMFZO, feat_pmuv3p7),
NEEDS_FEAT(MDCR_EL2_PMSSE, FEAT_PMUv3_SS),
NEEDS_FEAT(MDCR_EL2_E2PB |
MDCR_EL2_TPMS,
FEAT_SPE),
NEEDS_FEAT(MDCR_EL2_HPMFZS, FEAT_SPEv1p2),
NEEDS_FEAT(MDCR_EL2_EnSPM, FEAT_SPMU),
NEEDS_FEAT(MDCR_EL2_EnSTEPOP, FEAT_STEP2),
NEEDS_FEAT(MDCR_EL2_E2TB, FEAT_TRBE),
NEEDS_FEAT(MDCR_EL2_TTRF, FEAT_TRF),
NEEDS_FEAT(MDCR_EL2_TDA |
MDCR_EL2_TDE |
MDCR_EL2_TDRA,
FEAT_AA64EL1),
};
static void __init check_feat_map(const struct reg_bits_to_feat_map *map,
int map_size, u64 res0, const char *str)
{
@ -863,6 +1079,14 @@ void __init check_feature_map(void)
__HCRX_EL2_RES0, "HCRX_EL2");
check_feat_map(hcr_feat_map, ARRAY_SIZE(hcr_feat_map),
HCR_EL2_RES0, "HCR_EL2");
check_feat_map(sctlr2_feat_map, ARRAY_SIZE(sctlr2_feat_map),
SCTLR2_EL1_RES0, "SCTLR2_EL1");
check_feat_map(tcr2_el2_feat_map, ARRAY_SIZE(tcr2_el2_feat_map),
TCR2_EL2_RES0, "TCR2_EL2");
check_feat_map(sctlr_el1_feat_map, ARRAY_SIZE(sctlr_el1_feat_map),
SCTLR_EL1_RES0, "SCTLR_EL1");
check_feat_map(mdcr_el2_feat_map, ARRAY_SIZE(mdcr_el2_feat_map),
MDCR_EL2_RES0, "MDCR_EL2");
}
static bool idreg_feat_match(struct kvm *kvm, const struct reg_bits_to_feat_map *map)
@ -1077,6 +1301,31 @@ void get_reg_fixed_bits(struct kvm *kvm, enum vcpu_sysreg reg, u64 *res0, u64 *r
*res0 |= HCR_EL2_RES0 | (mask & ~fixed);
*res1 = HCR_EL2_RES1 | (mask & fixed);
break;
case SCTLR2_EL1:
case SCTLR2_EL2:
*res0 = compute_res0_bits(kvm, sctlr2_feat_map,
ARRAY_SIZE(sctlr2_feat_map), 0, 0);
*res0 |= SCTLR2_EL1_RES0;
*res1 = SCTLR2_EL1_RES1;
break;
case TCR2_EL2:
*res0 = compute_res0_bits(kvm, tcr2_el2_feat_map,
ARRAY_SIZE(tcr2_el2_feat_map), 0, 0);
*res0 |= TCR2_EL2_RES0;
*res1 = TCR2_EL2_RES1;
break;
case SCTLR_EL1:
*res0 = compute_res0_bits(kvm, sctlr_el1_feat_map,
ARRAY_SIZE(sctlr_el1_feat_map), 0, 0);
*res0 |= SCTLR_EL1_RES0;
*res1 = SCTLR_EL1_RES1;
break;
case MDCR_EL2:
*res0 = compute_res0_bits(kvm, mdcr_el2_feat_map,
ARRAY_SIZE(mdcr_el2_feat_map), 0, 0);
*res0 |= MDCR_EL2_RES0;
*res1 = MDCR_EL2_RES1;
break;
default:
WARN_ON_ONCE(1);
*res0 = *res1 = 0;

49
arch/arm64/kvm/emulate-nested.c
View File

@ -88,6 +88,7 @@ enum cgt_group_id {
CGT_HCRX_EnFPM,
CGT_HCRX_TCR2En,
CGT_HCRX_SCTLR2En,
CGT_CNTHCTL_EL1TVT,
CGT_CNTHCTL_EL1TVCT,
@ -108,6 +109,7 @@ enum cgt_group_id {
CGT_HCR_TTLB_TTLBOS,
CGT_HCR_TVM_TRVM,
CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
CGT_HCR_TPU_TICAB,
CGT_HCR_TPU_TOCU,
CGT_HCR_NV1_nNV2_ENSCXT,
@ -398,6 +400,12 @@ static const struct trap_bits coarse_trap_bits[] = {
.mask = HCRX_EL2_TCR2En,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_HCRX_SCTLR2En] = {
.index = HCRX_EL2,
.value = 0,
.mask = HCRX_EL2_SCTLR2En,
.behaviour = BEHAVE_FORWARD_RW,
},
[CGT_CNTHCTL_EL1TVT] = {
.index = CNTHCTL_EL2,
.value = CNTHCTL_EL1TVT,
@ -449,6 +457,8 @@ static const enum cgt_group_id *coarse_control_combo[] = {
MCB(CGT_HCR_TVM_TRVM, CGT_HCR_TVM, CGT_HCR_TRVM),
MCB(CGT_HCR_TVM_TRVM_HCRX_TCR2En,
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_TCR2En),
MCB(CGT_HCR_TVM_TRVM_HCRX_SCTLR2En,
CGT_HCR_TVM, CGT_HCR_TRVM, CGT_HCRX_SCTLR2En),
MCB(CGT_HCR_TPU_TICAB, CGT_HCR_TPU, CGT_HCR_TICAB),
MCB(CGT_HCR_TPU_TOCU, CGT_HCR_TPU, CGT_HCR_TOCU),
MCB(CGT_HCR_NV1_nNV2_ENSCXT, CGT_HCR_NV1_nNV2, CGT_HCR_ENSCXT),
@ -782,6 +792,7 @@ static const struct encoding_to_trap_config encoding_to_cgt[] __initconst = {
SR_TRAP(OP_TLBI_RVALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(OP_TLBI_RVAALE1OSNXS, CGT_HCR_TTLB_TTLBOS),
SR_TRAP(SYS_SCTLR_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_SCTLR2_EL1, CGT_HCR_TVM_TRVM_HCRX_SCTLR2En),
SR_TRAP(SYS_TTBR0_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TTBR1_EL1, CGT_HCR_TVM_TRVM),
SR_TRAP(SYS_TCR_EL1, CGT_HCR_TVM_TRVM),
@ -1354,6 +1365,7 @@ static const struct encoding_to_trap_config encoding_to_fgt[] __initconst = {
SR_FGT(SYS_SCXTNUM_EL0, HFGRTR, SCXTNUM_EL0, 1),
SR_FGT(SYS_SCXTNUM_EL1, HFGRTR, SCXTNUM_EL1, 1),
SR_FGT(SYS_SCTLR_EL1, HFGRTR, SCTLR_EL1, 1),
SR_FGT(SYS_SCTLR2_EL1, HFGRTR, SCTLR_EL1, 1),
SR_FGT(SYS_REVIDR_EL1, HFGRTR, REVIDR_EL1, 1),
SR_FGT(SYS_PAR_EL1, HFGRTR, PAR_EL1, 1),
SR_FGT(SYS_MPIDR_EL1, HFGRTR, MPIDR_EL1, 1),
@ -2592,13 +2604,8 @@ inject:
static bool __forward_traps(struct kvm_vcpu *vcpu, unsigned int reg, u64 control_bit)
{
bool control_bit_set;
if (!vcpu_has_nv(vcpu))
return false;
control_bit_set = __vcpu_sys_reg(vcpu, reg) & control_bit;
if (!is_hyp_ctxt(vcpu) && control_bit_set) {
if (is_nested_ctxt(vcpu) &&
(__vcpu_sys_reg(vcpu, reg) & control_bit)) {
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return true;
}
@ -2719,6 +2726,9 @@ static void kvm_inject_el2_exception(struct kvm_vcpu *vcpu, u64 esr_el2,
case except_type_irq:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_IRQ);
break;
case except_type_serror:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
break;
default:
WARN_ONCE(1, "Unsupported EL2 exception injection %d\n", type);
}
@ -2816,3 +2826,28 @@ int kvm_inject_nested_irq(struct kvm_vcpu *vcpu)
/* esr_el2 value doesn't matter for exits due to irqs. */
return kvm_inject_nested(vcpu, 0, except_type_irq);
}
int kvm_inject_nested_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
u64 esr = FIELD_PREP(ESR_ELx_EC_MASK,
iabt ? ESR_ELx_EC_IABT_LOW : ESR_ELx_EC_DABT_LOW);
esr |= ESR_ELx_FSC_EXTABT | ESR_ELx_IL;
vcpu_write_sys_reg(vcpu, FAR_EL2, addr);
if (__vcpu_sys_reg(vcpu, SCTLR2_EL2) & SCTLR2_EL1_EASE)
return kvm_inject_nested(vcpu, esr, except_type_serror);
return kvm_inject_nested_sync(vcpu, esr);
}
int kvm_inject_nested_serror(struct kvm_vcpu *vcpu, u64 esr)
{
/*
* Hardware sets up the EC field when propagating ESR as a result of
* vSError injection. Manually populate EC for an emulated SError
* exception.
*/
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
return kvm_inject_nested(vcpu, esr, except_type_serror);
}

62
arch/arm64/kvm/guest.c
View File

@ -818,8 +818,9 @@ int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events)
{
events->exception.serror_pending = !!(vcpu->arch.hcr_el2 & HCR_VSE);
events->exception.serror_has_esr = cpus_have_final_cap(ARM64_HAS_RAS_EXTN);
events->exception.serror_pending = (vcpu->arch.hcr_el2 & HCR_VSE) ||
vcpu_get_flag(vcpu, NESTED_SERROR_PENDING);
if (events->exception.serror_pending && events->exception.serror_has_esr)
events->exception.serror_esr = vcpu_get_vsesr(vcpu);
@ -833,29 +834,62 @@ int __kvm_arm_vcpu_get_events(struct kvm_vcpu *vcpu,
return 0;
}
static void commit_pending_events(struct kvm_vcpu *vcpu)
{
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
return;
/*
* Reset the MMIO emulation state to avoid stepping PC after emulating
* the exception entry.
*/
vcpu->mmio_needed = false;
kvm_call_hyp(__kvm_adjust_pc, vcpu);
}
int __kvm_arm_vcpu_set_events(struct kvm_vcpu *vcpu,
struct kvm_vcpu_events *events)
{
bool serror_pending = events->exception.serror_pending;
bool has_esr = events->exception.serror_has_esr;
bool ext_dabt_pending = events->exception.ext_dabt_pending;
u64 esr = events->exception.serror_esr;
int ret = 0;
if (serror_pending && has_esr) {
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return -EINVAL;
if (!((events->exception.serror_esr) & ~ESR_ELx_ISS_MASK))
kvm_set_sei_esr(vcpu, events->exception.serror_esr);
else
return -EINVAL;
} else if (serror_pending) {
kvm_inject_vabt(vcpu);
/*
* Immediately commit the pending SEA to the vCPU's architectural
* state which is necessary since we do not return a pending SEA
* to userspace via KVM_GET_VCPU_EVENTS.
*/
if (ext_dabt_pending) {
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
commit_pending_events(vcpu);
}
if (ext_dabt_pending)
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
if (ret < 0)
return ret;
return 0;
if (!serror_pending)
return 0;
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && has_esr)
return -EINVAL;
if (has_esr && (esr & ~ESR_ELx_ISS_MASK))
return -EINVAL;
if (has_esr)
ret = kvm_inject_serror_esr(vcpu, esr);
else
ret = kvm_inject_serror(vcpu);
/*
* We could've decided that the SError is due for immediate software
* injection; commit the exception in case userspace decides it wants
* to inject more exceptions for some strange reason.
*/
commit_pending_events(vcpu);
return (ret < 0) ? ret : 0;
}
u32 __attribute_const__ kvm_target_cpu(void)

24
arch/arm64/kvm/handle_exit.c
View File

@ -32,7 +32,7 @@ typedef int (*exit_handle_fn)(struct kvm_vcpu *);
static void kvm_handle_guest_serror(struct kvm_vcpu *vcpu, u64 esr)
{
if (!arm64_is_ras_serror(esr) || arm64_is_fatal_ras_serror(NULL, esr))
kvm_inject_vabt(vcpu);
kvm_inject_serror(vcpu);
}
static int handle_hvc(struct kvm_vcpu *vcpu)
@ -252,7 +252,7 @@ static int kvm_handle_ptrauth(struct kvm_vcpu *vcpu)
return 1;
}
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
kvm_inject_nested_sync(vcpu, kvm_vcpu_get_esr(vcpu));
return 1;
}
@ -311,12 +311,11 @@ static int kvm_handle_gcs(struct kvm_vcpu *vcpu)
static int handle_other(struct kvm_vcpu *vcpu)
{
bool is_l2 = vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu);
bool allowed, fwd = is_nested_ctxt(vcpu);
u64 hcrx = __vcpu_sys_reg(vcpu, HCRX_EL2);
u64 esr = kvm_vcpu_get_esr(vcpu);
u64 iss = ESR_ELx_ISS(esr);
struct kvm *kvm = vcpu->kvm;
bool allowed, fwd = false;
/*
* We only trap for two reasons:
@ -335,28 +334,23 @@ static int handle_other(struct kvm_vcpu *vcpu)
switch (iss) {
case ESR_ELx_ISS_OTHER_ST64BV:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_V);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnASR);
fwd &= !(hcrx & HCRX_EL2_EnASR);
break;
case ESR_ELx_ISS_OTHER_ST64BV0:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64_ACCDATA);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnAS0);
fwd &= !(hcrx & HCRX_EL2_EnAS0);
break;
case ESR_ELx_ISS_OTHER_LDST64B:
allowed = kvm_has_feat(kvm, ID_AA64ISAR1_EL1, LS64, LS64);
if (is_l2)
fwd = !(hcrx & HCRX_EL2_EnALS);
fwd &= !(hcrx & HCRX_EL2_EnALS);
break;
case ESR_ELx_ISS_OTHER_TSBCSYNC:
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, TRBE_V1P1);
if (is_l2)
fwd = (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
fwd &= (__vcpu_sys_reg(vcpu, HFGITR2_EL2) & HFGITR2_EL2_TSBCSYNC);
break;
case ESR_ELx_ISS_OTHER_PSBCSYNC:
allowed = kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P5);
if (is_l2)
fwd = (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
fwd &= (__vcpu_sys_reg(vcpu, HFGITR_EL2) & HFGITR_EL2_PSBCSYNC);
break;
default:
/* Clearly, we're missing something. */
@ -496,7 +490,7 @@ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index)
kvm_handle_guest_serror(vcpu, disr_to_esr(disr));
} else {
kvm_inject_vabt(vcpu);
kvm_inject_serror(vcpu);
}
return;

16
arch/arm64/kvm/hyp/exception.c
View File

@ -26,7 +26,8 @@ static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
if (unlikely(vcpu_has_nv(vcpu)))
return vcpu_read_sys_reg(vcpu, reg);
else if (__vcpu_read_sys_reg_from_cpu(reg, &val))
else if (vcpu_get_flag(vcpu, SYSREGS_ON_CPU) &&
__vcpu_read_sys_reg_from_cpu(reg, &val))
return val;
return __vcpu_sys_reg(vcpu, reg);
@ -36,7 +37,8 @@ static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
{
if (unlikely(vcpu_has_nv(vcpu)))
vcpu_write_sys_reg(vcpu, val, reg);
else if (!__vcpu_write_sys_reg_to_cpu(val, reg))
else if (!vcpu_get_flag(vcpu, SYSREGS_ON_CPU) ||
!__vcpu_write_sys_reg_to_cpu(val, reg))
__vcpu_assign_sys_reg(vcpu, reg, val);
}
@ -339,6 +341,10 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu)
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_sync);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
enter_exception64(vcpu, PSR_MODE_EL1h, except_type_serror);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_sync);
break;
@ -347,9 +353,13 @@ static void kvm_inject_exception(struct kvm_vcpu *vcpu)
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_irq);
break;
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
enter_exception64(vcpu, PSR_MODE_EL2h, except_type_serror);
break;
default:
/*
* Only EL1_SYNC and EL2_{SYNC,IRQ} makes
* Only EL1_{SYNC,SERR} and EL2_{SYNC,IRQ,SERR} makes
* sense so far. Everything else gets silently
* ignored.
*/

53
arch/arm64/kvm/hyp/include/hyp/switch.h
View File

@ -298,7 +298,7 @@ static inline void __deactivate_cptr_traps(struct kvm_vcpu *vcpu)
u64 val; \
\
ctxt_sys_reg(hctxt, reg) = read_sysreg_s(SYS_ ## reg); \
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) \
if (is_nested_ctxt(vcpu)) \
compute_clr_set(vcpu, reg, c, s); \
\
compute_undef_clr_set(vcpu, kvm, reg, c, s); \
@ -436,7 +436,7 @@ static inline void __activate_traps_common(struct kvm_vcpu *vcpu)
if (cpus_have_final_cap(ARM64_HAS_HCX)) {
u64 hcrx = vcpu->arch.hcrx_el2;
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
u64 val = __vcpu_sys_reg(vcpu, HCRX_EL2);
hcrx |= val & __HCRX_EL2_MASK;
hcrx &= ~(~val & __HCRX_EL2_nMASK);
@ -476,21 +476,56 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr)
write_sysreg_hcr(hcr);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE)) {
u64 vsesr;
/*
* When HCR_EL2.AMO is set, physical SErrors are taken to EL2
* and vSError injection is enabled for EL1. Conveniently, for
* NV this means that it is never the case where a 'physical'
* SError (injected by KVM or userspace) and vSError are
* deliverable to the same context.
*
* As such, we can trivially select between the host or guest's
* VSESR_EL2. Except for the case that FEAT_RAS hasn't been
* exposed to the guest, where ESR propagation in hardware
* occurs unconditionally.
*
* Paper over the architectural wart and use an IMPLEMENTATION
* DEFINED ESR value in case FEAT_RAS is hidden from the guest.
*/
if (!vserror_state_is_nested(vcpu))
vsesr = vcpu->arch.vsesr_el2;
else if (kvm_has_ras(kern_hyp_va(vcpu->kvm)))
vsesr = __vcpu_sys_reg(vcpu, VSESR_EL2);
else
vsesr = ESR_ELx_ISV;
write_sysreg_s(vsesr, SYS_VSESR_EL2);
}
}
static inline void ___deactivate_traps(struct kvm_vcpu *vcpu)
{
u64 *hcr;
if (vserror_state_is_nested(vcpu))
hcr = __ctxt_sys_reg(&vcpu->arch.ctxt, HCR_EL2);
else
hcr = &vcpu->arch.hcr_el2;
/*
* If we pended a virtual abort, preserve it until it gets
* cleared. See D1.14.3 (Virtual Interrupts) for details, but
* the crucial bit is "On taking a vSError interrupt,
* HCR_EL2.VSE is cleared to 0."
*
* Additionally, when in a nested context we need to propagate the
* updated state to the guest hypervisor's HCR_EL2.
*/
if (vcpu->arch.hcr_el2 & HCR_VSE) {
vcpu->arch.hcr_el2 &= ~HCR_VSE;
vcpu->arch.hcr_el2 |= read_sysreg(hcr_el2) & HCR_VSE;
if (*hcr & HCR_VSE) {
*hcr &= ~HCR_VSE;
*hcr |= read_sysreg(hcr_el2) & HCR_VSE;
}
}
@ -531,7 +566,7 @@ static inline void __hyp_sve_restore_guest(struct kvm_vcpu *vcpu)
* nested guest, as the guest hypervisor could select a smaller VL. Slap
* that into hardware before wrapping up.
*/
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
if (is_nested_ctxt(vcpu))
sve_cond_update_zcr_vq(__vcpu_sys_reg(vcpu, ZCR_EL2), SYS_ZCR_EL2);
write_sysreg_el1(__vcpu_sys_reg(vcpu, vcpu_sve_zcr_elx(vcpu)), SYS_ZCR);
@ -557,7 +592,7 @@ static inline void fpsimd_lazy_switch_to_guest(struct kvm_vcpu *vcpu)
if (vcpu_has_sve(vcpu)) {
/* A guest hypervisor may restrict the effective max VL. */
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu))
if (is_nested_ctxt(vcpu))
zcr_el2 = __vcpu_sys_reg(vcpu, ZCR_EL2);
else
zcr_el2 = vcpu_sve_max_vq(vcpu) - 1;

49
arch/arm64/kvm/hyp/include/hyp/sysreg-sr.h
View File

@ -109,6 +109,28 @@ static inline bool ctxt_has_s1poe(struct kvm_cpu_context *ctxt)
return kvm_has_s1poe(kern_hyp_va(vcpu->kvm));
}
static inline bool ctxt_has_ras(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu;
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return false;
vcpu = ctxt_to_vcpu(ctxt);
return kvm_has_ras(kern_hyp_va(vcpu->kvm));
}
static inline bool ctxt_has_sctlr2(struct kvm_cpu_context *ctxt)
{
struct kvm_vcpu *vcpu;
if (!cpus_have_final_cap(ARM64_HAS_SCTLR2))
return false;
vcpu = ctxt_to_vcpu(ctxt);
return kvm_has_sctlr2(kern_hyp_va(vcpu->kvm));
}
static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
{
ctxt_sys_reg(ctxt, SCTLR_EL1) = read_sysreg_el1(SYS_SCTLR);
@ -147,6 +169,9 @@ static inline void __sysreg_save_el1_state(struct kvm_cpu_context *ctxt)
ctxt_sys_reg(ctxt, SP_EL1) = read_sysreg(sp_el1);
ctxt_sys_reg(ctxt, ELR_EL1) = read_sysreg_el1(SYS_ELR);
ctxt_sys_reg(ctxt, SPSR_EL1) = read_sysreg_el1(SYS_SPSR);
if (ctxt_has_sctlr2(ctxt))
ctxt_sys_reg(ctxt, SCTLR2_EL1) = read_sysreg_el1(SYS_SCTLR2);
}
static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
@ -159,8 +184,13 @@ static inline void __sysreg_save_el2_return_state(struct kvm_cpu_context *ctxt)
if (!has_vhe() && ctxt->__hyp_running_vcpu)
ctxt->regs.pstate = read_sysreg_el2(SYS_SPSR);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return;
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
ctxt_sys_reg(ctxt, DISR_EL1) = read_sysreg_s(SYS_VDISR_EL2);
else if (ctxt_has_ras(ctxt))
ctxt_sys_reg(ctxt, VDISR_EL2) = read_sysreg_s(SYS_VDISR_EL2);
}
static inline void __sysreg_restore_common_state(struct kvm_cpu_context *ctxt)
@ -252,6 +282,9 @@ static inline void __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt,
write_sysreg(ctxt_sys_reg(ctxt, SP_EL1), sp_el1);
write_sysreg_el1(ctxt_sys_reg(ctxt, ELR_EL1), SYS_ELR);
write_sysreg_el1(ctxt_sys_reg(ctxt, SPSR_EL1), SYS_SPSR);
if (ctxt_has_sctlr2(ctxt))
write_sysreg_el1(ctxt_sys_reg(ctxt, SCTLR2_EL1), SYS_SCTLR2);
}
/* Read the VCPU state's PSTATE, but translate (v)EL2 to EL1. */
@ -275,6 +308,7 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
{
u64 pstate = to_hw_pstate(ctxt);
u64 mode = pstate & PSR_AA32_MODE_MASK;
u64 vdisr;
/*
* Safety check to ensure we're setting the CPU up to enter the guest
@ -293,8 +327,17 @@ static inline void __sysreg_restore_el2_return_state(struct kvm_cpu_context *ctx
write_sysreg_el2(ctxt->regs.pc, SYS_ELR);
write_sysreg_el2(pstate, SYS_SPSR);
if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
write_sysreg_s(ctxt_sys_reg(ctxt, DISR_EL1), SYS_VDISR_EL2);
if (!cpus_have_final_cap(ARM64_HAS_RAS_EXTN))
return;
if (!vserror_state_is_nested(ctxt_to_vcpu(ctxt)))
vdisr = ctxt_sys_reg(ctxt, DISR_EL1);
else if (ctxt_has_ras(ctxt))
vdisr = ctxt_sys_reg(ctxt, VDISR_EL2);
else
vdisr = 0;
write_sysreg_s(vdisr, SYS_VDISR_EL2);
}
static inline void __sysreg32_save_state(struct kvm_vcpu *vcpu)

53
arch/arm64/kvm/hyp/vgic-v3-sr.c
View File

@ -296,12 +296,19 @@ void __vgic_v3_activate_traps(struct vgic_v3_cpu_if *cpu_if)
}
/*
* Prevent the guest from touching the ICC_SRE_EL1 system
* register. Note that this may not have any effect, as
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
* GICv5 BET0 FEAT_GCIE_LEGACY doesn't include ICC_SRE_EL2. This is due
* to be relaxed in a future spec release, at which point this in
* condition can be dropped.
*/
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
ICC_SRE_EL2);
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
/*
* Prevent the guest from touching the ICC_SRE_EL1 system
* register. Note that this may not have any effect, as
* ICC_SRE_EL2.Enable being RAO/WI is a valid implementation.
*/
write_gicreg(read_gicreg(ICC_SRE_EL2) & ~ICC_SRE_EL2_ENABLE,
ICC_SRE_EL2);
}
/*
* If we need to trap system registers, we must write
@ -322,8 +329,14 @@ void __vgic_v3_deactivate_traps(struct vgic_v3_cpu_if *cpu_if)
cpu_if->vgic_vmcr = read_gicreg(ICH_VMCR_EL2);
}
val = read_gicreg(ICC_SRE_EL2);
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
/*
* Can be dropped in the future when GICv5 spec is relaxed. See comment
* above.
*/
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) {
val = read_gicreg(ICC_SRE_EL2);
write_gicreg(val | ICC_SRE_EL2_ENABLE, ICC_SRE_EL2);
}
if (!cpu_if->vgic_sre) {
/* Make sure ENABLE is set at EL2 before setting SRE at EL1 */
@ -423,9 +436,19 @@ void __vgic_v3_init_lrs(void)
*/
u64 __vgic_v3_get_gic_config(void)
{
u64 val, sre = read_gicreg(ICC_SRE_EL1);
u64 val, sre;
unsigned long flags = 0;
/*
* In compat mode, we cannot access ICC_SRE_EL1 at any EL
* other than EL1 itself; just return the
* ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5
* system, so we first check if we have GICv5 support.
*/
if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
return read_gicreg(ICH_VTR_EL2);
sre = read_gicreg(ICC_SRE_EL1);
/*
* To check whether we have a MMIO-based (GICv2 compatible)
* CPU interface, we need to disable the system register
@ -471,6 +494,16 @@ u64 __vgic_v3_get_gic_config(void)
return val;
}
static void __vgic_v3_compat_mode_enable(void)
{
if (!cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF))
return;
sysreg_clear_set_s(SYS_ICH_VCTLR_EL2, 0, ICH_VCTLR_EL2_V3);
/* Wait for V3 to become enabled */
isb();
}
static u64 __vgic_v3_read_vmcr(void)
{
return read_gicreg(ICH_VMCR_EL2);
@ -490,6 +523,8 @@ void __vgic_v3_save_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if)
{
__vgic_v3_compat_mode_enable();
/*
* If dealing with a GICv2 emulation on GICv3, VMCR_EL2.VFIQen
* is dependent on ICC_SRE_EL1.SRE, and we have to perform the
@ -1050,7 +1085,7 @@ static bool __vgic_v3_check_trap_forwarding(struct kvm_vcpu *vcpu,
{
u64 ich_hcr;
if (!vcpu_has_nv(vcpu) || is_hyp_ctxt(vcpu))
if (!is_nested_ctxt(vcpu))
return false;
ich_hcr = __vcpu_sys_reg(vcpu, ICH_HCR_EL2);

14
arch/arm64/kvm/hyp/vhe/switch.c
View File

@ -48,8 +48,7 @@ DEFINE_PER_CPU(unsigned long, kvm_hyp_vector);
static u64 __compute_hcr(struct kvm_vcpu *vcpu)
{
u64 guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
u64 hcr = vcpu->arch.hcr_el2;
u64 guest_hcr, hcr = vcpu->arch.hcr_el2;
if (!vcpu_has_nv(vcpu))
return hcr;
@ -68,10 +67,21 @@ static u64 __compute_hcr(struct kvm_vcpu *vcpu)
if (!vcpu_el2_e2h_is_set(vcpu))
hcr |= HCR_NV1;
/*
* Nothing in HCR_EL2 should impact running in hypervisor
* context, apart from bits we have defined as RESx (E2H,
* HCD and co), or that cannot be set directly (the EXCLUDE
* bits). Given that we OR the guest's view with the host's,
* we can use the 0 value as the starting point, and only
* use the config-driven RES1 bits.
*/
guest_hcr = kvm_vcpu_apply_reg_masks(vcpu, HCR_EL2, 0);
write_sysreg_s(vcpu->arch.ctxt.vncr_array, SYS_VNCR_EL2);
} else {
host_data_clear_flag(VCPU_IN_HYP_CONTEXT);
guest_hcr = __vcpu_sys_reg(vcpu, HCR_EL2);
if (guest_hcr & HCR_NV) {
u64 va = __fix_to_virt(vncr_fixmap(smp_processor_id()));

6
arch/arm64/kvm/hyp/vhe/sysreg-sr.c
View File

@ -77,6 +77,9 @@ static void __sysreg_save_vel2_state(struct kvm_vcpu *vcpu)
__vcpu_assign_sys_reg(vcpu, SP_EL2, read_sysreg(sp_el1));
__vcpu_assign_sys_reg(vcpu, ELR_EL2, read_sysreg_el1(SYS_ELR));
__vcpu_assign_sys_reg(vcpu, SPSR_EL2, read_sysreg_el1(SYS_SPSR));
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
__vcpu_assign_sys_reg(vcpu, SCTLR2_EL2, read_sysreg_el1(SYS_SCTLR2));
}
static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
@ -139,6 +142,9 @@ static void __sysreg_restore_vel2_state(struct kvm_vcpu *vcpu)
write_sysreg(__vcpu_sys_reg(vcpu, SP_EL2), sp_el1);
write_sysreg_el1(__vcpu_sys_reg(vcpu, ELR_EL2), SYS_ELR);
write_sysreg_el1(__vcpu_sys_reg(vcpu, SPSR_EL2), SYS_SPSR);
if (ctxt_has_sctlr2(&vcpu->arch.ctxt))
write_sysreg_el1(__vcpu_sys_reg(vcpu, SCTLR2_EL2), SYS_SCTLR2);
}
/*

235
arch/arm64/kvm/inject_fault.c
View File

@ -15,13 +15,11 @@
#include <asm/kvm_nested.h>
#include <asm/esr.h>
static void pend_sync_exception(struct kvm_vcpu *vcpu)
static unsigned int exception_target_el(struct kvm_vcpu *vcpu)
{
/* If not nesting, EL1 is the only possible exception target */
if (likely(!vcpu_has_nv(vcpu))) {
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
return;
}
if (likely(!vcpu_has_nv(vcpu)))
return PSR_MODE_EL1h;
/*
* With NV, we need to pick between EL1 and EL2. Note that we
@ -32,26 +30,76 @@ static void pend_sync_exception(struct kvm_vcpu *vcpu)
switch(*vcpu_cpsr(vcpu) & PSR_MODE_MASK) {
case PSR_MODE_EL2h:
case PSR_MODE_EL2t:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
break;
return PSR_MODE_EL2h;
case PSR_MODE_EL1h:
case PSR_MODE_EL1t:
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
break;
return PSR_MODE_EL1h;
case PSR_MODE_EL0t:
if (vcpu_el2_tge_is_set(vcpu))
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
break;
return vcpu_el2_tge_is_set(vcpu) ? PSR_MODE_EL2h : PSR_MODE_EL1h;
default:
BUG();
}
}
static bool match_target_el(struct kvm_vcpu *vcpu, unsigned long target)
static enum vcpu_sysreg exception_esr_elx(struct kvm_vcpu *vcpu)
{
return (vcpu_get_flag(vcpu, EXCEPT_MASK) == target);
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
return ESR_EL2;
return ESR_EL1;
}
static enum vcpu_sysreg exception_far_elx(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL2h)
return FAR_EL2;
return FAR_EL1;
}
static void pend_sync_exception(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SYNC);
}
static void pend_serror_exception(struct kvm_vcpu *vcpu)
{
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SERR);
else
kvm_pend_exception(vcpu, EXCEPT_AA64_EL2_SERR);
}
static bool __effective_sctlr2_bit(struct kvm_vcpu *vcpu, unsigned int idx)
{
u64 sctlr2;
if (!kvm_has_sctlr2(vcpu->kvm))
return false;
if (is_nested_ctxt(vcpu) &&
!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_SCTLR2En))
return false;
if (exception_target_el(vcpu) == PSR_MODE_EL1h)
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL1);
else
sctlr2 = vcpu_read_sys_reg(vcpu, SCTLR2_EL2);
return sctlr2 & BIT(idx);
}
static bool effective_sctlr2_ease(struct kvm_vcpu *vcpu)
{
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_EASE_SHIFT);
}
static bool effective_sctlr2_nmea(struct kvm_vcpu *vcpu)
{
return __effective_sctlr2_bit(vcpu, SCTLR2_EL1_NMEA_SHIFT);
}
static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr)
@ -60,7 +108,11 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
bool is_aarch32 = vcpu_mode_is_32bit(vcpu);
u64 esr = 0;
pend_sync_exception(vcpu);
/* This delight is brought to you by FEAT_DoubleFault2. */
if (effective_sctlr2_ease(vcpu))
pend_serror_exception(vcpu);
else
pend_sync_exception(vcpu);
/*
* Build an {i,d}abort, depending on the level and the
@ -83,13 +135,8 @@ static void inject_abt64(struct kvm_vcpu *vcpu, bool is_iabt, unsigned long addr
esr |= ESR_ELx_FSC_EXTABT;
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC))) {
vcpu_write_sys_reg(vcpu, addr, FAR_EL1);
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
} else {
vcpu_write_sys_reg(vcpu, addr, FAR_EL2);
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
}
vcpu_write_sys_reg(vcpu, addr, exception_far_elx(vcpu));
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
static void inject_undef64(struct kvm_vcpu *vcpu)
@ -105,10 +152,7 @@ static void inject_undef64(struct kvm_vcpu *vcpu)
if (kvm_vcpu_trap_il_is32bit(vcpu))
esr |= ESR_ELx_IL;
if (match_target_el(vcpu, unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC)))
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
else
vcpu_write_sys_reg(vcpu, esr, ESR_EL2);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
#define DFSR_FSC_EXTABT_LPAE 0x10
@ -155,36 +199,35 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 addr)
vcpu_write_sys_reg(vcpu, far, FAR_EL1);
}
/**
* kvm_inject_dabt - inject a data abort into the guest
* @vcpu: The VCPU to receive the data abort
* @addr: The address to report in the DFAR
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*/
void kvm_inject_dabt(struct kvm_vcpu *vcpu, unsigned long addr)
static void __kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
if (vcpu_el1_is_32bit(vcpu))
inject_abt32(vcpu, false, addr);
inject_abt32(vcpu, iabt, addr);
else
inject_abt64(vcpu, false, addr);
inject_abt64(vcpu, iabt, addr);
}
/**
* kvm_inject_pabt - inject a prefetch abort into the guest
* @vcpu: The VCPU to receive the prefetch abort
* @addr: The address to report in the DFAR
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*/
void kvm_inject_pabt(struct kvm_vcpu *vcpu, unsigned long addr)
static bool kvm_sea_target_is_el2(struct kvm_vcpu *vcpu)
{
if (vcpu_el1_is_32bit(vcpu))
inject_abt32(vcpu, true, addr);
else
inject_abt64(vcpu, true, addr);
if (__vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_TGE | HCR_TEA))
return true;
if (!vcpu_mode_priv(vcpu))
return false;
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) &&
(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA);
}
int kvm_inject_sea(struct kvm_vcpu *vcpu, bool iabt, u64 addr)
{
lockdep_assert_held(&vcpu->mutex);
if (is_nested_ctxt(vcpu) && kvm_sea_target_is_el2(vcpu))
return kvm_inject_nested_sea(vcpu, iabt, addr);
__kvm_inject_sea(vcpu, iabt, addr);
return 1;
}
void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
@ -194,10 +237,7 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
addr = kvm_vcpu_get_fault_ipa(vcpu);
addr |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
if (kvm_vcpu_trap_is_iabt(vcpu))
kvm_inject_pabt(vcpu, addr);
else
kvm_inject_dabt(vcpu, addr);
__kvm_inject_sea(vcpu, kvm_vcpu_trap_is_iabt(vcpu), addr);
/*
* If AArch64 or LPAE, set FSC to 0 to indicate an Address
@ -210,9 +250,9 @@ void kvm_inject_size_fault(struct kvm_vcpu *vcpu)
!(vcpu_read_sys_reg(vcpu, TCR_EL1) & TTBCR_EAE))
return;
esr = vcpu_read_sys_reg(vcpu, ESR_EL1);
esr = vcpu_read_sys_reg(vcpu, exception_esr_elx(vcpu));
esr &= ~GENMASK_ULL(5, 0);
vcpu_write_sys_reg(vcpu, esr, ESR_EL1);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
}
/**
@ -230,25 +270,70 @@ void kvm_inject_undefined(struct kvm_vcpu *vcpu)
inject_undef64(vcpu);
}
void kvm_set_sei_esr(struct kvm_vcpu *vcpu, u64 esr)
static bool serror_is_masked(struct kvm_vcpu *vcpu)
{
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
*vcpu_hcr(vcpu) |= HCR_VSE;
return (*vcpu_cpsr(vcpu) & PSR_A_BIT) && !effective_sctlr2_nmea(vcpu);
}
/**
* kvm_inject_vabt - inject an async abort / SError into the guest
* @vcpu: The VCPU to receive the exception
*
* It is assumed that this code is called from the VCPU thread and that the
* VCPU therefore is not currently executing guest code.
*
* Systems with the RAS Extensions specify an imp-def ESR (ISV/IDS = 1) with
* the remaining ISS all-zeros so that this error is not interpreted as an
* uncategorized RAS error. Without the RAS Extensions we can't specify an ESR
* value, so the CPU generates an imp-def value.
*/
void kvm_inject_vabt(struct kvm_vcpu *vcpu)
static bool kvm_serror_target_is_el2(struct kvm_vcpu *vcpu)
{
kvm_set_sei_esr(vcpu, ESR_ELx_ISV);
if (is_hyp_ctxt(vcpu) || vcpu_el2_amo_is_set(vcpu))
return true;
if (!(__vcpu_sys_reg(vcpu, HCRX_EL2) & HCRX_EL2_TMEA))
return false;
/*
* In another example where FEAT_DoubleFault2 is entirely backwards,
* "masked" as it relates to the routing effects of HCRX_EL2.TMEA
* doesn't consider SCTLR2_EL1.NMEA. That is to say, even if EL1 asked
* for non-maskable SErrors, the EL2 bit takes priority if A is set.
*/
if (vcpu_mode_priv(vcpu))
return *vcpu_cpsr(vcpu) & PSR_A_BIT;
/*
* Otherwise SErrors are considered unmasked when taken from EL0 and
* NMEA is set.
*/
return serror_is_masked(vcpu);
}
static bool kvm_serror_undeliverable_at_el2(struct kvm_vcpu *vcpu)
{
return !(vcpu_el2_tge_is_set(vcpu) || vcpu_el2_amo_is_set(vcpu));
}
int kvm_inject_serror_esr(struct kvm_vcpu *vcpu, u64 esr)
{
lockdep_assert_held(&vcpu->mutex);
if (is_nested_ctxt(vcpu) && kvm_serror_target_is_el2(vcpu))
return kvm_inject_nested_serror(vcpu, esr);
if (vcpu_is_el2(vcpu) && kvm_serror_undeliverable_at_el2(vcpu)) {
vcpu_set_vsesr(vcpu, esr);
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
return 1;
}
/*
* Emulate the exception entry if SErrors are unmasked. This is useful if
* the vCPU is in a nested context w/ vSErrors enabled then we've already
* delegated he hardware vSError context (i.e. HCR_EL2.VSE, VSESR_EL2,
* VDISR_EL2) to the guest hypervisor.
*
* As we're emulating the SError injection we need to explicitly populate
* ESR_ELx.EC because hardware will not do it on our behalf.
*/
if (!serror_is_masked(vcpu)) {
pend_serror_exception(vcpu);
esr |= FIELD_PREP(ESR_ELx_EC_MASK, ESR_ELx_EC_SERROR);
vcpu_write_sys_reg(vcpu, esr, exception_esr_elx(vcpu));
return 1;
}
vcpu_set_vsesr(vcpu, esr & ESR_ELx_ISS_MASK);
*vcpu_hcr(vcpu) |= HCR_VSE;
return 1;
}

12
arch/arm64/kvm/mmio.c
View File

@ -72,7 +72,7 @@ unsigned long kvm_mmio_read_buf(const void *buf, unsigned int len)
return data;
}
static bool kvm_pending_sync_exception(struct kvm_vcpu *vcpu)
static bool kvm_pending_external_abort(struct kvm_vcpu *vcpu)
{
if (!vcpu_get_flag(vcpu, PENDING_EXCEPTION))
return false;
@ -90,6 +90,8 @@ static bool kvm_pending_sync_exception(struct kvm_vcpu *vcpu)
switch (vcpu_get_flag(vcpu, EXCEPT_MASK)) {
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SYNC):
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SYNC):
case unpack_vcpu_flag(EXCEPT_AA64_EL1_SERR):
case unpack_vcpu_flag(EXCEPT_AA64_EL2_SERR):
return true;
default:
return false;
@ -113,7 +115,7 @@ int kvm_handle_mmio_return(struct kvm_vcpu *vcpu)
* Detect if the MMIO return was already handled or if userspace aborted
* the MMIO access.
*/
if (unlikely(!vcpu->mmio_needed || kvm_pending_sync_exception(vcpu)))
if (unlikely(!vcpu->mmio_needed || kvm_pending_external_abort(vcpu)))
return 1;
vcpu->mmio_needed = 0;
@ -169,10 +171,8 @@ int io_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa)
trace_kvm_mmio_nisv(*vcpu_pc(vcpu), kvm_vcpu_get_esr(vcpu),
kvm_vcpu_get_hfar(vcpu), fault_ipa);
if (vcpu_is_protected(vcpu)) {
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
return 1;
}
if (vcpu_is_protected(vcpu))
return kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
if (test_bit(KVM_ARCH_FLAG_RETURN_NISV_IO_ABORT_TO_USER,
&vcpu->kvm->arch.flags)) {

105
arch/arm64/kvm/mmu.c
View File

@ -193,11 +193,6 @@ int kvm_arch_flush_remote_tlbs_range(struct kvm *kvm,
return 0;
}
static bool kvm_is_device_pfn(unsigned long pfn)
{
return !pfn_is_map_memory(pfn);
}
static void *stage2_memcache_zalloc_page(void *arg)
{
struct kvm_mmu_memory_cache *mc = arg;
@ -1470,6 +1465,18 @@ static bool kvm_vma_mte_allowed(struct vm_area_struct *vma)
return vma->vm_flags & VM_MTE_ALLOWED;
}
static bool kvm_vma_is_cacheable(struct vm_area_struct *vma)
{
switch (FIELD_GET(PTE_ATTRINDX_MASK, pgprot_val(vma->vm_page_prot))) {
case MT_NORMAL_NC:
case MT_DEVICE_nGnRnE:
case MT_DEVICE_nGnRE:
return false;
default:
return true;
}
}
static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
struct kvm_s2_trans *nested,
struct kvm_memory_slot *memslot, unsigned long hva,
@ -1477,8 +1484,8 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
{
int ret = 0;
bool write_fault, writable, force_pte = false;
bool exec_fault, mte_allowed;
bool device = false, vfio_allow_any_uc = false;
bool exec_fault, mte_allowed, is_vma_cacheable;
bool s2_force_noncacheable = false, vfio_allow_any_uc = false;
unsigned long mmu_seq;
phys_addr_t ipa = fault_ipa;
struct kvm *kvm = vcpu->kvm;
@ -1492,6 +1499,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
enum kvm_pgtable_prot prot = KVM_PGTABLE_PROT_R;
struct kvm_pgtable *pgt;
struct page *page;
vm_flags_t vm_flags;
enum kvm_pgtable_walk_flags flags = KVM_PGTABLE_WALK_HANDLE_FAULT | KVM_PGTABLE_WALK_SHARED;
if (fault_is_perm)
@ -1619,6 +1627,10 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
vfio_allow_any_uc = vma->vm_flags & VM_ALLOW_ANY_UNCACHED;
vm_flags = vma->vm_flags;
is_vma_cacheable = kvm_vma_is_cacheable(vma);
/* Don't use the VMA after the unlock -- it may have vanished */
vma = NULL;
@ -1642,18 +1654,39 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (is_error_noslot_pfn(pfn))
return -EFAULT;
if (kvm_is_device_pfn(pfn)) {
/*
* If the page was identified as device early by looking at
* the VMA flags, vma_pagesize is already representing the
* largest quantity we can map. If instead it was mapped
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
* and must not be upgraded.
*
* In both cases, we don't let transparent_hugepage_adjust()
* change things at the last minute.
*/
device = true;
/*
* Check if this is non-struct page memory PFN, and cannot support
* CMOs. It could potentially be unsafe to access as cachable.
*/
if (vm_flags & (VM_PFNMAP | VM_MIXEDMAP) && !pfn_is_map_memory(pfn)) {
if (is_vma_cacheable) {
/*
* Whilst the VMA owner expects cacheable mapping to this
* PFN, hardware also has to support the FWB and CACHE DIC
* features.
*
* ARM64 KVM relies on kernel VA mapping to the PFN to
* perform cache maintenance as the CMO instructions work on
* virtual addresses. VM_PFNMAP region are not necessarily
* mapped to a KVA and hence the presence of hardware features
* S2FWB and CACHE DIC are mandatory to avoid the need for
* cache maintenance.
*/
if (!kvm_supports_cacheable_pfnmap())
return -EFAULT;
} else {
/*
* If the page was identified as device early by looking at
* the VMA flags, vma_pagesize is already representing the
* largest quantity we can map. If instead it was mapped
* via __kvm_faultin_pfn(), vma_pagesize is set to PAGE_SIZE
* and must not be upgraded.
*
* In both cases, we don't let transparent_hugepage_adjust()
* change things at the last minute.
*/
s2_force_noncacheable = true;
}
} else if (logging_active && !write_fault) {
/*
* Only actually map the page as writable if this was a write
@ -1662,7 +1695,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
writable = false;
}
if (exec_fault && device)
if (exec_fault && s2_force_noncacheable)
return -ENOEXEC;
/*
@ -1695,7 +1728,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
* If we are not forced to use page mapping, check if we are
* backed by a THP and thus use block mapping if possible.
*/
if (vma_pagesize == PAGE_SIZE && !(force_pte || device)) {
if (vma_pagesize == PAGE_SIZE && !(force_pte || s2_force_noncacheable)) {
if (fault_is_perm && fault_granule > PAGE_SIZE)
vma_pagesize = fault_granule;
else
@ -1709,7 +1742,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
}
}
if (!fault_is_perm && !device && kvm_has_mte(kvm)) {
if (!fault_is_perm && !s2_force_noncacheable && kvm_has_mte(kvm)) {
/* Check the VMM hasn't introduced a new disallowed VMA */
if (mte_allowed) {
sanitise_mte_tags(kvm, pfn, vma_pagesize);
@ -1725,7 +1758,7 @@ static int user_mem_abort(struct kvm_vcpu *vcpu, phys_addr_t fault_ipa,
if (exec_fault)
prot |= KVM_PGTABLE_PROT_X;
if (device) {
if (s2_force_noncacheable) {
if (vfio_allow_any_uc)
prot |= KVM_PGTABLE_PROT_NORMAL_NC;
else
@ -1808,7 +1841,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
* There is no need to pass the error into the guest.
*/
if (kvm_handle_guest_sea())
kvm_inject_vabt(vcpu);
return kvm_inject_serror(vcpu);
return 1;
}
@ -1836,11 +1869,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
if (fault_ipa >= BIT_ULL(VTCR_EL2_IPA(vcpu->arch.hw_mmu->vtcr))) {
fault_ipa |= kvm_vcpu_get_hfar(vcpu) & GENMASK(11, 0);
if (is_iabt)
kvm_inject_pabt(vcpu, fault_ipa);
else
kvm_inject_dabt(vcpu, fault_ipa);
return 1;
return kvm_inject_sea(vcpu, is_iabt, fault_ipa);
}
}
@ -1912,8 +1941,7 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
}
if (kvm_vcpu_abt_iss1tw(vcpu)) {
kvm_inject_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
ret = 1;
ret = kvm_inject_sea_dabt(vcpu, kvm_vcpu_get_hfar(vcpu));
goto out_unlock;
}
@ -1958,10 +1986,8 @@ int kvm_handle_guest_abort(struct kvm_vcpu *vcpu)
if (ret == 0)
ret = 1;
out:
if (ret == -ENOEXEC) {
kvm_inject_pabt(vcpu, kvm_vcpu_get_hfar(vcpu));
ret = 1;
}
if (ret == -ENOEXEC)
ret = kvm_inject_sea_iabt(vcpu, kvm_vcpu_get_hfar(vcpu));
out_unlock:
srcu_read_unlock(&vcpu->kvm->srcu, idx);
return ret;
@ -2221,6 +2247,15 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm,
ret = -EINVAL;
break;
}
/*
* Cacheable PFNMAP is allowed only if the hardware
* supports it.
*/
if (kvm_vma_is_cacheable(vma) && !kvm_supports_cacheable_pfnmap()) {
ret = -EINVAL;
break;
}
}
hva = min(reg_end, vma->vm_end);
} while (hva < reg_end);

109
arch/arm64/kvm/nested.c
View File

@ -1441,12 +1441,11 @@ u64 limit_nv_id_reg(struct kvm *kvm, u32 reg, u64 val)
break;
case SYS_ID_AA64PFR0_EL1:
/* No RME, AMU, MPAM, S-EL2, or RAS */
/* No RME, AMU, MPAM, or S-EL2 */
val &= ~(ID_AA64PFR0_EL1_RME |
ID_AA64PFR0_EL1_AMU |
ID_AA64PFR0_EL1_MPAM |
ID_AA64PFR0_EL1_SEL2 |
ID_AA64PFR0_EL1_RAS |
ID_AA64PFR0_EL1_EL3 |
ID_AA64PFR0_EL1_EL2 |
ID_AA64PFR0_EL1_EL1 |
@ -1683,69 +1682,21 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu)
set_sysreg_masks(kvm, HFGITR2_EL2, res0, res1);
/* TCR2_EL2 */
res0 = TCR2_EL2_RES0;
res1 = TCR2_EL2_RES1;
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, D128, IMP))
res0 |= (TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1 | TCR2_EL2_D128);
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, MEC, IMP))
res0 |= TCR2_EL2_AMEC1 | TCR2_EL2_AMEC0;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, HAFDBS, HAFT))
res0 |= TCR2_EL2_HAFT;
if (!kvm_has_feat(kvm, ID_AA64PFR1_EL1, THE, IMP))
res0 |= TCR2_EL2_PTTWI | TCR2_EL2_PnCH;
if (!kvm_has_feat(kvm, ID_AA64MMFR3_EL1, AIE, IMP))
res0 |= TCR2_EL2_AIE;
if (!kvm_has_s1poe(kvm))
res0 |= TCR2_EL2_POE | TCR2_EL2_E0POE;
if (!kvm_has_s1pie(kvm))
res0 |= TCR2_EL2_PIE;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, VH, IMP))
res0 |= (TCR2_EL2_E0POE | TCR2_EL2_D128 |
TCR2_EL2_AMEC1 | TCR2_EL2_DisCH0 | TCR2_EL2_DisCH1);
get_reg_fixed_bits(kvm, TCR2_EL2, &res0, &res1);
set_sysreg_masks(kvm, TCR2_EL2, res0, res1);
/* SCTLR_EL1 */
res0 = SCTLR_EL1_RES0;
res1 = SCTLR_EL1_RES1;
if (!kvm_has_feat(kvm, ID_AA64MMFR1_EL1, PAN, PAN3))
res0 |= SCTLR_EL1_EPAN;
get_reg_fixed_bits(kvm, SCTLR_EL1, &res0, &res1);
set_sysreg_masks(kvm, SCTLR_EL1, res0, res1);
/* SCTLR2_ELx */
get_reg_fixed_bits(kvm, SCTLR2_EL1, &res0, &res1);
set_sysreg_masks(kvm, SCTLR2_EL1, res0, res1);
get_reg_fixed_bits(kvm, SCTLR2_EL2, &res0, &res1);
set_sysreg_masks(kvm, SCTLR2_EL2, res0, res1);
/* MDCR_EL2 */
res0 = MDCR_EL2_RES0;
res1 = MDCR_EL2_RES1;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, IMP))
res0 |= (MDCR_EL2_HPMN | MDCR_EL2_TPMCR |
MDCR_EL2_TPM | MDCR_EL2_HPME);
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, IMP))
res0 |= MDCR_EL2_E2PB | MDCR_EL2_TPMS;
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, SPMU, IMP))
res0 |= MDCR_EL2_EnSPM;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P1))
res0 |= MDCR_EL2_HPMD;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceFilt, IMP))
res0 |= MDCR_EL2_TTRF;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P5))
res0 |= MDCR_EL2_HCCD | MDCR_EL2_HLP;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, TraceBuffer, IMP))
res0 |= MDCR_EL2_E2TB;
if (!kvm_has_feat(kvm, ID_AA64MMFR0_EL1, FGT, IMP))
res0 |= MDCR_EL2_TDCC;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, MTPMU, IMP) ||
kvm_has_feat(kvm, ID_AA64PFR0_EL1, EL3, IMP))
res0 |= MDCR_EL2_MTPME;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMUVer, V3P7))
res0 |= MDCR_EL2_HPMFZO;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSS, IMP))
res0 |= MDCR_EL2_PMSSE;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, PMSVer, V1P2))
res0 |= MDCR_EL2_HPMFZS;
if (!kvm_has_feat(kvm, ID_AA64DFR1_EL1, EBEP, IMP))
res0 |= MDCR_EL2_PMEE;
if (!kvm_has_feat(kvm, ID_AA64DFR0_EL1, DebugVer, V8P9))
res0 |= MDCR_EL2_EBWE;
if (!kvm_has_feat(kvm, ID_AA64DFR2_EL1, STEP, IMP))
res0 |= MDCR_EL2_EnSTEPOP;
get_reg_fixed_bits(kvm, MDCR_EL2, &res0, &res1);
set_sysreg_masks(kvm, MDCR_EL2, res0, res1);
/* CNTHCTL_EL2 */
@ -1802,3 +1753,43 @@ void check_nested_vcpu_requests(struct kvm_vcpu *vcpu)
if (kvm_check_request(KVM_REQ_GUEST_HYP_IRQ_PENDING, vcpu))
kvm_inject_nested_irq(vcpu);
}
/*
* One of the many architectural bugs in FEAT_NV2 is that the guest hypervisor
* can write to HCR_EL2 behind our back, potentially changing the exception
* routing / masking for even the host context.
*
* What follows is some slop to (1) react to exception routing / masking and (2)
* preserve the pending SError state across translation regimes.
*/
void kvm_nested_flush_hwstate(struct kvm_vcpu *vcpu)
{
if (!vcpu_has_nv(vcpu))
return;
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
}
void kvm_nested_sync_hwstate(struct kvm_vcpu *vcpu)
{
unsigned long *hcr = vcpu_hcr(vcpu);
if (!vcpu_has_nv(vcpu))
return;
/*
* We previously decided that an SError was deliverable to the guest.
* Reap the pending state from HCR_EL2 and...
*/
if (unlikely(__test_and_clear_bit(__ffs(HCR_VSE), hcr)))
vcpu_set_flag(vcpu, NESTED_SERROR_PENDING);
/*
* Re-attempt SError injection in case the deliverability has changed,
* which is necessary to faithfully emulate WFI the case of a pending
* SError being a wakeup condition.
*/
if (unlikely(vcpu_test_and_clear_flag(vcpu, NESTED_SERROR_PENDING)))
kvm_inject_serror_esr(vcpu, vcpu_get_vsesr(vcpu));
}

205
arch/arm64/kvm/sys_regs.c
View File

@ -108,7 +108,6 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
PURE_EL2_SYSREG( HACR_EL2 );
PURE_EL2_SYSREG( VTTBR_EL2 );
PURE_EL2_SYSREG( VTCR_EL2 );
PURE_EL2_SYSREG( RVBAR_EL2 );
PURE_EL2_SYSREG( TPIDR_EL2 );
PURE_EL2_SYSREG( HPFAR_EL2 );
PURE_EL2_SYSREG( HCRX_EL2 );
@ -144,6 +143,7 @@ static bool get_el2_to_el1_mapping(unsigned int reg,
MAPPED_EL2_SYSREG(SPSR_EL2, SPSR_EL1, NULL );
MAPPED_EL2_SYSREG(ZCR_EL2, ZCR_EL1, NULL );
MAPPED_EL2_SYSREG(CONTEXTIDR_EL2, CONTEXTIDR_EL1, NULL );
MAPPED_EL2_SYSREG(SCTLR2_EL2, SCTLR2_EL1, NULL );
default:
return false;
}
@ -533,8 +533,7 @@ static bool access_gic_sre(struct kvm_vcpu *vcpu,
return ignore_write(vcpu, p);
if (p->Op1 == 4) { /* ICC_SRE_EL2 */
p->regval = (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE |
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB);
p->regval = KVM_ICC_SRE_EL2;
} else { /* ICC_SRE_EL1 */
p->regval = vcpu->arch.vgic_cpu.vgic_v3.vgic_sre;
}
@ -773,6 +772,12 @@ static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r)
return mpidr;
}
static unsigned int hidden_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
return REG_HIDDEN;
}
static unsigned int pmu_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *r)
{
@ -1612,7 +1617,6 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_GCS);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_THE);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MTEX);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_DF2);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_PFAR);
val &= ~ARM64_FEATURE_MASK(ID_AA64PFR1_EL1_MPAM_frac);
break;
@ -1643,8 +1647,10 @@ static u64 __kvm_read_sanitised_id_reg(const struct kvm_vcpu *vcpu,
val &= ~ID_AA64MMFR2_EL1_NV;
break;
case SYS_ID_AA64MMFR3_EL1:
val &= ID_AA64MMFR3_EL1_TCRX | ID_AA64MMFR3_EL1_S1POE |
ID_AA64MMFR3_EL1_S1PIE;
val &= ID_AA64MMFR3_EL1_TCRX |
ID_AA64MMFR3_EL1_SCTLRX |
ID_AA64MMFR3_EL1_S1POE |
ID_AA64MMFR3_EL1_S1PIE;
break;
case SYS_ID_MMFR4_EL1:
val &= ~ARM64_FEATURE_MASK(ID_MMFR4_EL1_CCIDX);
@ -1811,7 +1817,7 @@ static u64 sanitise_id_aa64pfr0_el1(const struct kvm_vcpu *vcpu, u64 val)
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, CSV3, IMP);
}
if (kvm_vgic_global_state.type == VGIC_V3) {
if (vgic_is_v3(vcpu->kvm)) {
val &= ~ID_AA64PFR0_EL1_GIC_MASK;
val |= SYS_FIELD_PREP_ENUM(ID_AA64PFR0_EL1, GIC, IMP);
}
@ -1953,6 +1959,14 @@ static int set_id_aa64pfr0_el1(struct kvm_vcpu *vcpu,
(vcpu_has_nv(vcpu) && !FIELD_GET(ID_AA64PFR0_EL1_EL2, user_val)))
return -EINVAL;
/*
* If we are running on a GICv5 host and support FEAT_GCIE_LEGACY, then
* we support GICv3. Fail attempts to do anything but set that to IMP.
*/
if (vgic_is_v3_compat(vcpu->kvm) &&
FIELD_GET(ID_AA64PFR0_EL1_GIC_MASK, user_val) != ID_AA64PFR0_EL1_GIC_IMP)
return -EINVAL;
return set_id_reg(vcpu, rd, user_val);
}
@ -2325,6 +2339,10 @@ static bool bad_redir_trap(struct kvm_vcpu *vcpu,
EL2_REG_FILTERED(name, acc, rst, v, el2_visibility)
#define EL2_REG_VNCR(name, rst, v) EL2_REG(name, bad_vncr_trap, rst, v)
#define EL2_REG_VNCR_FILT(name, vis) \
EL2_REG_FILTERED(name, bad_vncr_trap, reset_val, 0, vis)
#define EL2_REG_VNCR_GICv3(name) \
EL2_REG_VNCR_FILT(name, hidden_visibility)
#define EL2_REG_REDIR(name, rst, v) EL2_REG(name, bad_redir_trap, rst, v)
/*
@ -2483,6 +2501,21 @@ static unsigned int vncr_el2_visibility(const struct kvm_vcpu *vcpu,
return REG_HIDDEN;
}
static unsigned int sctlr2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (kvm_has_sctlr2(vcpu->kvm))
return 0;
return REG_HIDDEN;
}
static unsigned int sctlr2_el2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
return __el2_visibility(vcpu, rd, sctlr2_visibility);
}
static bool access_zcr_el2(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
@ -2513,11 +2546,7 @@ static bool access_gic_vtr(struct kvm_vcpu *vcpu,
if (p->is_write)
return write_to_read_only(vcpu, p, r);
p->regval = kvm_vgic_global_state.ich_vtr_el2;
p->regval &= ~(ICH_VTR_EL2_DVIM |
ICH_VTR_EL2_A3V |
ICH_VTR_EL2_IDbits);
p->regval |= ICH_VTR_EL2_nV4;
p->regval = kvm_get_guest_vtr_el2();
return true;
}
@ -2588,6 +2617,26 @@ static unsigned int tcr2_el2_visibility(const struct kvm_vcpu *vcpu,
return __el2_visibility(vcpu, rd, tcr2_visibility);
}
static unsigned int fgt2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (el2_visibility(vcpu, rd) == 0 &&
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, FGT2))
return 0;
return REG_HIDDEN;
}
static unsigned int fgt_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
if (el2_visibility(vcpu, rd) == 0 &&
kvm_has_feat(vcpu->kvm, ID_AA64MMFR0_EL1, FGT, IMP))
return 0;
return REG_HIDDEN;
}
static unsigned int s1pie_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
@ -2639,6 +2688,23 @@ static bool access_mdcr(struct kvm_vcpu *vcpu,
return true;
}
static bool access_ras(struct kvm_vcpu *vcpu,
struct sys_reg_params *p,
const struct sys_reg_desc *r)
{
struct kvm *kvm = vcpu->kvm;
switch(reg_to_encoding(r)) {
default:
if (!kvm_has_feat(kvm, ID_AA64PFR0_EL1, RAS, IMP)) {
kvm_inject_undefined(vcpu);
return false;
}
}
return trap_raz_wi(vcpu, p, r);
}
/*
* For historical (ahem ABI) reasons, KVM treated MIDR_EL1, REVIDR_EL1, and
* AIDR_EL1 as "invariant" registers, meaning userspace cannot change them.
@ -2866,7 +2932,6 @@ static const struct sys_reg_desc sys_reg_descs[] = {
ID_AA64PFR0_EL1_FP)),
ID_FILTERED(ID_AA64PFR1_EL1, id_aa64pfr1_el1,
~(ID_AA64PFR1_EL1_PFAR |
ID_AA64PFR1_EL1_DF2 |
ID_AA64PFR1_EL1_MTEX |
ID_AA64PFR1_EL1_THE |
ID_AA64PFR1_EL1_GCS |
@ -2945,6 +3010,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
ID_AA64MMFR2_EL1_NV |
ID_AA64MMFR2_EL1_CCIDX)),
ID_WRITABLE(ID_AA64MMFR3_EL1, (ID_AA64MMFR3_EL1_TCRX |
ID_AA64MMFR3_EL1_SCTLRX |
ID_AA64MMFR3_EL1_S1PIE |
ID_AA64MMFR3_EL1_S1POE)),
ID_WRITABLE(ID_AA64MMFR4_EL1, ID_AA64MMFR4_EL1_NV_frac),
@ -2955,6 +3021,8 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_SCTLR_EL1), access_vm_reg, reset_val, SCTLR_EL1, 0x00C50078 },
{ SYS_DESC(SYS_ACTLR_EL1), access_actlr, reset_actlr, ACTLR_EL1 },
{ SYS_DESC(SYS_CPACR_EL1), NULL, reset_val, CPACR_EL1, 0 },
{ SYS_DESC(SYS_SCTLR2_EL1), access_vm_reg, reset_val, SCTLR2_EL1, 0,
.visibility = sctlr2_visibility },
MTE_REG(RGSR_EL1),
MTE_REG(GCR_EL1),
@ -2984,14 +3052,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_AFSR1_EL1), access_vm_reg, reset_unknown, AFSR1_EL1 },
{ SYS_DESC(SYS_ESR_EL1), access_vm_reg, reset_unknown, ESR_EL1 },
{ SYS_DESC(SYS_ERRIDR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERRSELR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXFR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXCTLR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXSTATUS_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXADDR_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXMISC0_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERXMISC1_EL1), trap_raz_wi },
{ SYS_DESC(SYS_ERRIDR_EL1), access_ras },
{ SYS_DESC(SYS_ERRSELR_EL1), access_ras },
{ SYS_DESC(SYS_ERXFR_EL1), access_ras },
{ SYS_DESC(SYS_ERXCTLR_EL1), access_ras },
{ SYS_DESC(SYS_ERXSTATUS_EL1), access_ras },
{ SYS_DESC(SYS_ERXADDR_EL1), access_ras },
{ SYS_DESC(SYS_ERXMISC0_EL1), access_ras },
{ SYS_DESC(SYS_ERXMISC1_EL1), access_ras },
MTE_REG(TFSR_EL1),
MTE_REG(TFSRE0_EL1),
@ -3302,12 +3370,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
EL2_REG_VNCR(VMPIDR_EL2, reset_unknown, 0),
EL2_REG(SCTLR_EL2, access_rw, reset_val, SCTLR_EL2_RES1),
EL2_REG(ACTLR_EL2, access_rw, reset_val, 0),
EL2_REG_FILTERED(SCTLR2_EL2, access_vm_reg, reset_val, 0,
sctlr2_el2_visibility),
EL2_REG_VNCR(HCR_EL2, reset_hcr, 0),
EL2_REG(MDCR_EL2, access_mdcr, reset_mdcr, 0),
EL2_REG(CPTR_EL2, access_rw, reset_val, CPTR_NVHE_EL2_RES1),
EL2_REG_VNCR(HSTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR_FILT(HFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HFGWTR_EL2, fgt_visibility),
EL2_REG_VNCR(HFGITR_EL2, reset_val, 0),
EL2_REG_VNCR(HACR_EL2, reset_val, 0),
@ -3327,9 +3397,14 @@ static const struct sys_reg_desc sys_reg_descs[] = {
vncr_el2_visibility),
{ SYS_DESC(SYS_DACR32_EL2), undef_access, reset_unknown, DACR32_EL2 },
EL2_REG_VNCR(HDFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR(HDFGWTR_EL2, reset_val, 0),
EL2_REG_VNCR(HAFGRTR_EL2, reset_val, 0),
EL2_REG_VNCR_FILT(HDFGRTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HDFGWTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HFGRTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HFGWTR2_EL2, fgt2_visibility),
EL2_REG_VNCR_FILT(HDFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HDFGWTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HAFGRTR_EL2, fgt_visibility),
EL2_REG_VNCR_FILT(HFGITR2_EL2, fgt2_visibility),
EL2_REG_REDIR(SPSR_EL2, reset_val, 0),
EL2_REG_REDIR(ELR_EL2, reset_val, 0),
{ SYS_DESC(SYS_SP_EL1), access_sp_el1},
@ -3344,6 +3419,7 @@ static const struct sys_reg_desc sys_reg_descs[] = {
EL2_REG(AFSR0_EL2, access_rw, reset_val, 0),
EL2_REG(AFSR1_EL2, access_rw, reset_val, 0),
EL2_REG_REDIR(ESR_EL2, reset_val, 0),
EL2_REG_VNCR(VSESR_EL2, reset_unknown, 0),
{ SYS_DESC(SYS_FPEXC32_EL2), undef_access, reset_val, FPEXC32_EL2, 0x700 },
EL2_REG_REDIR(FAR_EL2, reset_val, 0),
@ -3370,43 +3446,44 @@ static const struct sys_reg_desc sys_reg_descs[] = {
{ SYS_DESC(SYS_MPAMVPM7_EL2), undef_access },
EL2_REG(VBAR_EL2, access_rw, reset_val, 0),
EL2_REG(RVBAR_EL2, access_rw, reset_val, 0),
{ SYS_DESC(SYS_RVBAR_EL2), undef_access },
{ SYS_DESC(SYS_RMR_EL2), undef_access },
EL2_REG_VNCR(VDISR_EL2, reset_unknown, 0),
EL2_REG_VNCR(ICH_AP0R0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP0R3_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_AP1R3_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_AP0R0_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R1_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R2_EL2),
EL2_REG_VNCR_GICv3(ICH_AP0R3_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R0_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R1_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R2_EL2),
EL2_REG_VNCR_GICv3(ICH_AP1R3_EL2),
{ SYS_DESC(SYS_ICC_SRE_EL2), access_gic_sre },
EL2_REG_VNCR(ICH_HCR_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_HCR_EL2),
{ SYS_DESC(SYS_ICH_VTR_EL2), access_gic_vtr },
{ SYS_DESC(SYS_ICH_MISR_EL2), access_gic_misr },
{ SYS_DESC(SYS_ICH_EISR_EL2), access_gic_eisr },
{ SYS_DESC(SYS_ICH_ELRSR_EL2), access_gic_elrsr },
EL2_REG_VNCR(ICH_VMCR_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_VMCR_EL2),
EL2_REG_VNCR(ICH_LR0_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR1_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR2_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR3_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR4_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR5_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR6_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR7_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR8_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR9_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR10_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR11_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR12_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR13_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR14_EL2, reset_val, 0),
EL2_REG_VNCR(ICH_LR15_EL2, reset_val, 0),
EL2_REG_VNCR_GICv3(ICH_LR0_EL2),
EL2_REG_VNCR_GICv3(ICH_LR1_EL2),
EL2_REG_VNCR_GICv3(ICH_LR2_EL2),
EL2_REG_VNCR_GICv3(ICH_LR3_EL2),
EL2_REG_VNCR_GICv3(ICH_LR4_EL2),
EL2_REG_VNCR_GICv3(ICH_LR5_EL2),
EL2_REG_VNCR_GICv3(ICH_LR6_EL2),
EL2_REG_VNCR_GICv3(ICH_LR7_EL2),
EL2_REG_VNCR_GICv3(ICH_LR8_EL2),
EL2_REG_VNCR_GICv3(ICH_LR9_EL2),
EL2_REG_VNCR_GICv3(ICH_LR10_EL2),
EL2_REG_VNCR_GICv3(ICH_LR11_EL2),
EL2_REG_VNCR_GICv3(ICH_LR12_EL2),
EL2_REG_VNCR_GICv3(ICH_LR13_EL2),
EL2_REG_VNCR_GICv3(ICH_LR14_EL2),
EL2_REG_VNCR_GICv3(ICH_LR15_EL2),
EL2_REG(CONTEXTIDR_EL2, access_rw, reset_val, 0),
EL2_REG(TPIDR_EL2, access_rw, reset_val, 0),
@ -4275,12 +4352,12 @@ static const struct sys_reg_desc cp15_64_regs[] = {
};
static bool check_sysreg_table(const struct sys_reg_desc *table, unsigned int n,
bool is_32)
bool reset_check)
{
unsigned int i;
for (i = 0; i < n; i++) {
if (!is_32 && table[i].reg && !table[i].reset) {
if (reset_check && table[i].reg && !table[i].reset) {
kvm_err("sys_reg table %pS entry %d (%s) lacks reset\n",
&table[i], i, table[i].name);
return false;
@ -4475,7 +4552,7 @@ static bool kvm_esr_cp10_id_to_sys64(u64 esr, struct sys_reg_params *params)
return true;
kvm_pr_unimpl("Unhandled cp10 register %s: %u\n",
params->is_write ? "write" : "read", reg_id);
str_write_read(params->is_write), reg_id);
return false;
}
@ -5269,18 +5346,22 @@ int kvm_finalize_sys_regs(struct kvm_vcpu *vcpu)
int __init kvm_sys_reg_table_init(void)
{
const struct sys_reg_desc *gicv3_regs;
bool valid = true;
unsigned int i;
unsigned int i, sz;
int ret = 0;
/* Make sure tables are unique and in order. */
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), false);
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), true);
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), true);
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), true);
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), true);
valid &= check_sysreg_table(sys_reg_descs, ARRAY_SIZE(sys_reg_descs), true);
valid &= check_sysreg_table(cp14_regs, ARRAY_SIZE(cp14_regs), false);
valid &= check_sysreg_table(cp14_64_regs, ARRAY_SIZE(cp14_64_regs), false);
valid &= check_sysreg_table(cp15_regs, ARRAY_SIZE(cp15_regs), false);
valid &= check_sysreg_table(cp15_64_regs, ARRAY_SIZE(cp15_64_regs), false);
valid &= check_sysreg_table(sys_insn_descs, ARRAY_SIZE(sys_insn_descs), false);
gicv3_regs = vgic_v3_get_sysreg_table(&sz);
valid &= check_sysreg_table(gicv3_regs, sz, false);
if (!valid)
return -EINVAL;

2
arch/arm64/kvm/sys_regs.h
View File

@ -108,7 +108,7 @@ inline void print_sys_reg_msg(const struct sys_reg_params *p,
/* Look, we even formatted it for you to paste into the table! */
kvm_pr_unimpl("%pV { Op0(%2u), Op1(%2u), CRn(%2u), CRm(%2u), Op2(%2u), func_%s },\n",
&(struct va_format){ fmt, &va },
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, p->is_write ? "write" : "read");
p->Op0, p->Op1, p->CRn, p->CRm, p->Op2, str_write_read(p->is_write));
va_end(va);
}

2
arch/arm64/kvm/trace_handle_exit.h
View File

@ -113,7 +113,7 @@ TRACE_EVENT(kvm_sys_access,
__entry->vcpu_pc, __entry->name ?: "UNKN",
__entry->Op0, __entry->Op1, __entry->CRn,
__entry->CRm, __entry->Op2,
__entry->is_write ? "write" : "read")
str_write_read(__entry->is_write))
);
TRACE_EVENT(kvm_set_guest_debug,

127
arch/arm64/kvm/vgic-sys-reg-v3.c
View File

@ -297,6 +297,91 @@ static int get_gic_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
return 0;
}
static int set_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
__vcpu_assign_sys_reg(vcpu, r->reg, val);
return 0;
}
static int get_gic_ich_reg(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = __vcpu_sys_reg(vcpu, r->reg);
return 0;
}
static int set_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
u8 idx = r->Op2 & 3;
if (idx > vgic_v3_max_apr_idx(vcpu))
return -EINVAL;
return set_gic_ich_reg(vcpu, r, val);
}
static int get_gic_ich_apr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
u8 idx = r->Op2 & 3;
if (idx > vgic_v3_max_apr_idx(vcpu))
return -EINVAL;
return get_gic_ich_reg(vcpu, r, val);
}
static int set_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
if (val != KVM_ICC_SRE_EL2)
return -EINVAL;
return 0;
}
static int get_gic_icc_sre(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = KVM_ICC_SRE_EL2;
return 0;
}
static int set_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 val)
{
if (val != kvm_get_guest_vtr_el2())
return -EINVAL;
return 0;
}
static int get_gic_ich_vtr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r,
u64 *val)
{
*val = kvm_get_guest_vtr_el2();
return 0;
}
static unsigned int el2_visibility(const struct kvm_vcpu *vcpu,
const struct sys_reg_desc *rd)
{
return vcpu_has_nv(vcpu) ? 0 : REG_HIDDEN;
}
#define __EL2_REG(r, acc, i) \
{ \
SYS_DESC(SYS_ ## r), \
.get_user = get_gic_ ## acc, \
.set_user = set_gic_ ## acc, \
.reg = i, \
.visibility = el2_visibility, \
}
#define EL2_REG(r, acc) __EL2_REG(r, acc, r)
#define EL2_REG_RO(r, acc) __EL2_REG(r, acc, 0)
static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
{ SYS_DESC(SYS_ICC_PMR_EL1),
.set_user = set_gic_pmr, .get_user = get_gic_pmr, },
@ -328,8 +413,42 @@ static const struct sys_reg_desc gic_v3_icc_reg_descs[] = {
.set_user = set_gic_grpen0, .get_user = get_gic_grpen0, },
{ SYS_DESC(SYS_ICC_IGRPEN1_EL1),
.set_user = set_gic_grpen1, .get_user = get_gic_grpen1, },
EL2_REG(ICH_AP0R0_EL2, ich_apr),
EL2_REG(ICH_AP0R1_EL2, ich_apr),
EL2_REG(ICH_AP0R2_EL2, ich_apr),
EL2_REG(ICH_AP0R3_EL2, ich_apr),
EL2_REG(ICH_AP1R0_EL2, ich_apr),
EL2_REG(ICH_AP1R1_EL2, ich_apr),
EL2_REG(ICH_AP1R2_EL2, ich_apr),
EL2_REG(ICH_AP1R3_EL2, ich_apr),
EL2_REG_RO(ICC_SRE_EL2, icc_sre),
EL2_REG(ICH_HCR_EL2, ich_reg),
EL2_REG_RO(ICH_VTR_EL2, ich_vtr),
EL2_REG(ICH_VMCR_EL2, ich_reg),
EL2_REG(ICH_LR0_EL2, ich_reg),
EL2_REG(ICH_LR1_EL2, ich_reg),
EL2_REG(ICH_LR2_EL2, ich_reg),
EL2_REG(ICH_LR3_EL2, ich_reg),
EL2_REG(ICH_LR4_EL2, ich_reg),
EL2_REG(ICH_LR5_EL2, ich_reg),
EL2_REG(ICH_LR6_EL2, ich_reg),
EL2_REG(ICH_LR7_EL2, ich_reg),
EL2_REG(ICH_LR8_EL2, ich_reg),
EL2_REG(ICH_LR9_EL2, ich_reg),
EL2_REG(ICH_LR10_EL2, ich_reg),
EL2_REG(ICH_LR11_EL2, ich_reg),
EL2_REG(ICH_LR12_EL2, ich_reg),
EL2_REG(ICH_LR13_EL2, ich_reg),
EL2_REG(ICH_LR14_EL2, ich_reg),
EL2_REG(ICH_LR15_EL2, ich_reg),
};
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz)
{
*sz = ARRAY_SIZE(gic_v3_icc_reg_descs);
return gic_v3_icc_reg_descs;
}
static u64 attr_to_id(u64 attr)
{
return ARM64_SYS_REG(FIELD_GET(KVM_REG_ARM_VGIC_SYSREG_OP0_MASK, attr),
@ -341,8 +460,12 @@ static u64 attr_to_id(u64 attr)
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr)
{
if (get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
ARRAY_SIZE(gic_v3_icc_reg_descs)))
const struct sys_reg_desc *r;
r = get_reg_by_id(attr_to_id(attr->attr), gic_v3_icc_reg_descs,
ARRAY_SIZE(gic_v3_icc_reg_descs));
if (r && !sysreg_hidden(vcpu, r))
return 0;
return -ENXIO;

30
arch/arm64/kvm/vgic/vgic-init.c
View File

@ -157,6 +157,7 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
kvm->arch.vgic.in_kernel = true;
kvm->arch.vgic.vgic_model = type;
kvm->arch.vgic.implementation_rev = KVM_VGIC_IMP_REV_LATEST;
kvm->arch.vgic.vgic_dist_base = VGIC_ADDR_UNDEF;
@ -165,6 +166,9 @@ int kvm_vgic_create(struct kvm *kvm, u32 type)
else
INIT_LIST_HEAD(&kvm->arch.vgic.rd_regions);
if (type == KVM_DEV_TYPE_ARM_VGIC_V3)
kvm->arch.vgic.nassgicap = system_supports_direct_sgis();
out_unlock:
mutex_unlock(&kvm->arch.config_lock);
kvm_unlock_all_vcpus(kvm);
@ -391,11 +395,10 @@ int vgic_init(struct kvm *kvm)
goto out;
/*
* If we have GICv4.1 enabled, unconditionally request enable the
* v4 support so that we get HW-accelerated vSGIs. Otherwise, only
* enable it if we present a virtual ITS to the guest.
* Ensure vPEs are allocated if direct IRQ injection (e.g. vSGIs,
* vLPIs) is supported.
*/
if (vgic_supports_direct_msis(kvm)) {
if (vgic_supports_direct_irqs(kvm)) {
ret = vgic_v4_init(kvm);
if (ret)
goto out;
@ -409,15 +412,7 @@ int vgic_init(struct kvm *kvm)
goto out;
vgic_debug_init(kvm);
/*
* If userspace didn't set the GIC implementation revision,
* default to the latest and greatest. You know want it.
*/
if (!dist->implementation_rev)
dist->implementation_rev = KVM_VGIC_IMP_REV_LATEST;
dist->initialized = true;
out:
return ret;
}
@ -443,7 +438,7 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)
dist->vgic_cpu_base = VGIC_ADDR_UNDEF;
}
if (vgic_supports_direct_msis(kvm))
if (vgic_supports_direct_irqs(kvm))
vgic_v4_teardown(kvm);
xa_destroy(&dist->lpi_xa);
@ -674,10 +669,12 @@ void kvm_vgic_init_cpu_hardware(void)
* We want to make sure the list registers start out clear so that we
* only have the program the used registers.
*/
if (kvm_vgic_global_state.type == VGIC_V2)
if (kvm_vgic_global_state.type == VGIC_V2) {
vgic_v2_init_lrs();
else
} else if (kvm_vgic_global_state.type == VGIC_V3 ||
kvm_vgic_global_state.has_gcie_v3_compat) {
kvm_call_hyp(__vgic_v3_init_lrs);
}
}
/**
@ -722,6 +719,9 @@ int kvm_vgic_hyp_init(void)
kvm_info("GIC system register CPU interface enabled\n");
}
break;
case GIC_V5:
ret = vgic_v5_probe(gic_kvm_info);
break;
default:
ret = -ENODEV;
}

3
arch/arm64/kvm/vgic/vgic-its.c
View File

@ -2694,6 +2694,9 @@ static int vgic_its_ctrl(struct kvm *kvm, struct vgic_its *its, u64 attr)
case KVM_DEV_ARM_ITS_RESTORE_TABLES:
ret = abi->restore_tables(its);
break;
default:
ret = -ENXIO;
break;
}
mutex_unlock(&its->its_lock);

70
arch/arm64/kvm/vgic/vgic-kvm-device.c
View File

@ -5,6 +5,7 @@
* Copyright (C) 2015 ARM Ltd.
* Author: Marc Zyngier <marc.zyngier@arm.com>
*/
#include <linux/irqchip/arm-gic-v3.h>
#include <linux/kvm_host.h>
#include <kvm/arm_vgic.h>
#include <linux/uaccess.h>
@ -303,12 +304,6 @@ static int vgic_get_common_attr(struct kvm_device *dev,
VGIC_NR_PRIVATE_IRQS, uaddr);
break;
}
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
r = put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
break;
}
}
return r;
@ -509,6 +504,24 @@ int vgic_v3_parse_attr(struct kvm_device *dev, struct kvm_device_attr *attr,
return 0;
}
/*
* Allow access to certain ID-like registers prior to VGIC initialization,
* thereby allowing the VMM to provision the features / sizing of the VGIC.
*/
static bool reg_allowed_pre_init(struct kvm_device_attr *attr)
{
if (attr->group != KVM_DEV_ARM_VGIC_GRP_DIST_REGS)
return false;
switch (attr->attr & KVM_DEV_ARM_VGIC_OFFSET_MASK) {
case GICD_IIDR:
case GICD_TYPER2:
return true;
default:
return false;
}
}
/*
* vgic_v3_attr_regs_access - allows user space to access VGIC v3 state
*
@ -523,7 +536,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
struct vgic_reg_attr reg_attr;
gpa_t addr;
struct kvm_vcpu *vcpu;
bool uaccess, post_init = true;
bool uaccess;
u32 val;
int ret;
@ -539,9 +552,6 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
/* Sysregs uaccess is performed by the sysreg handling code */
uaccess = false;
break;
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
post_init = false;
fallthrough;
default:
uaccess = true;
}
@ -561,7 +571,7 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
mutex_lock(&dev->kvm->arch.config_lock);
if (post_init != vgic_initialized(dev->kvm)) {
if (!(vgic_initialized(dev->kvm) || reg_allowed_pre_init(attr))) {
ret = -EBUSY;
goto out;
}
@ -591,19 +601,6 @@ static int vgic_v3_attr_regs_access(struct kvm_device *dev,
}
break;
}
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
if (!is_write) {
val = dev->kvm->arch.vgic.mi_intid;
ret = 0;
break;
}
ret = -EINVAL;
if ((val < VGIC_NR_PRIVATE_IRQS) && (val >= VGIC_NR_SGIS)) {
dev->kvm->arch.vgic.mi_intid = val;
ret = 0;
}
break;
default:
ret = -EINVAL;
break;
@ -630,8 +627,24 @@ static int vgic_v3_set_attr(struct kvm_device *dev,
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
return vgic_v3_attr_regs_access(dev, attr, true);
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)attr->addr;
u32 val;
if (get_user(val, uaddr))
return -EFAULT;
guard(mutex)(&dev->kvm->arch.config_lock);
if (vgic_initialized(dev->kvm))
return -EBUSY;
if (!irq_is_ppi(val))
return -EINVAL;
dev->kvm->arch.vgic.mi_intid = val;
return 0;
}
default:
return vgic_set_common_attr(dev, attr);
}
@ -645,8 +658,13 @@ static int vgic_v3_get_attr(struct kvm_device *dev,
case KVM_DEV_ARM_VGIC_GRP_REDIST_REGS:
case KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS:
case KVM_DEV_ARM_VGIC_GRP_LEVEL_INFO:
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ:
return vgic_v3_attr_regs_access(dev, attr, false);
case KVM_DEV_ARM_VGIC_GRP_MAINT_IRQ: {
u32 __user *uaddr = (u32 __user *)(long)attr->addr;
guard(mutex)(&dev->kvm->arch.config_lock);
return put_user(dev->kvm->arch.vgic.mi_intid, uaddr);
}
default:
return vgic_get_common_attr(dev, attr);
}

33
arch/arm64/kvm/vgic/vgic-mmio-v3.c
View File

@ -50,8 +50,17 @@ bool vgic_has_its(struct kvm *kvm)
bool vgic_supports_direct_msis(struct kvm *kvm)
{
return (kvm_vgic_global_state.has_gicv4_1 ||
(kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm)));
return kvm_vgic_global_state.has_gicv4 && vgic_has_its(kvm);
}
bool system_supports_direct_sgis(void)
{
return kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi();
}
bool vgic_supports_direct_sgis(struct kvm *kvm)
{
return kvm->arch.vgic.nassgicap;
}
/*
@ -86,7 +95,7 @@ static unsigned long vgic_mmio_read_v3_misc(struct kvm_vcpu *vcpu,
}
break;
case GICD_TYPER2:
if (kvm_vgic_global_state.has_gicv4_1 && gic_cpuif_has_vsgi())
if (vgic_supports_direct_sgis(vcpu->kvm))
value = GICD_TYPER2_nASSGIcap;
break;
case GICD_IIDR:
@ -119,7 +128,7 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;
/* Not a GICv4.1? No HW SGIs */
if (!kvm_vgic_global_state.has_gicv4_1 || !gic_cpuif_has_vsgi())
if (!vgic_supports_direct_sgis(vcpu->kvm))
val &= ~GICD_CTLR_nASSGIreq;
/* Dist stays enabled? nASSGIreq is RO */
@ -133,7 +142,7 @@ static void vgic_mmio_write_v3_misc(struct kvm_vcpu *vcpu,
if (is_hwsgi != dist->nassgireq)
vgic_v4_configure_vsgis(vcpu->kvm);
if (kvm_vgic_global_state.has_gicv4_1 &&
if (vgic_supports_direct_sgis(vcpu->kvm) &&
was_enabled != dist->enabled)
kvm_make_all_cpus_request(vcpu->kvm, KVM_REQ_RELOAD_GICv4);
else if (!was_enabled && dist->enabled)
@ -159,8 +168,18 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
switch (addr & 0x0c) {
case GICD_TYPER2:
if (val != vgic_mmio_read_v3_misc(vcpu, addr, len))
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
if (reg == val)
return 0;
if (vgic_initialized(vcpu->kvm))
return -EBUSY;
if ((reg ^ val) & ~GICD_TYPER2_nASSGIcap)
return -EINVAL;
if (!system_supports_direct_sgis() && val)
return -EINVAL;
dist->nassgicap = val & GICD_TYPER2_nASSGIcap;
return 0;
case GICD_IIDR:
reg = vgic_mmio_read_v3_misc(vcpu, addr, len);
@ -178,7 +197,7 @@ static int vgic_mmio_uaccess_write_v3_misc(struct kvm_vcpu *vcpu,
}
case GICD_CTLR:
/* Not a GICv4.1? No HW SGIs */
if (!kvm_vgic_global_state.has_gicv4_1)
if (!vgic_supports_direct_sgis(vcpu->kvm))
val &= ~GICD_CTLR_nASSGIreq;
dist->enabled = val & GICD_CTLR_ENABLE_SS_G1;

2
arch/arm64/kvm/vgic/vgic-v3-nested.c
View File

@ -116,7 +116,7 @@ bool vgic_state_is_nested(struct kvm_vcpu *vcpu)
{
u64 xmo;
if (vcpu_has_nv(vcpu) && !is_hyp_ctxt(vcpu)) {
if (is_nested_ctxt(vcpu)) {
xmo = __vcpu_sys_reg(vcpu, HCR_EL2) & (HCR_IMO | HCR_FMO);
WARN_ONCE(xmo && xmo != (HCR_IMO | HCR_FMO),
"Separate virtual IRQ/FIQ settings not supported\n");

4
arch/arm64/kvm/vgic/vgic-v4.c
View File

@ -356,7 +356,7 @@ int vgic_v4_put(struct kvm_vcpu *vcpu)
{
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
if (!vgic_supports_direct_msis(vcpu->kvm) || !vpe->resident)
if (!vgic_supports_direct_irqs(vcpu->kvm) || !vpe->resident)
return 0;
return its_make_vpe_non_resident(vpe, vgic_v4_want_doorbell(vcpu));
@ -367,7 +367,7 @@ int vgic_v4_load(struct kvm_vcpu *vcpu)
struct its_vpe *vpe = &vcpu->arch.vgic_cpu.vgic_v3.its_vpe;
int err;
if (!vgic_supports_direct_msis(vcpu->kvm) || vpe->resident)
if (!vgic_supports_direct_irqs(vcpu->kvm) || vpe->resident)
return 0;
if (vcpu_get_flag(vcpu, IN_WFI))

52
arch/arm64/kvm/vgic/vgic-v5.c Normal file
View File

@ -0,0 +1,52 @@
// SPDX-License-Identifier: GPL-2.0-only
#include <kvm/arm_vgic.h>
#include <linux/irqchip/arm-vgic-info.h>
#include "vgic.h"
/*
* Probe for a vGICv5 compatible interrupt controller, returning 0 on success.
* Currently only supports GICv3-based VMs on a GICv5 host, and hence only
* registers a VGIC_V3 device.
*/
int vgic_v5_probe(const struct gic_kvm_info *info)
{
u64 ich_vtr_el2;
int ret;
if (!info->has_gcie_v3_compat)
return -ENODEV;
kvm_vgic_global_state.type = VGIC_V5;
kvm_vgic_global_state.has_gcie_v3_compat = true;
/* We only support v3 compat mode - use vGICv3 limits */
kvm_vgic_global_state.max_gic_vcpus = VGIC_V3_MAX_CPUS;
kvm_vgic_global_state.vcpu_base = 0;
kvm_vgic_global_state.vctrl_base = NULL;
kvm_vgic_global_state.can_emulate_gicv2 = false;
kvm_vgic_global_state.has_gicv4 = false;
kvm_vgic_global_state.has_gicv4_1 = false;
ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config);
kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2;
/*
* The ListRegs field is 5 bits, but there is an architectural
* maximum of 16 list registers. Just ignore bit 4...
*/
kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1;
ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3);
if (ret) {
kvm_err("Cannot register GICv3-legacy KVM device.\n");
return ret;
}
static_branch_enable(&kvm_vgic_global_state.gicv3_cpuif);
kvm_info("GCIE legacy system register CPU interface\n");
return 0;
}

4
arch/arm64/kvm/vgic/vgic.c
View File

@ -951,7 +951,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
* can be directly injected (GICv4).
*/
if (list_empty(&vcpu->arch.vgic_cpu.ap_list_head) &&
!vgic_supports_direct_msis(vcpu->kvm))
!vgic_supports_direct_irqs(vcpu->kvm))
return;
DEBUG_SPINLOCK_BUG_ON(!irqs_disabled());
@ -965,7 +965,7 @@ void kvm_vgic_flush_hwstate(struct kvm_vcpu *vcpu)
if (can_access_vgic_from_kernel())
vgic_restore_state(vcpu);
if (vgic_supports_direct_msis(vcpu->kvm))
if (vgic_supports_direct_irqs(vcpu->kvm))
vgic_v4_commit(vcpu);
}

48
arch/arm64/kvm/vgic/vgic.h
View File

@ -64,6 +64,24 @@
KVM_REG_ARM_VGIC_SYSREG_CRM_MASK | \
KVM_REG_ARM_VGIC_SYSREG_OP2_MASK)
#define KVM_ICC_SRE_EL2 (ICC_SRE_EL2_ENABLE | ICC_SRE_EL2_SRE | \
ICC_SRE_EL1_DIB | ICC_SRE_EL1_DFB)
#define KVM_ICH_VTR_EL2_RES0 (ICH_VTR_EL2_DVIM | \
ICH_VTR_EL2_A3V | \
ICH_VTR_EL2_IDbits)
#define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4
static inline u64 kvm_get_guest_vtr_el2(void)
{
u64 vtr;
vtr = kvm_vgic_global_state.ich_vtr_el2;
vtr &= ~KVM_ICH_VTR_EL2_RES0;
vtr |= KVM_ICH_VTR_EL2_RES1;
return vtr;
}
/*
* As per Documentation/virt/kvm/devices/arm-vgic-its.rst,
* below macros are defined for ITS table entry encoding.
@ -297,6 +315,7 @@ int vgic_v3_redist_uaccess(struct kvm_vcpu *vcpu, bool is_write,
int vgic_v3_cpu_sysregs_uaccess(struct kvm_vcpu *vcpu,
struct kvm_device_attr *attr, bool is_write);
int vgic_v3_has_cpu_sysregs_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr);
const struct sys_reg_desc *vgic_v3_get_sysreg_table(unsigned int *sz);
int vgic_v3_line_level_info_uaccess(struct kvm_vcpu *vcpu, bool is_write,
u32 intid, u32 *val);
int kvm_register_vgic_device(unsigned long type);
@ -308,6 +327,8 @@ int vgic_init(struct kvm *kvm);
void vgic_debug_init(struct kvm *kvm);
void vgic_debug_destroy(struct kvm *kvm);
int vgic_v5_probe(const struct gic_kvm_info *info);
static inline int vgic_v3_max_apr_idx(struct kvm_vcpu *vcpu)
{
struct vgic_cpu *cpu_if = &vcpu->arch.vgic_cpu;
@ -369,7 +390,23 @@ void vgic_its_invalidate_all_caches(struct kvm *kvm);
int vgic_its_inv_lpi(struct kvm *kvm, struct vgic_irq *irq);
int vgic_its_invall(struct kvm_vcpu *vcpu);
bool system_supports_direct_sgis(void);
bool vgic_supports_direct_msis(struct kvm *kvm);
bool vgic_supports_direct_sgis(struct kvm *kvm);
static inline bool vgic_supports_direct_irqs(struct kvm *kvm)
{
/*
* Deliberately conflate vLPI and vSGI support on GICv4.1 hardware,
* indirectly allowing userspace to control whether or not vPEs are
* allocated for the VM.
*/
if (system_supports_direct_sgis())
return vgic_supports_direct_sgis(kvm);
return vgic_supports_direct_msis(kvm);
}
int vgic_v4_init(struct kvm *kvm);
void vgic_v4_teardown(struct kvm *kvm);
void vgic_v4_configure_vsgis(struct kvm *kvm);
@ -389,6 +426,17 @@ void vgic_v3_put_nested(struct kvm_vcpu *vcpu);
void vgic_v3_handle_nested_maint_irq(struct kvm_vcpu *vcpu);
void vgic_v3_nested_update_mi(struct kvm_vcpu *vcpu);
static inline bool vgic_is_v3_compat(struct kvm *kvm)
{
return cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF) &&
kvm_vgic_global_state.has_gcie_v3_compat;
}
static inline bool vgic_is_v3(struct kvm *kvm)
{
return kvm_vgic_global_state.type == VGIC_V3 || vgic_is_v3_compat(kvm);
}
int vgic_its_debug_init(struct kvm_device *dev);
void vgic_its_debug_destroy(struct kvm_device *dev);

4
arch/arm64/tools/cpucaps
View File

@ -35,7 +35,8 @@ HAS_GENERIC_AUTH
HAS_GENERIC_AUTH_ARCH_QARMA3
HAS_GENERIC_AUTH_ARCH_QARMA5
HAS_GENERIC_AUTH_IMP_DEF
HAS_GIC_CPUIF_SYSREGS
HAS_GICV3_CPUIF
HAS_GICV5_CPUIF
HAS_GIC_PRIO_MASKING
HAS_GIC_PRIO_RELAXED_SYNC
HAS_HCR_NV1
@ -49,6 +50,7 @@ HAS_PAN
HAS_PMUV3
HAS_S1PIE
HAS_S1POE
HAS_SCTLR2
HAS_RAS_EXTN
HAS_RNG
HAS_SB

514
arch/arm64/tools/sysreg
View File

@ -1314,7 +1314,10 @@ UnsignedEnum 19:16 UINJ
0b0000 NI
0b0001 IMP
EndEnum
Res0 15:12
UnsignedEnum 15:12 GCIE
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 11:8 MTEFAR
0b0000 NI
0b0001 IMP
@ -3021,6 +3024,435 @@ Sysreg PMIAR_EL1 3 0 9 14 7
Field 63:0 ADDRESS
EndSysreg
SysregFields ICC_PPI_HMRx_EL1
Field 63 HM63
Field 62 HM62
Field 61 HM61
Field 60 HM60
Field 59 HM59
Field 58 HM58
Field 57 HM57
Field 56 HM56
Field 55 HM55
Field 54 HM54
Field 53 HM53
Field 52 HM52
Field 51 HM51
Field 50 HM50
Field 49 HM49
Field 48 HM48
Field 47 HM47
Field 46 HM46
Field 45 HM45
Field 44 HM44
Field 43 HM43
Field 42 HM42
Field 41 HM41
Field 40 HM40
Field 39 HM39
Field 38 HM38
Field 37 HM37
Field 36 HM36
Field 35 HM35
Field 34 HM34
Field 33 HM33
Field 32 HM32
Field 31 HM31
Field 30 HM30
Field 29 HM29
Field 28 HM28
Field 27 HM27
Field 26 HM26
Field 25 HM25
Field 24 HM24
Field 23 HM23
Field 22 HM22
Field 21 HM21
Field 20 HM20
Field 19 HM19
Field 18 HM18
Field 17 HM17
Field 16 HM16
Field 15 HM15
Field 14 HM14
Field 13 HM13
Field 12 HM12
Field 11 HM11
Field 10 HM10
Field 9 HM9
Field 8 HM8
Field 7 HM7
Field 6 HM6
Field 5 HM5
Field 4 HM4
Field 3 HM3
Field 2 HM2
Field 1 HM1
Field 0 HM0
EndSysregFields
Sysreg ICC_PPI_HMR0_EL1 3 0 12 10 0
Fields ICC_PPI_HMRx_EL1
EndSysreg
Sysreg ICC_PPI_HMR1_EL1 3 0 12 10 1
Fields ICC_PPI_HMRx_EL1
EndSysreg
Sysreg ICC_IDR0_EL1 3 0 12 10 2
Res0 63:12
UnsignedEnum 11:8 GCIE_LEGACY
0b0000 NI
0b0001 IMP
EndEnum
UnsignedEnum 7:4 PRI_BITS
0b0011 4BITS
0b0100 5BITS
EndEnum
UnsignedEnum 3:0 ID_BITS
0b0000 16BITS
0b0001 24BITS
EndEnum
EndSysreg
Sysreg ICC_ICSR_EL1 3 0 12 10 4
Res0 63:48
Field 47:32 IAFFID
Res0 31:16
Field 15:11 Priority
Res0 10:6
Field 5 HM
Field 4 Active
Field 3 IRM
Field 2 Pending
Field 1 Enabled
Field 0 F
EndSysreg
SysregFields ICC_PPI_ENABLERx_EL1
Field 63 EN63
Field 62 EN62
Field 61 EN61
Field 60 EN60
Field 59 EN59
Field 58 EN58
Field 57 EN57
Field 56 EN56
Field 55 EN55
Field 54 EN54
Field 53 EN53
Field 52 EN52
Field 51 EN51
Field 50 EN50
Field 49 EN49
Field 48 EN48
Field 47 EN47
Field 46 EN46
Field 45 EN45
Field 44 EN44
Field 43 EN43
Field 42 EN42
Field 41 EN41
Field 40 EN40
Field 39 EN39
Field 38 EN38
Field 37 EN37
Field 36 EN36
Field 35 EN35
Field 34 EN34
Field 33 EN33
Field 32 EN32
Field 31 EN31
Field 30 EN30
Field 29 EN29
Field 28 EN28
Field 27 EN27
Field 26 EN26
Field 25 EN25
Field 24 EN24
Field 23 EN23
Field 22 EN22
Field 21 EN21
Field 20 EN20
Field 19 EN19
Field 18 EN18
Field 17 EN17
Field 16 EN16
Field 15 EN15
Field 14 EN14
Field 13 EN13
Field 12 EN12
Field 11 EN11
Field 10 EN10
Field 9 EN9
Field 8 EN8
Field 7 EN7
Field 6 EN6
Field 5 EN5
Field 4 EN4
Field 3 EN3
Field 2 EN2
Field 1 EN1
Field 0 EN0
EndSysregFields
Sysreg ICC_PPI_ENABLER0_EL1 3 0 12 10 6
Fields ICC_PPI_ENABLERx_EL1
EndSysreg
Sysreg ICC_PPI_ENABLER1_EL1 3 0 12 10 7
Fields ICC_PPI_ENABLERx_EL1
EndSysreg
SysregFields ICC_PPI_ACTIVERx_EL1
Field 63 Active63
Field 62 Active62
Field 61 Active61
Field 60 Active60
Field 59 Active59
Field 58 Active58
Field 57 Active57
Field 56 Active56
Field 55 Active55
Field 54 Active54
Field 53 Active53
Field 52 Active52
Field 51 Active51
Field 50 Active50
Field 49 Active49
Field 48 Active48
Field 47 Active47
Field 46 Active46
Field 45 Active45
Field 44 Active44
Field 43 Active43
Field 42 Active42
Field 41 Active41
Field 40 Active40
Field 39 Active39
Field 38 Active38
Field 37 Active37
Field 36 Active36
Field 35 Active35
Field 34 Active34
Field 33 Active33
Field 32 Active32
Field 31 Active31
Field 30 Active30
Field 29 Active29
Field 28 Active28
Field 27 Active27
Field 26 Active26
Field 25 Active25
Field 24 Active24
Field 23 Active23
Field 22 Active22
Field 21 Active21
Field 20 Active20
Field 19 Active19
Field 18 Active18
Field 17 Active17
Field 16 Active16
Field 15 Active15
Field 14 Active14
Field 13 Active13
Field 12 Active12
Field 11 Active11
Field 10 Active10
Field 9 Active9
Field 8 Active8
Field 7 Active7
Field 6 Active6
Field 5 Active5
Field 4 Active4
Field 3 Active3
Field 2 Active2
Field 1 Active1
Field 0 Active0
EndSysregFields
Sysreg ICC_PPI_CACTIVER0_EL1 3 0 12 13 0
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_CACTIVER1_EL1 3 0 12 13 1
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_SACTIVER0_EL1 3 0 12 13 2
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
Sysreg ICC_PPI_SACTIVER1_EL1 3 0 12 13 3
Fields ICC_PPI_ACTIVERx_EL1
EndSysreg
SysregFields ICC_PPI_PENDRx_EL1
Field 63 Pend63
Field 62 Pend62
Field 61 Pend61
Field 60 Pend60
Field 59 Pend59
Field 58 Pend58
Field 57 Pend57
Field 56 Pend56
Field 55 Pend55
Field 54 Pend54
Field 53 Pend53
Field 52 Pend52
Field 51 Pend51
Field 50 Pend50
Field 49 Pend49
Field 48 Pend48
Field 47 Pend47
Field 46 Pend46
Field 45 Pend45
Field 44 Pend44
Field 43 Pend43
Field 42 Pend42
Field 41 Pend41
Field 40 Pend40
Field 39 Pend39
Field 38 Pend38
Field 37 Pend37
Field 36 Pend36
Field 35 Pend35
Field 34 Pend34
Field 33 Pend33
Field 32 Pend32
Field 31 Pend31
Field 30 Pend30
Field 29 Pend29
Field 28 Pend28
Field 27 Pend27
Field 26 Pend26
Field 25 Pend25
Field 24 Pend24
Field 23 Pend23
Field 22 Pend22
Field 21 Pend21
Field 20 Pend20
Field 19 Pend19
Field 18 Pend18
Field 17 Pend17
Field 16 Pend16
Field 15 Pend15
Field 14 Pend14
Field 13 Pend13
Field 12 Pend12
Field 11 Pend11
Field 10 Pend10
Field 9 Pend9
Field 8 Pend8
Field 7 Pend7
Field 6 Pend6
Field 5 Pend5
Field 4 Pend4
Field 3 Pend3
Field 2 Pend2
Field 1 Pend1
Field 0 Pend0
EndSysregFields
Sysreg ICC_PPI_CPENDR0_EL1 3 0 12 13 4
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_CPENDR1_EL1 3 0 12 13 5
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_SPENDR0_EL1 3 0 12 13 6
Fields ICC_PPI_PENDRx_EL1
EndSysreg
Sysreg ICC_PPI_SPENDR1_EL1 3 0 12 13 7
Fields ICC_PPI_PENDRx_EL1
EndSysreg
SysregFields ICC_PPI_PRIORITYRx_EL1
Res0 63:61
Field 60:56 Priority7
Res0 55:53
Field 52:48 Priority6
Res0 47:45
Field 44:40 Priority5
Res0 39:37
Field 36:32 Priority4
Res0 31:29
Field 28:24 Priority3
Res0 23:21
Field 20:16 Priority2
Res0 15:13
Field 12:8 Priority1
Res0 7:5
Field 4:0 Priority0
EndSysregFields
Sysreg ICC_PPI_PRIORITYR0_EL1 3 0 12 14 0
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR1_EL1 3 0 12 14 1
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR2_EL1 3 0 12 14 2
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR3_EL1 3 0 12 14 3
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR4_EL1 3 0 12 14 4
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR5_EL1 3 0 12 14 5
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR6_EL1 3 0 12 14 6
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR7_EL1 3 0 12 14 7
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR8_EL1 3 0 12 15 0
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR9_EL1 3 0 12 15 1
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR10_EL1 3 0 12 15 2
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR11_EL1 3 0 12 15 3
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR12_EL1 3 0 12 15 4
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR13_EL1 3 0 12 15 5
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR14_EL1 3 0 12 15 6
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg ICC_PPI_PRIORITYR15_EL1 3 0 12 15 7
Fields ICC_PPI_PRIORITYRx_EL1
EndSysreg
Sysreg PMSELR_EL0 3 3 9 12 5
Res0 63:5
Field 4:0 SEL
@ -3103,6 +3535,19 @@ Res0 14:12
Field 11:0 AFFINITY
EndSysreg
Sysreg ICC_CR0_EL1 3 1 12 0 1
Res0 63:39
Field 38 PID
Field 37:32 IPPT
Res0 31:1
Field 0 EN
EndSysreg
Sysreg ICC_PCR_EL1 3 1 12 0 2
Res0 63:5
Field 4:0 PRIORITY
EndSysreg
Sysreg CSSELR_EL1 3 2 0 0 0
Res0 63:5
Field 4 TnD
@ -3989,6 +4434,54 @@ Field 31:16 PhyPARTID29
Field 15:0 PhyPARTID28
EndSysreg
Sysreg ICH_HFGRTR_EL2 3 4 12 9 4
Res0 63:21
Field 20 ICC_PPI_ACTIVERn_EL1
Field 19 ICC_PPI_PRIORITYRn_EL1
Field 18 ICC_PPI_PENDRn_EL1
Field 17 ICC_PPI_ENABLERn_EL1
Field 16 ICC_PPI_HMRn_EL1
Res0 15:8
Field 7 ICC_IAFFIDR_EL1
Field 6 ICC_ICSR_EL1
Field 5 ICC_PCR_EL1
Field 4 ICC_HPPIR_EL1
Field 3 ICC_HAPR_EL1
Field 2 ICC_CR0_EL1
Field 1 ICC_IDRn_EL1
Field 0 ICC_APR_EL1
EndSysreg
Sysreg ICH_HFGWTR_EL2 3 4 12 9 6
Res0 63:21
Field 20 ICC_PPI_ACTIVERn_EL1
Field 19 ICC_PPI_PRIORITYRn_EL1
Field 18 ICC_PPI_PENDRn_EL1
Field 17 ICC_PPI_ENABLERn_EL1
Res0 16:7
Field 6 ICC_ICSR_EL1
Field 5 ICC_PCR_EL1
Res0 4:3
Field 2 ICC_CR0_EL1
Res0 1
Field 0 ICC_APR_EL1
EndSysreg
Sysreg ICH_HFGITR_EL2 3 4 12 9 7
Res0 63:11
Field 10 GICRCDNMIA
Field 9 GICRCDIA
Field 8 GICCDDI
Field 7 GICCDEOI
Field 6 GICCDHM
Field 5 GICCDRCFG
Field 4 GICCDPEND
Field 3 GICCDAFF
Field 2 GICCDPRI
Field 1 GICCDDIS
Field 0 GICCDEN
EndSysreg
Sysreg ICH_HCR_EL2 3 4 12 11 0
Res0 63:32
Field 31:27 EOIcount
@ -4037,6 +4530,12 @@ Field 1 U
Field 0 EOI
EndSysreg
Sysreg ICH_VCTLR_EL2 3 4 12 11 4
Res0 63:2
Field 1 V3
Field 0 En
EndSysreg
Sysreg CONTEXTIDR_EL2 3 4 13 0 1
Fields CONTEXTIDR_ELx
EndSysreg
@ -4150,7 +4649,13 @@ Mapping TCR_EL1
EndSysreg
Sysreg TCR2_EL1 3 0 2 0 3
Res0 63:16
Res0 63:22
Field 21 FNGNA1
Field 20 FNGNA0
Res0 19
Field 18 FNG1
Field 17 FNG0
Field 16 A2
Field 15 DisCH1
Field 14 DisCH0
Res0 13:12
@ -4174,7 +4679,10 @@ Mapping TCR2_EL1
EndSysreg
Sysreg TCR2_EL2 3 4 2 0 3
Res0 63:16
Res0 63:19
Field 18 FNG1
Field 17 FNG0
Field 16 A2
Field 15 DisCH1
Field 14 DisCH0
Field 13 AMEC1

12
drivers/irqchip/Kconfig
View File

@ -41,10 +41,14 @@ config ARM_GIC_V3
select HAVE_ARM_SMCCC_DISCOVERY
select IRQ_MSI_IOMMU
config ARM_GIC_ITS_PARENT
bool
config ARM_GIC_V3_ITS
bool
select GENERIC_MSI_IRQ
select IRQ_MSI_LIB
select ARM_GIC_ITS_PARENT
default ARM_GIC_V3
select IRQ_MSI_IOMMU
@ -54,6 +58,14 @@ config ARM_GIC_V3_ITS_FSL_MC
depends on FSL_MC_BUS
default ARM_GIC_V3_ITS
config ARM_GIC_V5
bool
select IRQ_DOMAIN_HIERARCHY
select GENERIC_IRQ_EFFECTIVE_AFF_MASK
select GENERIC_MSI_IRQ
select IRQ_MSI_LIB
select ARM_GIC_ITS_PARENT
config ARM_NVIC
bool
select IRQ_DOMAIN_HIERARCHY

5
drivers/irqchip/Makefile
View File

@ -33,9 +33,12 @@ obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o
obj-$(CONFIG_IRQ_MSI_LIB) += irq-msi-lib.o
obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o
obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o
obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o irq-gic-v3-its-msi-parent.o
obj-$(CONFIG_ARM_GIC_ITS_PARENT) += irq-gic-its-msi-parent.o
obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o
obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC) += irq-gic-v3-its-fsl-mc-msi.o
obj-$(CONFIG_PARTITION_PERCPU) += irq-partition-percpu.o
obj-$(CONFIG_ARM_GIC_V5) += irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \
irq-gic-v5-iwb.o
obj-$(CONFIG_HISILICON_IRQ_MBIGEN) += irq-mbigen.o
obj-$(CONFIG_ARM_NVIC) += irq-nvic.o
obj-$(CONFIG_ARM_VIC) += irq-vic.o

2
drivers/irqchip/irq-gic-common.h
View File

@ -29,8 +29,6 @@ void gic_enable_quirks(u32 iidr, const struct gic_quirk *quirks,
void gic_enable_of_quirks(const struct device_node *np,
const struct gic_quirk *quirks, void *data);
extern const struct msi_parent_ops gic_v3_its_msi_parent_ops;
#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1)
#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1 << 2)

168
drivers/irqchip/irq-gic-v3-its-msi-parent.c → drivers/irqchip/irq-gic-its-msi-parent.c
View File

@ -5,9 +5,10 @@
// Copyright (C) 2022 Intel
#include <linux/acpi_iort.h>
#include <linux/of_address.h>
#include <linux/pci.h>
#include "irq-gic-common.h"
#include "irq-gic-its-msi-parent.h"
#include <linux/irqchip/irq-msi-lib.h>
#define ITS_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \
@ -18,6 +19,23 @@
MSI_FLAG_PCI_MSIX | \
MSI_FLAG_MULTI_PCI_MSI)
static int its_translate_frame_address(struct device_node *msi_node, phys_addr_t *pa)
{
struct resource res;
int ret;
ret = of_property_match_string(msi_node, "reg-names", "ns-translate");
if (ret < 0)
return ret;
ret = of_address_to_resource(msi_node, ret, &res);
if (ret)
return ret;
*pa = res.start;
return 0;
}
#ifdef CONFIG_PCI_MSI
static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data)
{
@ -82,8 +100,46 @@ static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev,
msi_info = msi_get_domain_info(domain->parent);
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
}
static int its_v5_pci_msi_prepare(struct irq_domain *domain, struct device *dev,
int nvec, msi_alloc_info_t *info)
{
struct device_node *msi_node = NULL;
struct msi_domain_info *msi_info;
struct pci_dev *pdev;
phys_addr_t pa;
u32 rid;
int ret;
if (!dev_is_pci(dev))
return -EINVAL;
pdev = to_pci_dev(dev);
rid = pci_msi_map_rid_ctlr_node(pdev, &msi_node);
if (!msi_node)
return -ENODEV;
ret = its_translate_frame_address(msi_node, &pa);
if (ret)
return -ENODEV;
of_node_put(msi_node);
/* ITS specific DeviceID */
info->scratchpad[0].ul = rid;
/* ITS translate frame physical address */
info->scratchpad[1].ul = pa;
/* Always allocate power of two vectors */
nvec = roundup_pow_of_two(nvec);
msi_info = msi_get_domain_info(domain->parent);
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
}
#else /* CONFIG_PCI_MSI */
#define its_pci_msi_prepare NULL
#define its_v5_pci_msi_prepare NULL
#endif /* !CONFIG_PCI_MSI */
static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev,
@ -118,6 +174,53 @@ static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev,
return ret;
}
static int of_v5_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev,
u32 *dev_id, phys_addr_t *pa)
{
int ret, index = 0;
/*
* Retrieve the DeviceID and the ITS translate frame node pointer
* out of the msi-parent property.
*/
do {
struct of_phandle_args args;
ret = of_parse_phandle_with_args(dev->of_node,
"msi-parent", "#msi-cells",
index, &args);
if (ret)
break;
/*
* The IRQ domain fwnode is the msi controller parent
* in GICv5 (where the msi controller nodes are the
* ITS translate frames).
*/
if (args.np->parent == irq_domain_get_of_node(domain)) {
if (WARN_ON(args.args_count != 1))
return -EINVAL;
*dev_id = args.args[0];
ret = its_translate_frame_address(args.np, pa);
if (ret)
return -ENODEV;
break;
}
index++;
} while (!ret);
if (ret) {
struct device_node *np = NULL;
ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id);
if (np) {
ret = its_translate_frame_address(np, pa);
of_node_put(np);
}
}
return ret;
}
int __weak iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id)
{
return -1;
@ -148,6 +251,33 @@ static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev,
dev, nvec, info);
}
static int its_v5_pmsi_prepare(struct irq_domain *domain, struct device *dev,
int nvec, msi_alloc_info_t *info)
{
struct msi_domain_info *msi_info;
phys_addr_t pa;
u32 dev_id;
int ret;
if (!dev->of_node)
return -ENODEV;
ret = of_v5_pmsi_get_msi_info(domain->parent, dev, &dev_id, &pa);
if (ret)
return ret;
/* ITS specific DeviceID */
info->scratchpad[0].ul = dev_id;
/* ITS translate frame physical address */
info->scratchpad[1].ul = pa;
/* Allocate always as a power of 2 */
nvec = roundup_pow_of_two(nvec);
msi_info = msi_get_domain_info(domain->parent);
return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info);
}
static void its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info)
{
struct msi_domain_info *msi_info;
@ -199,6 +329,32 @@ static bool its_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
return true;
}
static bool its_v5_init_dev_msi_info(struct device *dev, struct irq_domain *domain,
struct irq_domain *real_parent, struct msi_domain_info *info)
{
if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info))
return false;
switch (info->bus_token) {
case DOMAIN_BUS_PCI_DEVICE_MSI:
case DOMAIN_BUS_PCI_DEVICE_MSIX:
info->ops->msi_prepare = its_v5_pci_msi_prepare;
info->ops->msi_teardown = its_msi_teardown;
break;
case DOMAIN_BUS_DEVICE_MSI:
case DOMAIN_BUS_WIRED_TO_MSI:
info->ops->msi_prepare = its_v5_pmsi_prepare;
info->ops->msi_teardown = its_msi_teardown;
break;
default:
/* Confused. How did the lib return true? */
WARN_ON_ONCE(1);
return false;
}
return true;
}
const struct msi_parent_ops gic_v3_its_msi_parent_ops = {
.supported_flags = ITS_MSI_FLAGS_SUPPORTED,
.required_flags = ITS_MSI_FLAGS_REQUIRED,
@ -208,3 +364,13 @@ const struct msi_parent_ops gic_v3_its_msi_parent_ops = {
.prefix = "ITS-",
.init_dev_msi_info = its_init_dev_msi_info,
};
const struct msi_parent_ops gic_v5_its_msi_parent_ops = {
.supported_flags = ITS_MSI_FLAGS_SUPPORTED,
.required_flags = ITS_MSI_FLAGS_REQUIRED,
.chip_flags = MSI_CHIP_FLAG_SET_EOI,
.bus_select_token = DOMAIN_BUS_NEXUS,
.bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI,
.prefix = "ITS-v5-",
.init_dev_msi_info = its_v5_init_dev_msi_info,
};

12
drivers/irqchip/irq-gic-its-msi-parent.h Normal file
View File

@ -0,0 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2024 ARM Limited, All Rights Reserved.
*/
#ifndef _IRQ_GIC_ITS_MSI_PARENT_H
#define _IRQ_GIC_ITS_MSI_PARENT_H
extern const struct msi_parent_ops gic_v3_its_msi_parent_ops;
extern const struct msi_parent_ops gic_v5_its_msi_parent_ops;
#endif /* _IRQ_GIC_ITS_MSI_PARENT_H */

1
drivers/irqchip/irq-gic-v3-its.c
View File

@ -41,6 +41,7 @@
#include <asm/exception.h>
#include "irq-gic-common.h"
#include "irq-gic-its-msi-parent.h"
#include <linux/irqchip/irq-msi-lib.h>
#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)

822
drivers/irqchip/irq-gic-v5-irs.c Normal file
View File

@ -0,0 +1,822 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
*/
#define pr_fmt(fmt) "GICv5 IRS: " fmt
#include <linux/log2.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/irqchip.h>
#include <linux/irqchip/arm-gic-v5.h>
/*
* Hardcoded ID_BITS limit for systems supporting only a 1-level IST
* table. Systems supporting only a 1-level IST table aren't expected
* to require more than 2^12 LPIs. Tweak as required.
*/
#define LPI_ID_BITS_LINEAR 12
#define IRS_FLAGS_NON_COHERENT BIT(0)
static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data);
static LIST_HEAD(irs_nodes);
static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data,
const u32 reg_offset)
{
return readl_relaxed(irs_data->irs_base + reg_offset);
}
static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data,
const u32 val, const u32 reg_offset)
{
writel_relaxed(val, irs_data->irs_base + reg_offset);
}
static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data,
const u32 reg_offset)
{
return readq_relaxed(irs_data->irs_base + reg_offset);
}
static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data,
const u64 val, const u32 reg_offset)
{
writeq_relaxed(val, irs_data->irs_base + reg_offset);
}
/*
* The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register
* provides the memory barriers (through MMIO accessors)
* required to synchronize CPU and GIC access to IST memory.
*/
static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data)
{
return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR,
GICV5_IRS_IST_STATUSR_IDLE, NULL);
}
static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data,
unsigned int lpi_id_bits,
unsigned int istsz)
{
size_t l2istsz;
u32 n, cfgr;
void *ist;
u64 baser;
int ret;
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
n = max(5, lpi_id_bits + 1 + istsz);
l2istsz = BIT(n + 1);
/*
* Check memory requirements. For a linear IST we cap the
* number of ID bits to a value that should never exceed
* kmalloc interface memory allocation limits, so this
* check is really belt and braces.
*/
if (l2istsz > KMALLOC_MAX_SIZE) {
u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz;
pr_warn("Limiting LPI ID bits from %u to %u\n",
lpi_id_bits, lpi_id_cap);
lpi_id_bits = lpi_id_cap;
l2istsz = KMALLOC_MAX_SIZE;
}
ist = kzalloc(l2istsz, GFP_KERNEL);
if (!ist)
return -ENOMEM;
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
dcache_clean_inval_poc((unsigned long)ist,
(unsigned long)ist + l2istsz);
else
dsb(ishst);
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) |
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ,
GICV5_IRS_IST_CFGR_L2SZ_4K) |
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
gicv5_global_data.ist.l2 = false;
baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
ret = gicv5_irs_ist_synchronise(irs_data);
if (ret) {
kfree(ist);
return ret;
}
return 0;
}
static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data,
unsigned int lpi_id_bits,
unsigned int istsz,
unsigned int l2sz)
{
__le64 *l1ist;
u32 cfgr, n;
size_t l1sz;
u64 baser;
int ret;
/* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */
n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2);
l1sz = BIT(n + 1);
l1ist = kzalloc(l1sz, GFP_KERNEL);
if (!l1ist)
return -ENOMEM;
if (irs_data->flags & IRS_FLAGS_NON_COHERENT)
dcache_clean_inval_poc((unsigned long)l1ist,
(unsigned long)l1ist + l1sz);
else
dsb(ishst);
cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE,
GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) |
FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) |
FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) |
FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits);
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR);
/*
* The L2SZ determine bits required at L2 level. Number of bytes
* required by metadata is reported through istsz - the number of bits
* covered by L2 entries scales accordingly.
*/
gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1);
gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz);
gicv5_global_data.ist.l1ist_addr = l1ist;
gicv5_global_data.ist.l2 = true;
baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) |
FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1);
irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER);
ret = gicv5_irs_ist_synchronise(irs_data);
if (ret) {
kfree(l1ist);
return ret;
}
return 0;
}
/*
* Alloc L2 IST entries on demand.
*
* Locking/serialization is guaranteed by irqdomain core code by
* taking the hierarchical domain struct irq_domain.root->mutex.
*/
int gicv5_irs_iste_alloc(const u32 lpi)
{
struct gicv5_irs_chip_data *irs_data;
unsigned int index;
u32 l2istr, l2bits;
__le64 *l1ist;
size_t l2size;
void *l2ist;
int ret;
if (!gicv5_global_data.ist.l2)
return 0;
irs_data = per_cpu(per_cpu_irs_data, smp_processor_id());
if (!irs_data)
return -ENOENT;
l2size = gicv5_global_data.ist.l2_size;
l2bits = gicv5_global_data.ist.l2_bits;
l1ist = gicv5_global_data.ist.l1ist_addr;
index = lpi >> l2bits;
if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index])))
return 0;
l2ist = kzalloc(l2size, GFP_KERNEL);
if (!l2ist)
return -ENOMEM;
l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK);
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
dcache_clean_inval_poc((unsigned long)l2ist,
(unsigned long)l2ist + l2size);
dcache_clean_poc((unsigned long)(l1ist + index),
(unsigned long)(l1ist + index) + sizeof(*l1ist));
} else {
dsb(ishst);
}
l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi);
irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR);
ret = gicv5_irs_ist_synchronise(irs_data);
if (ret) {
l1ist[index] = 0;
kfree(l2ist);
return ret;
}
/*
* Make sure we invalidate the cache line pulled before the IRS
* had a chance to update the L1 entry and mark it valid.
*/
if (irs_data->flags & IRS_FLAGS_NON_COHERENT) {
/*
* gicv5_irs_ist_synchronise() includes memory
* barriers (MMIO accessors) required to guarantee that the
* following dcache invalidation is not executed before the
* IST mapping operation has completed.
*/
dcache_inval_poc((unsigned long)(l1ist + index),
(unsigned long)(l1ist + index) + sizeof(*l1ist));
}
return 0;
}
/*
* Try to match the L2 IST size to the pagesize, and if this is not possible
* pick the smallest supported L2 size in order to minimise the requirement for
* physically contiguous blocks of memory as page-sized allocations are
* guaranteed to be physically contiguous, and are by definition the easiest to
* find.
*
* Fall back to the smallest supported size (in the event that the pagesize
* itself is not supported) again serves to make it easier to find physically
* contiguous blocks of memory.
*/
static unsigned int gicv5_irs_l2_sz(u32 idr2)
{
switch (PAGE_SIZE) {
case SZ_64K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2))
return GICV5_IRS_IST_CFGR_L2SZ_64K;
fallthrough;
case SZ_4K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
return GICV5_IRS_IST_CFGR_L2SZ_4K;
fallthrough;
case SZ_16K:
if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2))
return GICV5_IRS_IST_CFGR_L2SZ_16K;
break;
}
if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2))
return GICV5_IRS_IST_CFGR_L2SZ_4K;
return GICV5_IRS_IST_CFGR_L2SZ_64K;
}
static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data)
{
u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz;
u32 l2_iste_sz_split, idr2;
bool two_levels, istmd;
u64 baser;
int ret;
baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER);
if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) {
pr_err("IST is marked as valid already; cannot allocate\n");
return -EPERM;
}
idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2);
idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2);
idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2);
/*
* For two level tables we are always supporting the maximum allowed
* number of IDs.
*
* For 1-level tables, we should support a number of bits that
* is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest
* the level 1-table gets too large and its memory allocation
* may fail.
*/
if (two_levels) {
lpi_id_bits = idr2_id_bits;
} else {
lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits);
lpi_id_bits = min(lpi_id_bits, idr2_id_bits);
}
/*
* Cap the ID bits according to the CPUIF supported ID bits
*/
lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits);
if (two_levels)
l2sz = gicv5_irs_l2_sz(idr2);
istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2);
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4;
if (istmd) {
l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2);
if (lpi_id_bits < l2_iste_sz_split)
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8;
else
l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16;
}
/*
* Follow GICv5 specification recommendation to opt in for two
* level tables (ref: 10.2.1.14 IRS_IST_CFGR).
*/
if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) {
ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits,
l2_iste_sz, l2sz);
} else {
ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits,
l2_iste_sz);
}
if (ret)
return ret;
gicv5_init_lpis(BIT(lpi_id_bits));
return 0;
}
struct iaffid_entry {
u16 iaffid;
bool valid;
};
static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid);
int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid)
{
if (!per_cpu(cpu_iaffid, cpuid).valid) {
pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
return -ENODEV;
}
*iaffid = per_cpu(cpu_iaffid, cpuid).iaffid;
return 0;
}
struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id)
{
struct gicv5_irs_chip_data *irs_data;
u32 min, max;
list_for_each_entry(irs_data, &irs_nodes, entry) {
if (!irs_data->spi_range)
continue;
min = irs_data->spi_min;
max = irs_data->spi_min + irs_data->spi_range - 1;
if (spi_id >= min && spi_id <= max)
return irs_data;
}
return NULL;
}
static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data)
{
u32 statusr;
int ret;
ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR,
GICV5_IRS_SPI_STATUSR_IDLE, &statusr);
if (ret)
return ret;
return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO;
}
static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data,
bool selr)
{
bool valid = true;
u32 statusr;
int ret;
ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR,
GICV5_IRS_PE_STATUSR_IDLE, &statusr);
if (ret)
return ret;
if (selr)
valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr);
return valid ? 0 : -EIO;
}
static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data)
{
return gicv5_irs_wait_for_irs_pe(irs_data, true);
}
static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data)
{
return gicv5_irs_wait_for_irs_pe(irs_data, false);
}
int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type)
{
struct gicv5_irs_chip_data *irs_data = d->chip_data;
u32 selr, cfgr;
bool level;
int ret;
/*
* There is no distinction between HIGH/LOW for level IRQs
* and RISING/FALLING for edge IRQs in the architecture,
* hence consider them equivalent.
*/
switch (type) {
case IRQ_TYPE_EDGE_RISING:
case IRQ_TYPE_EDGE_FALLING:
level = false;
break;
case IRQ_TYPE_LEVEL_HIGH:
case IRQ_TYPE_LEVEL_LOW:
level = true;
break;
default:
return -EINVAL;
}
guard(raw_spinlock)(&irs_data->spi_config_lock);
selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq);
irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR);
ret = gicv5_irs_wait_for_spi_op(irs_data);
if (ret)
return ret;
cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level);
irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR);
return gicv5_irs_wait_for_spi_op(irs_data);
}
static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data)
{
return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0,
GICV5_IRS_CR0_IDLE, NULL);
}
void gicv5_irs_syncr(void)
{
struct gicv5_irs_chip_data *irs_data;
u32 syncr;
irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry);
if (WARN_ON_ONCE(!irs_data))
return;
syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1);
irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR);
gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR,
GICV5_IRS_SYNC_STATUSR_IDLE);
}
int gicv5_irs_register_cpu(int cpuid)
{
struct gicv5_irs_chip_data *irs_data;
u32 selr, cr0;
u16 iaffid;
int ret;
ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid);
if (ret) {
pr_err("IAFFID for CPU %d has not been initialised\n", cpuid);
return ret;
}
irs_data = per_cpu(per_cpu_irs_data, cpuid);
if (!irs_data) {
pr_err("No IRS associated with CPU %u\n", cpuid);
return -ENXIO;
}
selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid);
irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR);
ret = gicv5_irs_wait_for_pe_selr(irs_data);
if (ret) {
pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid);
return -ENXIO;
}
cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1);
irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0);
ret = gicv5_irs_wait_for_pe_cr0(irs_data);
if (ret)
return ret;
pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid);
return 0;
}
static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data,
void __iomem *irs_base,
struct fwnode_handle *handle)
{
struct device_node *np = to_of_node(handle);
u32 cr0, cr1;
irs_data->fwnode = handle;
irs_data->irs_base = irs_base;
if (of_property_read_bool(np, "dma-noncoherent")) {
/*
* A non-coherent IRS implies that some cache levels cannot be
* used coherently by the cores and GIC. Our only option is to mark
* memory attributes for the GIC as non-cacheable; by default,
* non-cacheable memory attributes imply outer-shareable
* shareability, the value written into IRS_CR1_SH is ignored.
*/
cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) |
FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE);
irs_data->flags |= IRS_FLAGS_NON_COHERENT;
} else {
cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) |
FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) |
FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) |
FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE);
}
irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1);
cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1);
irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0);
gicv5_irs_wait_for_idle(irs_data);
}
static int __init gicv5_irs_of_init_affinity(struct device_node *node,
struct gicv5_irs_chip_data *irs_data,
u8 iaffid_bits)
{
/*
* Detect IAFFID<->CPU mappings from the device tree and
* record IRS<->CPU topology information.
*/
u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0);
int ret, i, ncpus, niaffids;
ncpus = of_count_phandle_with_args(node, "cpus", NULL);
if (ncpus < 0)
return -EINVAL;
niaffids = of_property_count_elems_of_size(node, "arm,iaffids",
sizeof(u16));
if (niaffids != ncpus)
return -EINVAL;
u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL);
if (!iaffids)
return -ENOMEM;
ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids);
if (ret)
return ret;
for (i = 0; i < ncpus; i++) {
struct device_node *cpu_node;
int cpu;
cpu_node = of_parse_phandle(node, "cpus", i);
if (WARN_ON(!cpu_node))
continue;
cpu = of_cpu_node_to_id(cpu_node);
of_node_put(cpu_node);
if (WARN_ON(cpu < 0))
continue;
if (iaffids[i] & ~iaffid_mask) {
pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n",
cpu, iaffids[i]);
continue;
}
per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i];
per_cpu(cpu_iaffid, cpu).valid = true;
/* We also know that the CPU is connected to this IRS */
per_cpu(per_cpu_irs_data, cpu) = irs_data;
}
return ret;
}
static void irs_setup_pri_bits(u32 idr1)
{
switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) {
case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS:
gicv5_global_data.irs_pri_bits = 1;
break;
case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS:
gicv5_global_data.irs_pri_bits = 2;
break;
case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS:
gicv5_global_data.irs_pri_bits = 3;
break;
case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS:
gicv5_global_data.irs_pri_bits = 4;
break;
case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS:
gicv5_global_data.irs_pri_bits = 5;
break;
default:
pr_warn("Detected wrong IDR priority bits value 0x%lx\n",
FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1));
gicv5_global_data.irs_pri_bits = 1;
break;
}
}
static int __init gicv5_irs_init(struct device_node *node)
{
struct gicv5_irs_chip_data *irs_data;
void __iomem *irs_base;
u32 idr, spi_count;
u8 iaffid_bits;
int ret;
irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL);
if (!irs_data)
return -ENOMEM;
raw_spin_lock_init(&irs_data->spi_config_lock);
ret = of_property_match_string(node, "reg-names", "ns-config");
if (ret < 0) {
pr_err("%pOF: ns-config reg-name not present\n", node);
goto out_err;
}
irs_base = of_io_request_and_map(node, ret, of_node_full_name(node));
if (IS_ERR(irs_base)) {
pr_err("%pOF: unable to map GICv5 IRS registers\n", node);
ret = PTR_ERR(irs_base);
goto out_err;
}
gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode);
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1;
ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits);
if (ret) {
pr_err("Failed to parse CPU IAFFIDs from the device tree!\n");
goto out_iomem;
}
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2);
if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr),
"LPI support not available - no IPIs, can't proceed\n")) {
ret = -ENODEV;
goto out_iomem;
}
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7);
irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr);
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6);
irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr);
if (irs_data->spi_range) {
pr_info("%s detected SPI range [%u-%u]\n",
of_node_full_name(node),
irs_data->spi_min,
irs_data->spi_min +
irs_data->spi_range - 1);
}
/*
* Do the global setting only on the first IRS.
* Global properties (iaffid_bits, global spi count) are guaranteed to
* be consistent across IRSes by the architecture.
*/
if (list_empty(&irs_nodes)) {
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1);
irs_setup_pri_bits(idr);
idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5);
spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr);
gicv5_global_data.global_spi_count = spi_count;
gicv5_init_lpi_domain();
pr_debug("Detected %u SPIs globally\n", spi_count);
}
list_add_tail(&irs_data->entry, &irs_nodes);
return 0;
out_iomem:
iounmap(irs_base);
out_err:
kfree(irs_data);
return ret;
}
void __init gicv5_irs_remove(void)
{
struct gicv5_irs_chip_data *irs_data, *tmp_data;
gicv5_free_lpi_domain();
gicv5_deinit_lpis();
list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) {
iounmap(irs_data->irs_base);
list_del(&irs_data->entry);
kfree(irs_data);
}
}
int __init gicv5_irs_enable(void)
{
struct gicv5_irs_chip_data *irs_data;
int ret;
irs_data = list_first_entry_or_null(&irs_nodes,
struct gicv5_irs_chip_data, entry);
if (!irs_data)
return -ENODEV;
ret = gicv5_irs_init_ist(irs_data);
if (ret) {
pr_err("Failed to init IST\n");
return ret;
}
return 0;
}
void __init gicv5_irs_its_probe(void)
{
struct gicv5_irs_chip_data *irs_data;
list_for_each_entry(irs_data, &irs_nodes, entry)
gicv5_its_of_probe(to_of_node(irs_data->fwnode));
}
int __init gicv5_irs_of_probe(struct device_node *parent)
{
struct device_node *np;
int ret;
for_each_available_child_of_node(parent, np) {
if (!of_device_is_compatible(np, "arm,gic-v5-irs"))
continue;
ret = gicv5_irs_init(np);
if (ret)
pr_err("Failed to init IRS %s\n", np->full_name);
}
return list_empty(&irs_nodes) ? -ENODEV : 0;
}

1228
drivers/irqchip/irq-gic-v5-its.c Normal file
View File

File diff suppressed because it is too large Load Diff

284
drivers/irqchip/irq-gic-v5-iwb.c Normal file
View File

@ -0,0 +1,284 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* Copyright (C) 2024-2025 ARM Limited, All Rights Reserved.
*/
#define pr_fmt(fmt) "GICv5 IWB: " fmt
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/msi.h>
#include <linux/of.h>
#include <linux/of_address.h>
#include <linux/of_platform.h>
#include <linux/irqchip.h>
#include <linux/irqchip/arm-gic-v5.h>
struct gicv5_iwb_chip_data {
void __iomem *iwb_base;
u16 nr_regs;
};
static u32 iwb_readl_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 reg_offset)
{
return readl_relaxed(iwb_node->iwb_base + reg_offset);
}
static void iwb_writel_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 val,
const u32 reg_offset)
{
writel_relaxed(val, iwb_node->iwb_base + reg_offset);
}
static int gicv5_iwb_wait_for_wenabler(struct gicv5_iwb_chip_data *iwb_node)
{
return gicv5_wait_for_op_atomic(iwb_node->iwb_base, GICV5_IWB_WENABLE_STATUSR,
GICV5_IWB_WENABLE_STATUSR_IDLE, NULL);
}
static int __gicv5_iwb_set_wire_enable(struct gicv5_iwb_chip_data *iwb_node,
u32 iwb_wire, bool enable)
{
u32 n = iwb_wire / 32;
u8 i = iwb_wire % 32;
u32 val;
if (n >= iwb_node->nr_regs) {
pr_err("IWB_WENABLER<n> is invalid for n=%u\n", n);
return -EINVAL;
}
/*
* Enable IWB wire/pin at this point
* Note: This is not the same as enabling the interrupt
*/
val = iwb_readl_relaxed(iwb_node, GICV5_IWB_WENABLER + (4 * n));
if (enable)
val |= BIT(i);
else
val &= ~BIT(i);
iwb_writel_relaxed(iwb_node, val, GICV5_IWB_WENABLER + (4 * n));
return gicv5_iwb_wait_for_wenabler(iwb_node);
}
static int gicv5_iwb_enable_wire(struct gicv5_iwb_chip_data *iwb_node,
u32 iwb_wire)
{
return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, true);
}
static int gicv5_iwb_disable_wire(struct gicv5_iwb_chip_data *iwb_node,
u32 iwb_wire)
{
return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, false);
}
static void gicv5_iwb_irq_disable(struct irq_data *d)
{
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
gicv5_iwb_disable_wire(iwb_node, d->hwirq);
irq_chip_disable_parent(d);
}
static void gicv5_iwb_irq_enable(struct irq_data *d)
{
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
gicv5_iwb_enable_wire(iwb_node, d->hwirq);
irq_chip_enable_parent(d);
}
static int gicv5_iwb_set_type(struct irq_data *d, unsigned int type)
{
struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d);
u32 iwb_wire, n, wtmr;
u8 i;
iwb_wire = d->hwirq;
i = iwb_wire % 32;
n = iwb_wire / 32;
if (n >= iwb_node->nr_regs) {
pr_err_once("reg %u out of range\n", n);
return -EINVAL;
}
wtmr = iwb_readl_relaxed(iwb_node, GICV5_IWB_WTMR + (4 * n));
switch (type) {
case IRQ_TYPE_LEVEL_HIGH:
case IRQ_TYPE_LEVEL_LOW:
wtmr |= BIT(i);
break;
case IRQ_TYPE_EDGE_RISING:
case IRQ_TYPE_EDGE_FALLING:
wtmr &= ~BIT(i);
break;
default:
pr_debug("unexpected wire trigger mode");
return -EINVAL;
}
iwb_writel_relaxed(iwb_node, wtmr, GICV5_IWB_WTMR + (4 * n));
return 0;
}
static void gicv5_iwb_domain_set_desc(msi_alloc_info_t *alloc_info, struct msi_desc *desc)
{
alloc_info->desc = desc;
alloc_info->hwirq = (u32)desc->data.icookie.value;
}
static int gicv5_iwb_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec,
irq_hw_number_t *hwirq,
unsigned int *type)
{
if (!is_of_node(fwspec->fwnode))
return -EINVAL;
if (fwspec->param_count < 2)
return -EINVAL;
/*
* param[0] is be the wire
* param[1] is the interrupt type
*/
*hwirq = fwspec->param[0];
*type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK;
return 0;
}
static void gicv5_iwb_write_msi_msg(struct irq_data *d, struct msi_msg *msg) {}
static const struct msi_domain_template iwb_msi_template = {
.chip = {
.name = "GICv5-IWB",
.irq_mask = irq_chip_mask_parent,
.irq_unmask = irq_chip_unmask_parent,
.irq_enable = gicv5_iwb_irq_enable,
.irq_disable = gicv5_iwb_irq_disable,
.irq_eoi = irq_chip_eoi_parent,
.irq_set_type = gicv5_iwb_set_type,
.irq_write_msi_msg = gicv5_iwb_write_msi_msg,
.irq_set_affinity = irq_chip_set_affinity_parent,
.irq_get_irqchip_state = irq_chip_get_parent_state,
.irq_set_irqchip_state = irq_chip_set_parent_state,
.flags = IRQCHIP_SET_TYPE_MASKED |
IRQCHIP_SKIP_SET_WAKE |
IRQCHIP_MASK_ON_SUSPEND,
},
.ops = {
.set_desc = gicv5_iwb_domain_set_desc,
.msi_translate = gicv5_iwb_irq_domain_translate,
},
.info = {
.bus_token = DOMAIN_BUS_WIRED_TO_MSI,
.flags = MSI_FLAG_USE_DEV_FWNODE,
},
.alloc_info = {
.flags = MSI_ALLOC_FLAGS_FIXED_MSG_DATA,
},
};
static bool gicv5_iwb_create_device_domain(struct device *dev, unsigned int size,
struct gicv5_iwb_chip_data *iwb_node)
{
if (WARN_ON_ONCE(!dev->msi.domain))
return false;
return msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN,
&iwb_msi_template, size,
NULL, iwb_node);
}
static struct gicv5_iwb_chip_data *
gicv5_iwb_init_bases(void __iomem *iwb_base, struct platform_device *pdev)
{
u32 nr_wires, idr0, cr0;
unsigned int n;
int ret;
struct gicv5_iwb_chip_data *iwb_node __free(kfree) = kzalloc(sizeof(*iwb_node),
GFP_KERNEL);
if (!iwb_node)
return ERR_PTR(-ENOMEM);
iwb_node->iwb_base = iwb_base;
idr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_IDR0);
nr_wires = (FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1) * 32;
cr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_CR0);
if (!FIELD_GET(GICV5_IWB_CR0_IWBEN, cr0)) {
dev_err(&pdev->dev, "IWB must be enabled in firmware\n");
return ERR_PTR(-EINVAL);
}
iwb_node->nr_regs = FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1;
for (n = 0; n < iwb_node->nr_regs; n++)
iwb_writel_relaxed(iwb_node, 0, GICV5_IWB_WENABLER + (sizeof(u32) * n));
ret = gicv5_iwb_wait_for_wenabler(iwb_node);
if (ret)
return ERR_PTR(ret);
if (!gicv5_iwb_create_device_domain(&pdev->dev, nr_wires, iwb_node))
return ERR_PTR(-ENOMEM);
return_ptr(iwb_node);
}
static int gicv5_iwb_device_probe(struct platform_device *pdev)
{
struct gicv5_iwb_chip_data *iwb_node;
void __iomem *iwb_base;
struct resource *res;
int ret;
res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
if (!res)
return -EINVAL;
iwb_base = devm_ioremap(&pdev->dev, res->start, resource_size(res));
if (!iwb_base) {
dev_err(&pdev->dev, "failed to ioremap %pR\n", res);
return -ENOMEM;
}
iwb_node = gicv5_iwb_init_bases(iwb_base, pdev);
if (IS_ERR(iwb_node)) {
ret = PTR_ERR(iwb_node);
goto out_unmap;
}
return 0;
out_unmap:
iounmap(iwb_base);
return ret;
}
static const struct of_device_id gicv5_iwb_of_match[] = {
{ .compatible = "arm,gic-v5-iwb" },
{ /* END */ }
};
MODULE_DEVICE_TABLE(of, gicv5_iwb_of_match);
static struct platform_driver gicv5_iwb_platform_driver = {
.driver = {
.name = "GICv5 IWB",
.of_match_table = gicv5_iwb_of_match,
.suppress_bind_attrs = true,
},
.probe = gicv5_iwb_device_probe,
};
module_platform_driver(gicv5_iwb_platform_driver);

1137
drivers/irqchip/irq-gic-v5.c Normal file
View File

File diff suppressed because it is too large Load Diff

2
drivers/irqchip/irq-gic.c
View File

@ -54,7 +54,7 @@
static void gic_check_cpu_features(void)
{
WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_GIC_CPUIF_SYSREGS),
WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF),
TAINT_CPU_OUT_OF_SPEC,
"GICv3 system registers enabled, broken firmware!\n");
}

5
drivers/irqchip/irq-msi-lib.c
View File

@ -133,11 +133,14 @@ int msi_lib_irq_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec,
{
const struct msi_parent_ops *ops = d->msi_parent_ops;
u32 busmask = BIT(bus_token);
struct fwnode_handle *fwh;
if (!ops)
return 0;
if (fwspec->fwnode != d->fwnode || fwspec->param_count != 0)
fwh = d->flags & IRQ_DOMAIN_FLAG_FWNODE_PARENT ? fwnode_get_parent(fwspec->fwnode)
: fwspec->fwnode;
if (fwh != d->fwnode || fwspec->param_count != 0)
return 0;
/* Handle pure domain searches */

22
drivers/of/irq.c
View File

@ -670,8 +670,20 @@ err:
}
}
static u32 __of_msi_map_id(struct device *dev, struct device_node **np,
u32 id_in)
/**
* of_msi_xlate - map a MSI ID and find relevant MSI controller node
* @dev: device for which the mapping is to be done.
* @msi_np: Pointer to store the MSI controller node
* @id_in: Device ID.
*
* Walk up the device hierarchy looking for devices with a "msi-map"
* property. If found, apply the mapping to @id_in. @msi_np pointed
* value must be NULL on entry, if an MSI controller is found @msi_np is
* initialized to the MSI controller node with a reference held.
*
* Returns: The mapped MSI id.
*/
u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
{
struct device *parent_dev;
u32 id_out = id_in;
@ -682,7 +694,7 @@ static u32 __of_msi_map_id(struct device *dev, struct device_node **np,
*/
for (parent_dev = dev; parent_dev; parent_dev = parent_dev->parent)
if (!of_map_id(parent_dev->of_node, id_in, "msi-map",
"msi-map-mask", np, &id_out))
"msi-map-mask", msi_np, &id_out))
break;
return id_out;
}
@ -700,7 +712,7 @@ static u32 __of_msi_map_id(struct device *dev, struct device_node **np,
*/
u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in)
{
return __of_msi_map_id(dev, &msi_np, id_in);
return of_msi_xlate(dev, &msi_np, id_in);
}
/**
@ -719,7 +731,7 @@ struct irq_domain *of_msi_map_get_device_domain(struct device *dev, u32 id,
{
struct device_node *np = NULL;
__of_msi_map_id(dev, &np, id);
of_msi_xlate(dev, &np, id);
return irq_find_matching_host(np, bus_token);
}

20
drivers/pci/msi/irqdomain.c
View File

@ -427,6 +427,26 @@ u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
return rid;
}
/**
* pci_msi_map_rid_ctlr_node - Get the MSI controller node and MSI requester id (RID)
* @pdev: The PCI device
* @node: Pointer to store the MSI controller device node
*
* Use the firmware data to find the MSI controller node for @pdev.
* If found map the RID and initialize @node with it. @node value must
* be set to NULL on entry.
*
* Returns: The RID.
*/
u32 pci_msi_map_rid_ctlr_node(struct pci_dev *pdev, struct device_node **node)
{
u32 rid = pci_dev_id(pdev);
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
return of_msi_xlate(&pdev->dev, node, rid);
}
/**
* pci_msi_get_device_domain - Get the MSI domain for a given PCI device
* @pdev: The PCI device

1
include/asm-generic/msi.h
View File

@ -33,6 +33,7 @@ typedef struct msi_alloc_info {
/* Device generating MSIs is proxying for another device */
#define MSI_ALLOC_FLAGS_PROXY_DEVICE (1UL << 0)
#define MSI_ALLOC_FLAGS_FIXED_MSG_DATA (1UL << 1)
#define GENERIC_MSI_DOMAIN_OPS 1

9
include/kvm/arm_vgic.h
View File

@ -38,6 +38,7 @@
enum vgic_type {
VGIC_V2, /* Good ol' GICv2 */
VGIC_V3, /* New fancy GICv3 */
VGIC_V5, /* Newer, fancier GICv5 */
};
/* same for all guests, as depending only on the _host's_ GIC model */
@ -77,9 +78,12 @@ struct vgic_global {
/* Pseudo GICv3 from outer space */
bool no_hw_deactivation;
/* GIC system register CPU interface */
/* GICv3 system register CPU interface */
struct static_key_false gicv3_cpuif;
/* GICv3 compat mode on a GICv5 host */
bool has_gcie_v3_compat;
u32 ich_vtr_el2;
};
@ -264,6 +268,9 @@ struct vgic_dist {
/* distributor enabled */
bool enabled;
/* Supports SGIs without active state */
bool nassgicap;
/* Wants SGIs without active state */
bool nassgireq;

394
include/linux/irqchip/arm-gic-v5.h Normal file
View File

@ -0,0 +1,394 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
* Copyright (C) 2025 ARM Limited, All Rights Reserved.
*/
#ifndef __LINUX_IRQCHIP_ARM_GIC_V5_H
#define __LINUX_IRQCHIP_ARM_GIC_V5_H
#include <linux/iopoll.h>
#include <asm/cacheflush.h>
#include <asm/smp.h>
#include <asm/sysreg.h>
#define GICV5_IPIS_PER_CPU MAX_IPI
/*
* INTID handling
*/
#define GICV5_HWIRQ_ID GENMASK(23, 0)
#define GICV5_HWIRQ_TYPE GENMASK(31, 29)
#define GICV5_HWIRQ_INTID GENMASK_ULL(31, 0)
#define GICV5_HWIRQ_TYPE_PPI UL(0x1)
#define GICV5_HWIRQ_TYPE_LPI UL(0x2)
#define GICV5_HWIRQ_TYPE_SPI UL(0x3)
/*
* Tables attributes
*/
#define GICV5_NO_READ_ALLOC 0b0
#define GICV5_READ_ALLOC 0b1
#define GICV5_NO_WRITE_ALLOC 0b0
#define GICV5_WRITE_ALLOC 0b1
#define GICV5_NON_CACHE 0b00
#define GICV5_WB_CACHE 0b01
#define GICV5_WT_CACHE 0b10
#define GICV5_NON_SHARE 0b00
#define GICV5_OUTER_SHARE 0b10
#define GICV5_INNER_SHARE 0b11
/*
* IRS registers and tables structures
*/
#define GICV5_IRS_IDR1 0x0004
#define GICV5_IRS_IDR2 0x0008
#define GICV5_IRS_IDR5 0x0014
#define GICV5_IRS_IDR6 0x0018
#define GICV5_IRS_IDR7 0x001c
#define GICV5_IRS_CR0 0x0080
#define GICV5_IRS_CR1 0x0084
#define GICV5_IRS_SYNCR 0x00c0
#define GICV5_IRS_SYNC_STATUSR 0x00c4
#define GICV5_IRS_SPI_SELR 0x0108
#define GICV5_IRS_SPI_CFGR 0x0114
#define GICV5_IRS_SPI_STATUSR 0x0118
#define GICV5_IRS_PE_SELR 0x0140
#define GICV5_IRS_PE_STATUSR 0x0144
#define GICV5_IRS_PE_CR0 0x0148
#define GICV5_IRS_IST_BASER 0x0180
#define GICV5_IRS_IST_CFGR 0x0190
#define GICV5_IRS_IST_STATUSR 0x0194
#define GICV5_IRS_MAP_L2_ISTR 0x01c0
#define GICV5_IRS_IDR1_PRIORITY_BITS GENMASK(22, 20)
#define GICV5_IRS_IDR1_IAFFID_BITS GENMASK(19, 16)
#define GICV5_IRS_IDR1_PRIORITY_BITS_1BITS 0b000
#define GICV5_IRS_IDR1_PRIORITY_BITS_2BITS 0b001
#define GICV5_IRS_IDR1_PRIORITY_BITS_3BITS 0b010
#define GICV5_IRS_IDR1_PRIORITY_BITS_4BITS 0b011
#define GICV5_IRS_IDR1_PRIORITY_BITS_5BITS 0b100
#define GICV5_IRS_IDR2_ISTMD_SZ GENMASK(19, 15)
#define GICV5_IRS_IDR2_ISTMD BIT(14)
#define GICV5_IRS_IDR2_IST_L2SZ GENMASK(13, 11)
#define GICV5_IRS_IDR2_IST_LEVELS BIT(10)
#define GICV5_IRS_IDR2_MIN_LPI_ID_BITS GENMASK(9, 6)
#define GICV5_IRS_IDR2_LPI BIT(5)
#define GICV5_IRS_IDR2_ID_BITS GENMASK(4, 0)
#define GICV5_IRS_IDR5_SPI_RANGE GENMASK(24, 0)
#define GICV5_IRS_IDR6_SPI_IRS_RANGE GENMASK(24, 0)
#define GICV5_IRS_IDR7_SPI_BASE GENMASK(23, 0)
#define GICV5_IRS_IST_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(11), (r))
#define GICV5_IRS_IST_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(12), (r))
#define GICV5_IRS_IST_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(13), (r))
#define GICV5_IRS_CR0_IDLE BIT(1)
#define GICV5_IRS_CR0_IRSEN BIT(0)
#define GICV5_IRS_CR1_VPED_WA BIT(15)
#define GICV5_IRS_CR1_VPED_RA BIT(14)
#define GICV5_IRS_CR1_VMD_WA BIT(13)
#define GICV5_IRS_CR1_VMD_RA BIT(12)
#define GICV5_IRS_CR1_VPET_WA BIT(11)
#define GICV5_IRS_CR1_VPET_RA BIT(10)
#define GICV5_IRS_CR1_VMT_WA BIT(9)
#define GICV5_IRS_CR1_VMT_RA BIT(8)
#define GICV5_IRS_CR1_IST_WA BIT(7)
#define GICV5_IRS_CR1_IST_RA BIT(6)
#define GICV5_IRS_CR1_IC GENMASK(5, 4)
#define GICV5_IRS_CR1_OC GENMASK(3, 2)
#define GICV5_IRS_CR1_SH GENMASK(1, 0)
#define GICV5_IRS_SYNCR_SYNC BIT(31)
#define GICV5_IRS_SYNC_STATUSR_IDLE BIT(0)
#define GICV5_IRS_SPI_STATUSR_V BIT(1)
#define GICV5_IRS_SPI_STATUSR_IDLE BIT(0)
#define GICV5_IRS_SPI_SELR_ID GENMASK(23, 0)
#define GICV5_IRS_SPI_CFGR_TM BIT(0)
#define GICV5_IRS_PE_SELR_IAFFID GENMASK(15, 0)
#define GICV5_IRS_PE_STATUSR_V BIT(1)
#define GICV5_IRS_PE_STATUSR_IDLE BIT(0)
#define GICV5_IRS_PE_CR0_DPS BIT(0)
#define GICV5_IRS_IST_STATUSR_IDLE BIT(0)
#define GICV5_IRS_IST_CFGR_STRUCTURE BIT(16)
#define GICV5_IRS_IST_CFGR_ISTSZ GENMASK(8, 7)
#define GICV5_IRS_IST_CFGR_L2SZ GENMASK(6, 5)
#define GICV5_IRS_IST_CFGR_LPI_ID_BITS GENMASK(4, 0)
#define GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR 0b0
#define GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL 0b1
#define GICV5_IRS_IST_CFGR_ISTSZ_4 0b00
#define GICV5_IRS_IST_CFGR_ISTSZ_8 0b01
#define GICV5_IRS_IST_CFGR_ISTSZ_16 0b10
#define GICV5_IRS_IST_CFGR_L2SZ_4K 0b00
#define GICV5_IRS_IST_CFGR_L2SZ_16K 0b01
#define GICV5_IRS_IST_CFGR_L2SZ_64K 0b10
#define GICV5_IRS_IST_BASER_ADDR_MASK GENMASK_ULL(55, 6)
#define GICV5_IRS_IST_BASER_VALID BIT_ULL(0)
#define GICV5_IRS_MAP_L2_ISTR_ID GENMASK(23, 0)
#define GICV5_ISTL1E_VALID BIT_ULL(0)
#define GICV5_ISTL1E_L2_ADDR_MASK GENMASK_ULL(55, 12)
/*
* ITS registers and tables structures
*/
#define GICV5_ITS_IDR1 0x0004
#define GICV5_ITS_IDR2 0x0008
#define GICV5_ITS_CR0 0x0080
#define GICV5_ITS_CR1 0x0084
#define GICV5_ITS_DT_BASER 0x00c0
#define GICV5_ITS_DT_CFGR 0x00d0
#define GICV5_ITS_DIDR 0x0100
#define GICV5_ITS_EIDR 0x0108
#define GICV5_ITS_INV_EVENTR 0x010c
#define GICV5_ITS_INV_DEVICER 0x0110
#define GICV5_ITS_STATUSR 0x0120
#define GICV5_ITS_SYNCR 0x0140
#define GICV5_ITS_SYNC_STATUSR 0x0148
#define GICV5_ITS_IDR1_L2SZ GENMASK(10, 8)
#define GICV5_ITS_IDR1_ITT_LEVELS BIT(7)
#define GICV5_ITS_IDR1_DT_LEVELS BIT(6)
#define GICV5_ITS_IDR1_DEVICEID_BITS GENMASK(5, 0)
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(r) FIELD_GET(BIT(8), (r))
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_16KB(r) FIELD_GET(BIT(9), (r))
#define GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(r) FIELD_GET(BIT(10), (r))
#define GICV5_ITS_IDR2_XDMN_EVENTs GENMASK(6, 5)
#define GICV5_ITS_IDR2_EVENTID_BITS GENMASK(4, 0)
#define GICV5_ITS_CR0_IDLE BIT(1)
#define GICV5_ITS_CR0_ITSEN BIT(0)
#define GICV5_ITS_CR1_ITT_RA BIT(7)
#define GICV5_ITS_CR1_DT_RA BIT(6)
#define GICV5_ITS_CR1_IC GENMASK(5, 4)
#define GICV5_ITS_CR1_OC GENMASK(3, 2)
#define GICV5_ITS_CR1_SH GENMASK(1, 0)
#define GICV5_ITS_DT_CFGR_STRUCTURE BIT(16)
#define GICV5_ITS_DT_CFGR_L2SZ GENMASK(7, 6)
#define GICV5_ITS_DT_CFGR_DEVICEID_BITS GENMASK(5, 0)
#define GICV5_ITS_DT_BASER_ADDR_MASK GENMASK_ULL(55, 3)
#define GICV5_ITS_INV_DEVICER_I BIT(31)
#define GICV5_ITS_INV_DEVICER_EVENTID_BITS GENMASK(5, 1)
#define GICV5_ITS_INV_DEVICER_L1 BIT(0)
#define GICV5_ITS_DIDR_DEVICEID GENMASK_ULL(31, 0)
#define GICV5_ITS_EIDR_EVENTID GENMASK(15, 0)
#define GICV5_ITS_INV_EVENTR_I BIT(31)
#define GICV5_ITS_INV_EVENTR_ITT_L2SZ GENMASK(2, 1)
#define GICV5_ITS_INV_EVENTR_L1 BIT(0)
#define GICV5_ITS_STATUSR_IDLE BIT(0)
#define GICV5_ITS_SYNCR_SYNC BIT_ULL(63)
#define GICV5_ITS_SYNCR_SYNCALL BIT_ULL(32)
#define GICV5_ITS_SYNCR_DEVICEID GENMASK_ULL(31, 0)
#define GICV5_ITS_SYNC_STATUSR_IDLE BIT(0)
#define GICV5_DTL1E_VALID BIT_ULL(0)
/* Note that there is no shift for the address by design */
#define GICV5_DTL1E_L2_ADDR_MASK GENMASK_ULL(55, 3)
#define GICV5_DTL1E_SPAN GENMASK_ULL(63, 60)
#define GICV5_DTL2E_VALID BIT_ULL(0)
#define GICV5_DTL2E_ITT_L2SZ GENMASK_ULL(2, 1)
/* Note that there is no shift for the address by design */
#define GICV5_DTL2E_ITT_ADDR_MASK GENMASK_ULL(55, 3)
#define GICV5_DTL2E_ITT_DSWE BIT_ULL(57)
#define GICV5_DTL2E_ITT_STRUCTURE BIT_ULL(58)
#define GICV5_DTL2E_EVENT_ID_BITS GENMASK_ULL(63, 59)
#define GICV5_ITTL1E_VALID BIT_ULL(0)
/* Note that there is no shift for the address by design */
#define GICV5_ITTL1E_L2_ADDR_MASK GENMASK_ULL(55, 3)
#define GICV5_ITTL1E_SPAN GENMASK_ULL(63, 60)
#define GICV5_ITTL2E_LPI_ID GENMASK_ULL(23, 0)
#define GICV5_ITTL2E_DAC GENMASK_ULL(29, 28)
#define GICV5_ITTL2E_VIRTUAL BIT_ULL(30)
#define GICV5_ITTL2E_VALID BIT_ULL(31)
#define GICV5_ITTL2E_VM_ID GENMASK_ULL(47, 32)
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_4k 0b00
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_16k 0b01
#define GICV5_ITS_DT_ITT_CFGR_L2SZ_64k 0b10
#define GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR 0
#define GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL 1
#define GICV5_ITS_HWIRQ_DEVICE_ID GENMASK_ULL(31, 0)
#define GICV5_ITS_HWIRQ_EVENT_ID GENMASK_ULL(63, 32)
/*
* IWB registers
*/
#define GICV5_IWB_IDR0 0x0000
#define GICV5_IWB_CR0 0x0080
#define GICV5_IWB_WENABLE_STATUSR 0x00c0
#define GICV5_IWB_WENABLER 0x2000
#define GICV5_IWB_WTMR 0x4000
#define GICV5_IWB_IDR0_INT_DOMS GENMASK(14, 11)
#define GICV5_IWB_IDR0_IW_RANGE GENMASK(10, 0)
#define GICV5_IWB_CR0_IDLE BIT(1)
#define GICV5_IWB_CR0_IWBEN BIT(0)
#define GICV5_IWB_WENABLE_STATUSR_IDLE BIT(0)
/*
* Global Data structures and functions
*/
struct gicv5_chip_data {
struct fwnode_handle *fwnode;
struct irq_domain *ppi_domain;
struct irq_domain *spi_domain;
struct irq_domain *lpi_domain;
struct irq_domain *ipi_domain;
u32 global_spi_count;
u8 cpuif_pri_bits;
u8 cpuif_id_bits;
u8 irs_pri_bits;
struct {
__le64 *l1ist_addr;
u32 l2_size;
u8 l2_bits;
bool l2;
} ist;
};
extern struct gicv5_chip_data gicv5_global_data __read_mostly;
struct gicv5_irs_chip_data {
struct list_head entry;
struct fwnode_handle *fwnode;
void __iomem *irs_base;
u32 flags;
u32 spi_min;
u32 spi_range;
raw_spinlock_t spi_config_lock;
};
static inline int gicv5_wait_for_op_s_atomic(void __iomem *addr, u32 offset,
const char *reg_s, u32 mask,
u32 *val)
{
void __iomem *reg = addr + offset;
u32 tmp;
int ret;
ret = readl_poll_timeout_atomic(reg, tmp, tmp & mask, 1, 10 * USEC_PER_MSEC);
if (unlikely(ret == -ETIMEDOUT)) {
pr_err_ratelimited("%s timeout...\n", reg_s);
return ret;
}
if (val)
*val = tmp;
return 0;
}
static inline int gicv5_wait_for_op_s(void __iomem *addr, u32 offset,
const char *reg_s, u32 mask)
{
void __iomem *reg = addr + offset;
u32 val;
int ret;
ret = readl_poll_timeout(reg, val, val & mask, 1, 10 * USEC_PER_MSEC);
if (unlikely(ret == -ETIMEDOUT)) {
pr_err_ratelimited("%s timeout...\n", reg_s);
return ret;
}
return 0;
}
#define gicv5_wait_for_op_atomic(base, reg, mask, val) \
gicv5_wait_for_op_s_atomic(base, reg, #reg, mask, val)
#define gicv5_wait_for_op(base, reg, mask) \
gicv5_wait_for_op_s(base, reg, #reg, mask)
void __init gicv5_init_lpi_domain(void);
void __init gicv5_free_lpi_domain(void);
int gicv5_irs_of_probe(struct device_node *parent);
void gicv5_irs_remove(void);
int gicv5_irs_enable(void);
void gicv5_irs_its_probe(void);
int gicv5_irs_register_cpu(int cpuid);
int gicv5_irs_cpu_to_iaffid(int cpu_id, u16 *iaffid);
struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id);
int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type);
int gicv5_irs_iste_alloc(u32 lpi);
void gicv5_irs_syncr(void);
struct gicv5_its_devtab_cfg {
union {
struct {
__le64 *devtab;
} linear;
struct {
__le64 *l1devtab;
__le64 **l2ptrs;
} l2;
};
u32 cfgr;
};
struct gicv5_its_itt_cfg {
union {
struct {
__le64 *itt;
unsigned int num_ents;
} linear;
struct {
__le64 *l1itt;
__le64 **l2ptrs;
unsigned int num_l1_ents;
u8 l2sz;
} l2;
};
u8 event_id_bits;
bool l2itt;
};
void gicv5_init_lpis(u32 max);
void gicv5_deinit_lpis(void);
int gicv5_alloc_lpi(void);
void gicv5_free_lpi(u32 lpi);
void __init gicv5_its_of_probe(struct device_node *parent);
#endif

4
include/linux/irqchip/arm-vgic-info.h
View File

@ -15,6 +15,8 @@ enum gic_type {
GIC_V2,
/* Full GICv3, optionally with v2 compat */
GIC_V3,
/* Full GICv5, optionally with v3 compat */
GIC_V5,
};
struct gic_kvm_info {
@ -34,6 +36,8 @@ struct gic_kvm_info {
bool has_v4_1;
/* Deactivation impared, subpar stuff */
bool no_hw_deactivation;
/* v3 compat support (GICv5 hosts, only) */
bool has_gcie_v3_compat;
};
#ifdef CONFIG_KVM

3
include/linux/irqdomain.h
View File

@ -212,6 +212,9 @@ enum {
/* Address and data pair is mutable when irq_set_affinity() */
IRQ_DOMAIN_FLAG_MSI_IMMUTABLE = (1 << 11),
/* IRQ domain requires parent fwnode matching */
IRQ_DOMAIN_FLAG_FWNODE_PARENT = (1 << 12),
/*
* Flags starting from IRQ_DOMAIN_FLAG_NONCORE are reserved
* for implementation specific purposes and ignored by the

1
include/linux/msi.h
View File

@ -705,6 +705,7 @@ struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
struct msi_domain_info *info,
struct irq_domain *parent);
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev);
u32 pci_msi_map_rid_ctlr_node(struct pci_dev *pdev, struct device_node **node);
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev);
#else /* CONFIG_PCI_MSI */
static inline struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)

5
include/linux/of_irq.h
View File

@ -54,6 +54,7 @@ extern struct irq_domain *of_msi_map_get_device_domain(struct device *dev,
u32 id,
u32 bus_token);
extern void of_msi_configure(struct device *dev, const struct device_node *np);
extern u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in);
u32 of_msi_map_id(struct device *dev, struct device_node *msi_np, u32 id_in);
#else
static inline void of_irq_init(const struct of_device_id *matches)
@ -100,6 +101,10 @@ static inline struct irq_domain *of_msi_map_get_device_domain(struct device *dev
static inline void of_msi_configure(struct device *dev, struct device_node *np)
{
}
static inline u32 of_msi_xlate(struct device *dev, struct device_node **msi_np, u32 id_in)
{
return id_in;
}
static inline u32 of_msi_map_id(struct device *dev,
struct device_node *msi_np, u32 id_in)
{

1
include/uapi/linux/kvm.h
View File

@ -961,6 +961,7 @@ struct kvm_enable_cap {
#define KVM_CAP_ARM_EL2 240
#define KVM_CAP_ARM_EL2_E2H0 241
#define KVM_CAP_RISCV_MP_STATE_RESET 242
#define KVM_CAP_ARM_CACHEABLE_PFNMAP_SUPPORTED 243
struct kvm_irq_routing_irqchip {
__u32 irqchip;

2
tools/testing/selftests/kvm/Makefile.kvm
View File

@ -158,7 +158,7 @@ TEST_GEN_PROGS_arm64 += arm64/arch_timer_edge_cases
TEST_GEN_PROGS_arm64 += arm64/debug-exceptions
TEST_GEN_PROGS_arm64 += arm64/host_sve
TEST_GEN_PROGS_arm64 += arm64/hypercalls
TEST_GEN_PROGS_arm64 += arm64/mmio_abort
TEST_GEN_PROGS_arm64 += arm64/external_aborts
TEST_GEN_PROGS_arm64 += arm64/page_fault_test
TEST_GEN_PROGS_arm64 += arm64/psci_test
TEST_GEN_PROGS_arm64 += arm64/set_id_regs

330
tools/testing/selftests/kvm/arm64/external_aborts.c Normal file
View File

@ -0,0 +1,330 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* external_abort - Tests for userspace external abort injection
*
* Copyright (c) 2024 Google LLC
*/
#include "processor.h"
#include "test_util.h"
#define MMIO_ADDR 0x8000000ULL
#define EXPECTED_SERROR_ISS (ESR_ELx_ISV | 0x1d1ed)
static u64 expected_abort_pc;
static void expect_sea_handler(struct ex_regs *regs)
{
u64 esr = read_sysreg(esr_el1);
GUEST_ASSERT_EQ(regs->pc, expected_abort_pc);
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR);
GUEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
GUEST_DONE();
}
static void unexpected_dabt_handler(struct ex_regs *regs)
{
GUEST_FAIL("Unexpected data abort at PC: %lx\n", regs->pc);
}
static struct kvm_vm *vm_create_with_dabt_handler(struct kvm_vcpu **vcpu, void *guest_code,
handler_fn dabt_handler)
{
struct kvm_vm *vm = vm_create_with_one_vcpu(vcpu, guest_code);
vm_init_descriptor_tables(vm);
vcpu_init_descriptor_tables(*vcpu);
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_DABT_CUR, dabt_handler);
virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1);
return vm;
}
static void vcpu_inject_sea(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_events events = {};
events.exception.ext_dabt_pending = true;
vcpu_events_set(vcpu, &events);
}
static bool vcpu_has_ras(struct kvm_vcpu *vcpu)
{
u64 pfr0 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR0_EL1));
return SYS_FIELD_GET(ID_AA64PFR0_EL1, RAS, pfr0);
}
static bool guest_has_ras(void)
{
return SYS_FIELD_GET(ID_AA64PFR0_EL1, RAS, read_sysreg(id_aa64pfr0_el1));
}
static void vcpu_inject_serror(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_events events = {};
events.exception.serror_pending = true;
if (vcpu_has_ras(vcpu)) {
events.exception.serror_has_esr = true;
events.exception.serror_esr = EXPECTED_SERROR_ISS;
}
vcpu_events_set(vcpu, &events);
}
static void __vcpu_run_expect(struct kvm_vcpu *vcpu, unsigned int cmd)
{
struct ucall uc;
vcpu_run(vcpu);
switch (get_ucall(vcpu, &uc)) {
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
break;
default:
if (uc.cmd == cmd)
return;
TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
}
}
static void vcpu_run_expect_done(struct kvm_vcpu *vcpu)
{
__vcpu_run_expect(vcpu, UCALL_DONE);
}
static void vcpu_run_expect_sync(struct kvm_vcpu *vcpu)
{
__vcpu_run_expect(vcpu, UCALL_SYNC);
}
extern char test_mmio_abort_insn;
static noinline void test_mmio_abort_guest(void)
{
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_abort_insn);
asm volatile("test_mmio_abort_insn:\n\t"
"ldr x0, [%0]\n\t"
: : "r" (MMIO_ADDR) : "x0", "memory");
GUEST_FAIL("MMIO instruction should not retire");
}
/*
* Test that KVM doesn't complete MMIO emulation when userspace has made an
* external abort pending for the instruction.
*/
static void test_mmio_abort(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_abort_guest,
expect_sea_handler);
struct kvm_run *run = vcpu->run;
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
vcpu_inject_sea(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
extern char test_mmio_nisv_insn;
static void test_mmio_nisv_guest(void)
{
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_nisv_insn);
asm volatile("test_mmio_nisv_insn:\n\t"
"ldr x0, [%0], #8\n\t"
: : "r" (MMIO_ADDR) : "x0", "memory");
GUEST_FAIL("MMIO instruction should not retire");
}
/*
* Test that the KVM_RUN ioctl fails for ESR_EL2.ISV=0 MMIO aborts if userspace
* hasn't enabled KVM_CAP_ARM_NISV_TO_USER.
*/
static void test_mmio_nisv(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
unexpected_dabt_handler);
TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN");
TEST_ASSERT_EQ(errno, ENOSYS);
kvm_vm_free(vm);
}
/*
* Test that ESR_EL2.ISV=0 MMIO aborts reach userspace and that an injected SEA
* reaches the guest.
*/
static void test_mmio_nisv_abort(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
expect_sea_handler);
struct kvm_run *run = vcpu->run;
vm_enable_cap(vm, KVM_CAP_ARM_NISV_TO_USER, 1);
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_NISV);
TEST_ASSERT_EQ(run->arm_nisv.fault_ipa, MMIO_ADDR);
vcpu_inject_sea(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
static void unexpected_serror_handler(struct ex_regs *regs)
{
GUEST_FAIL("Took unexpected SError exception");
}
static void test_serror_masked_guest(void)
{
GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
isb();
GUEST_DONE();
}
static void test_serror_masked(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_masked_guest,
unexpected_dabt_handler);
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, unexpected_serror_handler);
vcpu_inject_serror(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
static void expect_serror_handler(struct ex_regs *regs)
{
u64 esr = read_sysreg(esr_el1);
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_SERROR);
if (guest_has_ras())
GUEST_ASSERT_EQ(ESR_ELx_ISS(esr), EXPECTED_SERROR_ISS);
GUEST_DONE();
}
static void test_serror_guest(void)
{
GUEST_ASSERT(read_sysreg(isr_el1) & ISR_EL1_A);
local_serror_enable();
isb();
local_serror_disable();
GUEST_FAIL("Should've taken pending SError exception");
}
static void test_serror(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_guest,
unexpected_dabt_handler);
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
vcpu_inject_serror(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
static void test_serror_emulated_guest(void)
{
GUEST_ASSERT(!(read_sysreg(isr_el1) & ISR_EL1_A));
local_serror_enable();
GUEST_SYNC(0);
local_serror_disable();
GUEST_FAIL("Should've taken unmasked SError exception");
}
static void test_serror_emulated(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_serror_emulated_guest,
unexpected_dabt_handler);
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_serror_handler);
vcpu_run_expect_sync(vcpu);
vcpu_inject_serror(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
static void test_mmio_ease_guest(void)
{
sysreg_clear_set_s(SYS_SCTLR2_EL1, 0, SCTLR2_EL1_EASE);
isb();
test_mmio_abort_guest();
}
/*
* Test that KVM doesn't complete MMIO emulation when userspace has made an
* external abort pending for the instruction.
*/
static void test_mmio_ease(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_ease_guest,
unexpected_dabt_handler);
struct kvm_run *run = vcpu->run;
u64 pfr1;
pfr1 = vcpu_get_reg(vcpu, KVM_ARM64_SYS_REG(SYS_ID_AA64PFR1_EL1));
if (!SYS_FIELD_GET(ID_AA64PFR1_EL1, DF2, pfr1)) {
pr_debug("Skipping %s\n", __func__);
return;
}
/*
* SCTLR2_ELx.EASE changes the exception vector to the SError vector but
* doesn't further modify the exception context (e.g. ESR_ELx, FAR_ELx).
*/
vm_install_exception_handler(vm, VECTOR_ERROR_CURRENT, expect_sea_handler);
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
vcpu_inject_sea(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
int main(void)
{
test_mmio_abort();
test_mmio_nisv();
test_mmio_nisv_abort();
test_serror();
test_serror_masked();
test_serror_emulated();
test_mmio_ease();
}

203
tools/testing/selftests/kvm/arm64/get-reg-list.c
View File

@ -15,6 +15,12 @@
#include "test_util.h"
#include "processor.h"
#define SYS_REG(r) ARM64_SYS_REG(sys_reg_Op0(SYS_ ## r), \
sys_reg_Op1(SYS_ ## r), \
sys_reg_CRn(SYS_ ## r), \
sys_reg_CRm(SYS_ ## r), \
sys_reg_Op2(SYS_ ## r))
struct feature_id_reg {
__u64 reg;
__u64 id_reg;
@ -22,37 +28,43 @@ struct feature_id_reg {
__u64 feat_min;
};
static struct feature_id_reg feat_id_regs[] = {
{
ARM64_SYS_REG(3, 0, 2, 0, 3), /* TCR2_EL1 */
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
0,
1
},
{
ARM64_SYS_REG(3, 0, 10, 2, 2), /* PIRE0_EL1 */
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
8,
1
},
{
ARM64_SYS_REG(3, 0, 10, 2, 3), /* PIR_EL1 */
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
8,
1
},
{
ARM64_SYS_REG(3, 0, 10, 2, 4), /* POR_EL1 */
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
16,
1
},
{
ARM64_SYS_REG(3, 3, 10, 2, 4), /* POR_EL0 */
ARM64_SYS_REG(3, 0, 0, 7, 3), /* ID_AA64MMFR3_EL1 */
16,
1
#define FEAT(id, f, v) \
.id_reg = SYS_REG(id), \
.feat_shift = id ## _ ## f ## _SHIFT, \
.feat_min = id ## _ ## f ## _ ## v
#define REG_FEAT(r, id, f, v) \
{ \
.reg = SYS_REG(r), \
FEAT(id, f, v) \
}
static struct feature_id_reg feat_id_regs[] = {
REG_FEAT(TCR2_EL1, ID_AA64MMFR3_EL1, TCRX, IMP),
REG_FEAT(TCR2_EL2, ID_AA64MMFR3_EL1, TCRX, IMP),
REG_FEAT(PIRE0_EL1, ID_AA64MMFR3_EL1, S1PIE, IMP),
REG_FEAT(PIRE0_EL2, ID_AA64MMFR3_EL1, S1PIE, IMP),
REG_FEAT(PIR_EL1, ID_AA64MMFR3_EL1, S1PIE, IMP),
REG_FEAT(PIR_EL2, ID_AA64MMFR3_EL1, S1PIE, IMP),
REG_FEAT(POR_EL1, ID_AA64MMFR3_EL1, S1POE, IMP),
REG_FEAT(POR_EL0, ID_AA64MMFR3_EL1, S1POE, IMP),
REG_FEAT(POR_EL2, ID_AA64MMFR3_EL1, S1POE, IMP),
REG_FEAT(HCRX_EL2, ID_AA64MMFR1_EL1, HCX, IMP),
REG_FEAT(HFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HFGWTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HFGITR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HDFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HDFGWTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HAFGRTR_EL2, ID_AA64MMFR0_EL1, FGT, IMP),
REG_FEAT(HFGRTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
REG_FEAT(HFGWTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
REG_FEAT(HFGITR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
REG_FEAT(HDFGRTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
REG_FEAT(HDFGWTR2_EL2, ID_AA64MMFR0_EL1, FGT, FGT2),
REG_FEAT(ZCR_EL2, ID_AA64PFR0_EL1, SVE, IMP),
REG_FEAT(SCTLR2_EL1, ID_AA64MMFR3_EL1, SCTLRX, IMP),
REG_FEAT(VDISR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
REG_FEAT(VSESR_EL2, ID_AA64PFR0_EL1, RAS, IMP),
};
bool filter_reg(__u64 reg)
@ -469,6 +481,7 @@ static __u64 base_regs[] = {
ARM64_SYS_REG(3, 0, 1, 0, 0), /* SCTLR_EL1 */
ARM64_SYS_REG(3, 0, 1, 0, 1), /* ACTLR_EL1 */
ARM64_SYS_REG(3, 0, 1, 0, 2), /* CPACR_EL1 */
KVM_ARM64_SYS_REG(SYS_SCTLR2_EL1),
ARM64_SYS_REG(3, 0, 2, 0, 0), /* TTBR0_EL1 */
ARM64_SYS_REG(3, 0, 2, 0, 1), /* TTBR1_EL1 */
ARM64_SYS_REG(3, 0, 2, 0, 2), /* TCR_EL1 */
@ -686,6 +699,62 @@ static __u64 pauth_generic_regs[] = {
ARM64_SYS_REG(3, 0, 2, 3, 1), /* APGAKEYHI_EL1 */
};
static __u64 el2_regs[] = {
SYS_REG(VPIDR_EL2),
SYS_REG(VMPIDR_EL2),
SYS_REG(SCTLR_EL2),
SYS_REG(ACTLR_EL2),
SYS_REG(HCR_EL2),
SYS_REG(MDCR_EL2),
SYS_REG(CPTR_EL2),
SYS_REG(HSTR_EL2),
SYS_REG(HFGRTR_EL2),
SYS_REG(HFGWTR_EL2),
SYS_REG(HFGITR_EL2),
SYS_REG(HACR_EL2),
SYS_REG(ZCR_EL2),
SYS_REG(HCRX_EL2),
SYS_REG(TTBR0_EL2),
SYS_REG(TTBR1_EL2),
SYS_REG(TCR_EL2),
SYS_REG(TCR2_EL2),
SYS_REG(VTTBR_EL2),
SYS_REG(VTCR_EL2),
SYS_REG(VNCR_EL2),
SYS_REG(HDFGRTR2_EL2),
SYS_REG(HDFGWTR2_EL2),
SYS_REG(HFGRTR2_EL2),
SYS_REG(HFGWTR2_EL2),
SYS_REG(HDFGRTR_EL2),
SYS_REG(HDFGWTR_EL2),
SYS_REG(HAFGRTR_EL2),
SYS_REG(HFGITR2_EL2),
SYS_REG(SPSR_EL2),
SYS_REG(ELR_EL2),
SYS_REG(AFSR0_EL2),
SYS_REG(AFSR1_EL2),
SYS_REG(ESR_EL2),
SYS_REG(FAR_EL2),
SYS_REG(HPFAR_EL2),
SYS_REG(MAIR_EL2),
SYS_REG(PIRE0_EL2),
SYS_REG(PIR_EL2),
SYS_REG(POR_EL2),
SYS_REG(AMAIR_EL2),
SYS_REG(VBAR_EL2),
SYS_REG(CONTEXTIDR_EL2),
SYS_REG(TPIDR_EL2),
SYS_REG(CNTVOFF_EL2),
SYS_REG(CNTHCTL_EL2),
SYS_REG(CNTHP_CTL_EL2),
SYS_REG(CNTHP_CVAL_EL2),
SYS_REG(CNTHV_CTL_EL2),
SYS_REG(CNTHV_CVAL_EL2),
SYS_REG(SP_EL2),
SYS_REG(VDISR_EL2),
SYS_REG(VSESR_EL2),
};
#define BASE_SUBLIST \
{ "base", .regs = base_regs, .regs_n = ARRAY_SIZE(base_regs), }
#define VREGS_SUBLIST \
@ -712,6 +781,14 @@ static __u64 pauth_generic_regs[] = {
.regs = pauth_generic_regs, \
.regs_n = ARRAY_SIZE(pauth_generic_regs), \
}
#define EL2_SUBLIST \
{ \
.name = "EL2", \
.capability = KVM_CAP_ARM_EL2, \
.feature = KVM_ARM_VCPU_HAS_EL2, \
.regs = el2_regs, \
.regs_n = ARRAY_SIZE(el2_regs), \
}
static struct vcpu_reg_list vregs_config = {
.sublists = {
@ -761,6 +838,65 @@ static struct vcpu_reg_list pauth_pmu_config = {
},
};
static struct vcpu_reg_list el2_vregs_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
VREGS_SUBLIST,
{0},
},
};
static struct vcpu_reg_list el2_vregs_pmu_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
VREGS_SUBLIST,
PMU_SUBLIST,
{0},
},
};
static struct vcpu_reg_list el2_sve_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
SVE_SUBLIST,
{0},
},
};
static struct vcpu_reg_list el2_sve_pmu_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
SVE_SUBLIST,
PMU_SUBLIST,
{0},
},
};
static struct vcpu_reg_list el2_pauth_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
VREGS_SUBLIST,
PAUTH_SUBLIST,
{0},
},
};
static struct vcpu_reg_list el2_pauth_pmu_config = {
.sublists = {
BASE_SUBLIST,
EL2_SUBLIST,
VREGS_SUBLIST,
PAUTH_SUBLIST,
PMU_SUBLIST,
{0},
},
};
struct vcpu_reg_list *vcpu_configs[] = {
&vregs_config,
&vregs_pmu_config,
@ -768,5 +904,12 @@ struct vcpu_reg_list *vcpu_configs[] = {
&sve_pmu_config,
&pauth_config,
&pauth_pmu_config,
&el2_vregs_config,
&el2_vregs_pmu_config,
&el2_sve_config,
&el2_sve_pmu_config,
&el2_pauth_config,
&el2_pauth_pmu_config,
};
int vcpu_configs_n = ARRAY_SIZE(vcpu_configs);

159
tools/testing/selftests/kvm/arm64/mmio_abort.c
View File

@ -1,159 +0,0 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
* mmio_abort - Tests for userspace MMIO abort injection
*
* Copyright (c) 2024 Google LLC
*/
#include "processor.h"
#include "test_util.h"
#define MMIO_ADDR 0x8000000ULL
static u64 expected_abort_pc;
static void expect_sea_handler(struct ex_regs *regs)
{
u64 esr = read_sysreg(esr_el1);
GUEST_ASSERT_EQ(regs->pc, expected_abort_pc);
GUEST_ASSERT_EQ(ESR_ELx_EC(esr), ESR_ELx_EC_DABT_CUR);
GUEST_ASSERT_EQ(esr & ESR_ELx_FSC_TYPE, ESR_ELx_FSC_EXTABT);
GUEST_DONE();
}
static void unexpected_dabt_handler(struct ex_regs *regs)
{
GUEST_FAIL("Unexpected data abort at PC: %lx\n", regs->pc);
}
static struct kvm_vm *vm_create_with_dabt_handler(struct kvm_vcpu **vcpu, void *guest_code,
handler_fn dabt_handler)
{
struct kvm_vm *vm = vm_create_with_one_vcpu(vcpu, guest_code);
vm_init_descriptor_tables(vm);
vcpu_init_descriptor_tables(*vcpu);
vm_install_sync_handler(vm, VECTOR_SYNC_CURRENT, ESR_ELx_EC_DABT_CUR, dabt_handler);
virt_map(vm, MMIO_ADDR, MMIO_ADDR, 1);
return vm;
}
static void vcpu_inject_extabt(struct kvm_vcpu *vcpu)
{
struct kvm_vcpu_events events = {};
events.exception.ext_dabt_pending = true;
vcpu_events_set(vcpu, &events);
}
static void vcpu_run_expect_done(struct kvm_vcpu *vcpu)
{
struct ucall uc;
vcpu_run(vcpu);
switch (get_ucall(vcpu, &uc)) {
case UCALL_ABORT:
REPORT_GUEST_ASSERT(uc);
break;
case UCALL_DONE:
break;
default:
TEST_FAIL("Unexpected ucall: %lu", uc.cmd);
}
}
extern char test_mmio_abort_insn;
static void test_mmio_abort_guest(void)
{
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_abort_insn);
asm volatile("test_mmio_abort_insn:\n\t"
"ldr x0, [%0]\n\t"
: : "r" (MMIO_ADDR) : "x0", "memory");
GUEST_FAIL("MMIO instruction should not retire");
}
/*
* Test that KVM doesn't complete MMIO emulation when userspace has made an
* external abort pending for the instruction.
*/
static void test_mmio_abort(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_abort_guest,
expect_sea_handler);
struct kvm_run *run = vcpu->run;
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_MMIO);
TEST_ASSERT_EQ(run->mmio.phys_addr, MMIO_ADDR);
TEST_ASSERT_EQ(run->mmio.len, sizeof(unsigned long));
TEST_ASSERT(!run->mmio.is_write, "Expected MMIO read");
vcpu_inject_extabt(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
extern char test_mmio_nisv_insn;
static void test_mmio_nisv_guest(void)
{
WRITE_ONCE(expected_abort_pc, (u64)&test_mmio_nisv_insn);
asm volatile("test_mmio_nisv_insn:\n\t"
"ldr x0, [%0], #8\n\t"
: : "r" (MMIO_ADDR) : "x0", "memory");
GUEST_FAIL("MMIO instruction should not retire");
}
/*
* Test that the KVM_RUN ioctl fails for ESR_EL2.ISV=0 MMIO aborts if userspace
* hasn't enabled KVM_CAP_ARM_NISV_TO_USER.
*/
static void test_mmio_nisv(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
unexpected_dabt_handler);
TEST_ASSERT(_vcpu_run(vcpu), "Expected nonzero return code from KVM_RUN");
TEST_ASSERT_EQ(errno, ENOSYS);
kvm_vm_free(vm);
}
/*
* Test that ESR_EL2.ISV=0 MMIO aborts reach userspace and that an injected SEA
* reaches the guest.
*/
static void test_mmio_nisv_abort(void)
{
struct kvm_vcpu *vcpu;
struct kvm_vm *vm = vm_create_with_dabt_handler(&vcpu, test_mmio_nisv_guest,
expect_sea_handler);
struct kvm_run *run = vcpu->run;
vm_enable_cap(vm, KVM_CAP_ARM_NISV_TO_USER, 1);
vcpu_run(vcpu);
TEST_ASSERT_KVM_EXIT_REASON(vcpu, KVM_EXIT_ARM_NISV);
TEST_ASSERT_EQ(run->arm_nisv.fault_ipa, MMIO_ADDR);
vcpu_inject_extabt(vcpu);
vcpu_run_expect_done(vcpu);
kvm_vm_free(vm);
}
int main(void)
{
test_mmio_abort();
test_mmio_nisv();
test_mmio_nisv_abort();
}

14
tools/testing/selftests/kvm/arm64/set_id_regs.c
View File

@ -139,6 +139,7 @@ static const struct reg_ftr_bits ftr_id_aa64pfr0_el1[] = {
};
static const struct reg_ftr_bits ftr_id_aa64pfr1_el1[] = {
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, DF2, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, CSV2_frac, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, SSBS, ID_AA64PFR1_EL1_SSBS_NI),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64PFR1_EL1, BT, 0),
@ -187,6 +188,14 @@ static const struct reg_ftr_bits ftr_id_aa64mmfr2_el1[] = {
REG_FTR_END,
};
static const struct reg_ftr_bits ftr_id_aa64mmfr3_el1[] = {
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, S1POE, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, S1PIE, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, SCTLRX, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64MMFR3_EL1, TCRX, 0),
REG_FTR_END,
};
static const struct reg_ftr_bits ftr_id_aa64zfr0_el1[] = {
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ZFR0_EL1, F64MM, 0),
REG_FTR_BITS(FTR_LOWER_SAFE, ID_AA64ZFR0_EL1, F32MM, 0),
@ -217,6 +226,7 @@ static struct test_feature_reg test_regs[] = {
TEST_REG(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0_el1),
TEST_REG(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1_el1),
TEST_REG(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2_el1),
TEST_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3_el1),
TEST_REG(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0_el1),
};
@ -774,8 +784,8 @@ int main(void)
ARRAY_SIZE(ftr_id_aa64isar2_el1) + ARRAY_SIZE(ftr_id_aa64pfr0_el1) +
ARRAY_SIZE(ftr_id_aa64pfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr0_el1) +
ARRAY_SIZE(ftr_id_aa64mmfr1_el1) + ARRAY_SIZE(ftr_id_aa64mmfr2_el1) +
ARRAY_SIZE(ftr_id_aa64zfr0_el1) - ARRAY_SIZE(test_regs) + 3 +
MPAM_IDREG_TEST + MTE_IDREG_TEST;
ARRAY_SIZE(ftr_id_aa64mmfr3_el1) + ARRAY_SIZE(ftr_id_aa64zfr0_el1) -
ARRAY_SIZE(test_regs) + 3 + MPAM_IDREG_TEST + MTE_IDREG_TEST;
ksft_set_plan(test_cnt);

259
tools/testing/selftests/kvm/arm64/vgic_init.c
View File

@ -9,17 +9,18 @@
#include <asm/kvm.h>
#include <asm/kvm_para.h>
#include <arm64/gic_v3.h>
#include "test_util.h"
#include "kvm_util.h"
#include "processor.h"
#include "vgic.h"
#include "gic_v3.h"
#define NR_VCPUS 4
#define REG_OFFSET(vcpu, offset) (((uint64_t)vcpu << 32) | offset)
#define GICR_TYPER 0x8
#define VGIC_DEV_IS_V2(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V2)
#define VGIC_DEV_IS_V3(_d) ((_d) == KVM_DEV_TYPE_ARM_VGIC_V3)
@ -675,6 +676,44 @@ static void test_v3_its_region(void)
vm_gic_destroy(&v);
}
static void test_v3_nassgicap(void)
{
struct kvm_vcpu *vcpus[NR_VCPUS];
bool has_nassgicap;
struct vm_gic vm;
u32 typer2;
int ret;
vm = vm_gic_create_with_vcpus(KVM_DEV_TYPE_ARM_VGIC_V3, NR_VCPUS, vcpus);
kvm_device_attr_get(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
GICD_TYPER2, &typer2);
has_nassgicap = typer2 & GICD_TYPER2_nASSGIcap;
typer2 |= GICD_TYPER2_nASSGIcap;
ret = __kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
GICD_TYPER2, &typer2);
if (has_nassgicap)
TEST_ASSERT(!ret, KVM_IOCTL_ERROR(KVM_DEVICE_ATTR_SET, ret));
else
TEST_ASSERT(ret && errno == EINVAL,
"Enabled nASSGIcap even though it's unavailable");
typer2 &= ~GICD_TYPER2_nASSGIcap;
kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
GICD_TYPER2, &typer2);
kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_CTRL,
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
typer2 ^= GICD_TYPER2_nASSGIcap;
ret = __kvm_device_attr_set(vm.gic_fd, KVM_DEV_ARM_VGIC_GRP_DIST_REGS,
GICD_TYPER2, &typer2);
TEST_ASSERT(ret && errno == EBUSY,
"Changed nASSGIcap after initializing the VGIC");
vm_gic_destroy(&vm);
}
/*
* Returns 0 if it's possible to create GIC device of a given type (V2 or V3).
*/
@ -715,6 +754,220 @@ int test_kvm_device(uint32_t gic_dev_type)
return 0;
}
struct sr_def {
const char *name;
u32 encoding;
};
#define PACK_SR(r) \
((sys_reg_Op0(r) << 14) | \
(sys_reg_Op1(r) << 11) | \
(sys_reg_CRn(r) << 7) | \
(sys_reg_CRm(r) << 3) | \
(sys_reg_Op2(r)))
#define SR(r) \
{ \
.name = #r, \
.encoding = r, \
}
static const struct sr_def sysregs_el1[] = {
SR(SYS_ICC_PMR_EL1),
SR(SYS_ICC_BPR0_EL1),
SR(SYS_ICC_AP0R0_EL1),
SR(SYS_ICC_AP0R1_EL1),
SR(SYS_ICC_AP0R2_EL1),
SR(SYS_ICC_AP0R3_EL1),
SR(SYS_ICC_AP1R0_EL1),
SR(SYS_ICC_AP1R1_EL1),
SR(SYS_ICC_AP1R2_EL1),
SR(SYS_ICC_AP1R3_EL1),
SR(SYS_ICC_BPR1_EL1),
SR(SYS_ICC_CTLR_EL1),
SR(SYS_ICC_SRE_EL1),
SR(SYS_ICC_IGRPEN0_EL1),
SR(SYS_ICC_IGRPEN1_EL1),
};
static const struct sr_def sysregs_el2[] = {
SR(SYS_ICH_AP0R0_EL2),
SR(SYS_ICH_AP0R1_EL2),
SR(SYS_ICH_AP0R2_EL2),
SR(SYS_ICH_AP0R3_EL2),
SR(SYS_ICH_AP1R0_EL2),
SR(SYS_ICH_AP1R1_EL2),
SR(SYS_ICH_AP1R2_EL2),
SR(SYS_ICH_AP1R3_EL2),
SR(SYS_ICH_HCR_EL2),
SR(SYS_ICC_SRE_EL2),
SR(SYS_ICH_VTR_EL2),
SR(SYS_ICH_VMCR_EL2),
SR(SYS_ICH_LR0_EL2),
SR(SYS_ICH_LR1_EL2),
SR(SYS_ICH_LR2_EL2),
SR(SYS_ICH_LR3_EL2),
SR(SYS_ICH_LR4_EL2),
SR(SYS_ICH_LR5_EL2),
SR(SYS_ICH_LR6_EL2),
SR(SYS_ICH_LR7_EL2),
SR(SYS_ICH_LR8_EL2),
SR(SYS_ICH_LR9_EL2),
SR(SYS_ICH_LR10_EL2),
SR(SYS_ICH_LR11_EL2),
SR(SYS_ICH_LR12_EL2),
SR(SYS_ICH_LR13_EL2),
SR(SYS_ICH_LR14_EL2),
SR(SYS_ICH_LR15_EL2),
};
static void test_sysreg_array(int gic, const struct sr_def *sr, int nr,
int (*check)(int, const struct sr_def *, const char *))
{
for (int i = 0; i < nr; i++) {
u64 val;
u64 attr;
int ret;
/* Assume MPIDR_EL1.Aff*=0 */
attr = PACK_SR(sr[i].encoding);
/*
* The API is braindead. A register can be advertised as
* available, and yet not be readable or writable.
* ICC_APnR{1,2,3}_EL1 are examples of such non-sense, and
* ICH_APnR{1,2,3}_EL2 do follow suit for consistency.
*
* On the bright side, no known HW is implementing more than
* 5 bits of priority, so we're safe. Sort of...
*/
ret = __kvm_has_device_attr(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
attr);
TEST_ASSERT(ret == 0, "%s unavailable", sr[i].name);
/* Check that we can write back what we read */
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
attr, &val);
TEST_ASSERT(ret == 0 || !check(gic, &sr[i], "read"), "%s unreadable", sr[i].name);
ret = __kvm_device_attr_set(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
attr, &val);
TEST_ASSERT(ret == 0 || !check(gic, &sr[i], "write"), "%s unwritable", sr[i].name);
}
}
static u8 get_ctlr_pribits(int gic)
{
int ret;
u64 val;
u8 pri;
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
PACK_SR(SYS_ICC_CTLR_EL1), &val);
TEST_ASSERT(ret == 0, "ICC_CTLR_EL1 unreadable");
pri = FIELD_GET(ICC_CTLR_EL1_PRI_BITS_MASK, val) + 1;
TEST_ASSERT(pri >= 5 && pri <= 7, "Bad pribits %d", pri);
return pri;
}
static int check_unaccessible_el1_regs(int gic, const struct sr_def *sr, const char *what)
{
switch (sr->encoding) {
case SYS_ICC_AP0R1_EL1:
case SYS_ICC_AP1R1_EL1:
if (get_ctlr_pribits(gic) >= 6)
return -EINVAL;
break;
case SYS_ICC_AP0R2_EL1:
case SYS_ICC_AP0R3_EL1:
case SYS_ICC_AP1R2_EL1:
case SYS_ICC_AP1R3_EL1:
if (get_ctlr_pribits(gic) == 7)
return 0;
break;
default:
return -EINVAL;
}
pr_info("SKIP %s for %s\n", sr->name, what);
return 0;
}
static u8 get_vtr_pribits(int gic)
{
int ret;
u64 val;
u8 pri;
ret = __kvm_device_attr_get(gic, KVM_DEV_ARM_VGIC_GRP_CPU_SYSREGS,
PACK_SR(SYS_ICH_VTR_EL2), &val);
TEST_ASSERT(ret == 0, "ICH_VTR_EL2 unreadable");
pri = FIELD_GET(ICH_VTR_EL2_PRIbits, val) + 1;
TEST_ASSERT(pri >= 5 && pri <= 7, "Bad pribits %d", pri);
return pri;
}
static int check_unaccessible_el2_regs(int gic, const struct sr_def *sr, const char *what)
{
switch (sr->encoding) {
case SYS_ICH_AP0R1_EL2:
case SYS_ICH_AP1R1_EL2:
if (get_vtr_pribits(gic) >= 6)
return -EINVAL;
break;
case SYS_ICH_AP0R2_EL2:
case SYS_ICH_AP0R3_EL2:
case SYS_ICH_AP1R2_EL2:
case SYS_ICH_AP1R3_EL2:
if (get_vtr_pribits(gic) == 7)
return -EINVAL;
break;
default:
return -EINVAL;
}
pr_info("SKIP %s for %s\n", sr->name, what);
return 0;
}
static void test_v3_sysregs(void)
{
struct kvm_vcpu_init init = {};
struct kvm_vcpu *vcpu;
struct kvm_vm *vm;
u32 feat = 0;
int gic;
if (kvm_check_cap(KVM_CAP_ARM_EL2))
feat |= BIT(KVM_ARM_VCPU_HAS_EL2);
vm = vm_create(1);
vm_ioctl(vm, KVM_ARM_PREFERRED_TARGET, &init);
init.features[0] |= feat;
vcpu = aarch64_vcpu_add(vm, 0, &init, NULL);
TEST_ASSERT(vcpu, "Can't create a vcpu?");
gic = kvm_create_device(vm, KVM_DEV_TYPE_ARM_VGIC_V3);
TEST_ASSERT(gic >= 0, "No GIC???");
kvm_device_attr_set(gic, KVM_DEV_ARM_VGIC_GRP_CTRL,
KVM_DEV_ARM_VGIC_CTRL_INIT, NULL);
test_sysreg_array(gic, sysregs_el1, ARRAY_SIZE(sysregs_el1), check_unaccessible_el1_regs);
if (feat)
test_sysreg_array(gic, sysregs_el2, ARRAY_SIZE(sysregs_el2), check_unaccessible_el2_regs);
else
pr_info("SKIP EL2 registers, not available\n");
close(gic);
kvm_vm_free(vm);
}
void run_tests(uint32_t gic_dev_type)
{
test_vcpus_then_vgic(gic_dev_type);
@ -730,6 +983,8 @@ void run_tests(uint32_t gic_dev_type)
test_v3_last_bit_single_rdist();
test_v3_redist_ipa_range_check_at_vcpu_run();
test_v3_its_region();
test_v3_sysregs();
test_v3_nassgicap();
}
}

10
tools/testing/selftests/kvm/include/arm64/processor.h
View File

@ -254,6 +254,16 @@ static inline void local_irq_disable(void)
asm volatile("msr daifset, #3" : : : "memory");
}
static inline void local_serror_enable(void)
{
asm volatile("msr daifclr, #4" : : : "memory");
}
static inline void local_serror_disable(void)
{
asm volatile("msr daifset, #4" : : : "memory");
}
/**
* struct arm_smccc_res - Result from SMC/HVC call
* @a0-a3 result values from registers 0 to 3