From 4d84bb6c8b42cc781a02e1ac6648875966abc877 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 25 May 2022 08:59:04 -0400 Subject: [PATCH 01/90] hw/tpm/tpm_tis_common.c: Assert that locty is in range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In tpm_tis_mmio_read(), tpm_tis_mmio_write() and tpm_tis_dump_state(), we calculate a locality index with tpm_tis_locality_from_addr() and then use it as an index into the s->loc[] array. In all these cases, the array index can't overflow because the MemoryRegion is sized to be TPM_TIS_NUM_LOCALITIES << TPM_TIS_LOCALITY_SHIFT bytes. However, Coverity can't see that, and it complains (CID 1487138, 1487180, 1487188, 1487198, 1487240). Add an assertion to tpm_tis_locality_from_addr() that the calculated locality index is valid, which will help Coverity and also catch any potential future bug where the MemoryRegion isn't sized exactly. Signed-off-by: Peter Maydell Signed-off-by: Stefan Berger Reviewed-by: Marc-André Lureau Message-id: 20220525125904.483075-1-stefanb@linux.ibm.com --- hw/tpm/tpm_tis_common.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hw/tpm/tpm_tis_common.c b/hw/tpm/tpm_tis_common.c index e700d821816a..503be2a541d8 100644 --- a/hw/tpm/tpm_tis_common.c +++ b/hw/tpm/tpm_tis_common.c @@ -50,7 +50,12 @@ static uint64_t tpm_tis_mmio_read(void *opaque, hwaddr addr, static uint8_t tpm_tis_locality_from_addr(hwaddr addr) { - return (uint8_t)((addr >> TPM_TIS_LOCALITY_SHIFT) & 0x7); + uint8_t locty; + + locty = (uint8_t)((addr >> TPM_TIS_LOCALITY_SHIFT) & 0x7); + assert(TPM_TIS_IS_VALID_LOCTY(locty)); + + return locty; } From e37a0ef4605e5d2041785ff3fc89ca6021faf7a0 Mon Sep 17 00:00:00 2001 From: Anthony PERARD Date: Mon, 11 Apr 2022 15:47:49 +0100 Subject: [PATCH 02/90] tpm_crb: mark command buffer as dirty on request completion At the moment, there doesn't seems to be any way to know that QEMU made modification to the command buffer. This is potentially an issue on Xen while migrating a guest, as modification to the buffer after the migration as started could be ignored and not transfered to the destination. Mark the memory region of the command buffer as dirty once a request is completed. Signed-off-by: Anthony PERARD Reviewed-by: Stefan Berger Signed-off-by: Stefan Berger Message-id: 20220411144749.47185-1-anthony.perard@citrix.com --- hw/tpm/tpm_crb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/hw/tpm/tpm_crb.c b/hw/tpm/tpm_crb.c index aa9c00aad3a3..67db594c485b 100644 --- a/hw/tpm/tpm_crb.c +++ b/hw/tpm/tpm_crb.c @@ -197,6 +197,7 @@ static void tpm_crb_request_completed(TPMIf *ti, int ret) ARRAY_FIELD_DP32(s->regs, CRB_CTRL_STS, tpmSts, 1); /* fatal error */ } + memory_region_set_dirty(&s->cmdmem, 0, CRB_CTRL_CMD_SIZE); } static enum TPMVersion tpm_crb_get_version(TPMIf *ti) From ec6600be0dc16982181c7ad80d94c143c0807dd2 Mon Sep 17 00:00:00 2001 From: Eric Auger Date: Tue, 24 May 2022 05:14:05 -0400 Subject: [PATCH 03/90] vfio/common: remove spurious warning on vfio_listener_region_del 851d6d1a0f ("vfio/common: remove spurious tpm-crb-cmd misalignment warning") removed the warning on vfio_listener_region_add() path. However the same warning also hits on region_del path. Let's remove it and reword the dynamic trace as this can be called on both map and unmap path. Signed-off-by: Eric Auger Reviewed-by: Cornelia Huck Link: https://lore.kernel.org/r/20220524091405.416256-1-eric.auger@redhat.com Fixes: 851d6d1a0ff2 ("vfio/common: remove spurious tpm-crb-cmd misalignment warning") Signed-off-by: Alex Williamson --- hw/vfio/common.c | 10 +++++++++- hw/vfio/trace-events | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hw/vfio/common.c b/hw/vfio/common.c index 29982c7af8c4..ace9562a9ba1 100644 --- a/hw/vfio/common.c +++ b/hw/vfio/common.c @@ -1145,7 +1145,15 @@ static void vfio_listener_region_del(MemoryListener *listener, if (unlikely((section->offset_within_address_space & ~qemu_real_host_page_mask()) != (section->offset_within_region & ~qemu_real_host_page_mask()))) { - error_report("%s received unaligned region", __func__); + if (!vfio_known_safe_misalignment(section)) { + error_report("%s received unaligned region %s iova=0x%"PRIx64 + " offset_within_region=0x%"PRIx64 + " qemu_real_host_page_size=0x%"PRIxPTR, + __func__, memory_region_name(section->mr), + section->offset_within_address_space, + section->offset_within_region, + qemu_real_host_page_size()); + } return; } diff --git a/hw/vfio/trace-events b/hw/vfio/trace-events index 582882db91c3..73dffe9e00d5 100644 --- a/hw/vfio/trace-events +++ b/hw/vfio/trace-events @@ -100,7 +100,7 @@ vfio_listener_region_add_skip(uint64_t start, uint64_t end) "SKIPPING region_add vfio_spapr_group_attach(int groupfd, int tablefd) "Attached groupfd %d to liobn fd %d" vfio_listener_region_add_iommu(uint64_t start, uint64_t end) "region_add [iommu] 0x%"PRIx64" - 0x%"PRIx64 vfio_listener_region_add_ram(uint64_t iova_start, uint64_t iova_end, void *vaddr) "region_add [ram] 0x%"PRIx64" - 0x%"PRIx64" [%p]" -vfio_known_safe_misalignment(const char *name, uint64_t iova, uint64_t offset_within_region, uintptr_t page_size) "Region \"%s\" iova=0x%"PRIx64" offset_within_region=0x%"PRIx64" qemu_real_host_page_size=0x%"PRIxPTR ": cannot be mapped for DMA" +vfio_known_safe_misalignment(const char *name, uint64_t iova, uint64_t offset_within_region, uintptr_t page_size) "Region \"%s\" iova=0x%"PRIx64" offset_within_region=0x%"PRIx64" qemu_real_host_page_size=0x%"PRIxPTR vfio_listener_region_add_no_dma_map(const char *name, uint64_t iova, uint64_t size, uint64_t page_size) "Region \"%s\" 0x%"PRIx64" size=0x%"PRIx64" is not aligned to 0x%"PRIx64" and cannot be mapped for DMA" vfio_listener_region_del_skip(uint64_t start, uint64_t end) "SKIPPING region_del 0x%"PRIx64" - 0x%"PRIx64 vfio_listener_region_del(uint64_t start, uint64_t end) "region_del 0x%"PRIx64" - 0x%"PRIx64 From d507bc3b0541581fc9960f157968e80c02247255 Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:46 +0100 Subject: [PATCH 04/90] target/arm: Declare support for FEAT_RASv1p1 The architectural feature RASv1p1 introduces the following new features: * new registers ERXPFGCDN_EL1, ERXPFGCTL_EL1 and ERXPFGF_EL1 * new bits in the fine-grained trap registers that control traps for these new registers * new trap bits HCR_EL2.FIEN and SCR_EL3.FIEN that control traps for ERXPFGCDN_EL1, ERXPFGCTL_EL1, ERXPFGP_EL1 * a larger number of the ERXMISC_EL1 registers * the format of ERRSTATUS registers changes The architecture permits that if ERRIDR_EL1.NUM is 0 (as it is for QEMU) then all these new registers may UNDEF, and the HCR_EL2.FIEN and SCR_EL3.FIEN bits may be RES0. We don't have any ERRSTATUS registers (again, because ERRIDR_EL1.NUM is 0). QEMU does not yet implement the fine-grained-trap extension. So there is nothing we need to implement to be compliant with the feature spec. Make the 'max' CPU report the feature in its ID registers, and document it. Signed-off-by: Peter Maydell Reviewed-by: Richard Henderson Message-id: 20220531114258.855804-1-peter.maydell@linaro.org --- docs/system/arm/emulation.rst | 1 + target/arm/cpu64.c | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index 49cc3e8340e5..81467f02ce96 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -52,6 +52,7 @@ the following architecture extensions: - FEAT_PMUv3p1 (PMU Extensions v3.1) - FEAT_PMUv3p4 (PMU Extensions v3.4) - FEAT_RAS (Reliability, availability, and serviceability) +- FEAT_RASv1p1 (RAS Extension v1.1) - FEAT_RDM (Advanced SIMD rounding double multiply accumulate instructions) - FEAT_RNG (Random number generator) - FEAT_S2FWB (Stage 2 forced Write-Back) diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index 3ff9219ca3b7..bd1c62a34282 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -916,6 +916,7 @@ static void aarch64_max_initfn(Object *obj) * we do for EL2 with the virtualization=on property. */ t = FIELD_DP64(t, ID_AA64PFR1, MTE, 3); /* FEAT_MTE3 */ + t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 1); /* FEAT_RASv1p1 */ t = FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0); /* FEAT_CSV2_2 */ cpu->isar.id_aa64pfr1 = t; From 7ac610206a0049460c392c9559b51246af0afd6d Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:46 +0100 Subject: [PATCH 05/90] target/arm: Implement FEAT_DoubleFault The FEAT_DoubleFault extension adds the following: * All external aborts on instruction fetches and translation table walks for instruction fetches must be synchronous. For QEMU this is already true. * SCR_EL3 has a new bit NMEA which disables the masking of SError interrupts by PSTATE.A when the SError interrupt is taken to EL3. For QEMU we only need to make the bit writable, because we have no sources of SError interrupts. * SCR_EL3 has a new bit EASE which causes synchronous external aborts taken to EL3 to be taken at the same entry point as SError. (Note that this does not mean that they are SErrors for purposes of PSTATE.A masking or that the syndrome register reports them as SErrors: it just means that the vector offset is different.) * The existing SCTLR_EL3.IESB has an effective value of 1 when SCR_EL3.NMEA is 1. For QEMU this is a no-op because we don't need different behaviour based on IESB (we don't need to do anything to ensure that error exceptions are synchronized). So for QEMU the things we need to change are: * Make SCR_EL3.{NMEA,EASE} writable * When taking a synchronous external abort at EL3, adjust the vector entry point if SCR_EL3.EASE is set * Advertise the feature in the ID registers Signed-off-by: Peter Maydell Reviewed-by: Richard Henderson Message-id: 20220531151431.949322-1-peter.maydell@linaro.org --- docs/system/arm/emulation.rst | 1 + target/arm/cpu.h | 5 +++++ target/arm/cpu64.c | 4 ++-- target/arm/helper.c | 36 +++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 2 deletions(-) diff --git a/docs/system/arm/emulation.rst b/docs/system/arm/emulation.rst index 81467f02ce96..83b44100659a 100644 --- a/docs/system/arm/emulation.rst +++ b/docs/system/arm/emulation.rst @@ -23,6 +23,7 @@ the following architecture extensions: - FEAT_Debugv8p2 (Debug changes for v8.2) - FEAT_Debugv8p4 (Debug changes for v8.4) - FEAT_DotProd (Advanced SIMD dot product instructions) +- FEAT_DoubleFault (Double Fault Extension) - FEAT_FCMA (Floating-point complex number instructions) - FEAT_FHM (Floating-point half-precision multiplication instructions) - FEAT_FP16 (Half-precision floating-point data processing) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index c1865ad5dade..0ee1705a4fae 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3952,6 +3952,11 @@ static inline bool isar_feature_aa64_ras(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, RAS) != 0; } +static inline bool isar_feature_aa64_doublefault(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, RAS) >= 2; +} + static inline bool isar_feature_aa64_sve(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64pfr0, ID_AA64PFR0, SVE) != 0; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index bd1c62a34282..cce68dd82a23 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -899,7 +899,7 @@ static void aarch64_max_initfn(Object *obj) t = cpu->isar.id_aa64pfr0; t = FIELD_DP64(t, ID_AA64PFR0, FP, 1); /* FEAT_FP16 */ t = FIELD_DP64(t, ID_AA64PFR0, ADVSIMD, 1); /* FEAT_FP16 */ - t = FIELD_DP64(t, ID_AA64PFR0, RAS, 1); /* FEAT_RAS */ + t = FIELD_DP64(t, ID_AA64PFR0, RAS, 2); /* FEAT_RASv1p1 + FEAT_DoubleFault */ t = FIELD_DP64(t, ID_AA64PFR0, SVE, 1); t = FIELD_DP64(t, ID_AA64PFR0, SEL2, 1); /* FEAT_SEL2 */ t = FIELD_DP64(t, ID_AA64PFR0, DIT, 1); /* FEAT_DIT */ @@ -916,7 +916,7 @@ static void aarch64_max_initfn(Object *obj) * we do for EL2 with the virtualization=on property. */ t = FIELD_DP64(t, ID_AA64PFR1, MTE, 3); /* FEAT_MTE3 */ - t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 1); /* FEAT_RASv1p1 */ + t = FIELD_DP64(t, ID_AA64PFR1, RAS_FRAC, 0); /* FEAT_RASv1p1 + FEAT_DoubleFault */ t = FIELD_DP64(t, ID_AA64PFR1, CSV2_FRAC, 0); /* FEAT_CSV2_2 */ cpu->isar.id_aa64pfr1 = t; diff --git a/target/arm/helper.c b/target/arm/helper.c index 40da63913c93..7f2c14bea94d 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -1776,6 +1776,9 @@ static void scr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) if (cpu_isar_feature(aa64_scxtnum, cpu)) { valid_mask |= SCR_ENSCXT; } + if (cpu_isar_feature(aa64_doublefault, cpu)) { + valid_mask |= SCR_EASE | SCR_NMEA; + } } else { valid_mask &= ~(SCR_RW | SCR_ST); if (cpu_isar_feature(aa32_ras, cpu)) { @@ -10113,6 +10116,31 @@ static uint32_t cpsr_read_for_spsr_elx(CPUARMState *env) return ret; } +static bool syndrome_is_sync_extabt(uint32_t syndrome) +{ + /* Return true if this syndrome value is a synchronous external abort */ + switch (syn_get_ec(syndrome)) { + case EC_INSNABORT: + case EC_INSNABORT_SAME_EL: + case EC_DATAABORT: + case EC_DATAABORT_SAME_EL: + /* Look at fault status code for all the synchronous ext abort cases */ + switch (syndrome & 0x3f) { + case 0x10: + case 0x13: + case 0x14: + case 0x15: + case 0x16: + case 0x17: + return true; + default: + return false; + } + default: + return false; + } +} + /* Handle exception entry to a target EL which is using AArch64 */ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) { @@ -10168,6 +10196,14 @@ static void arm_cpu_do_interrupt_aarch64(CPUState *cs) switch (cs->exception_index) { case EXCP_PREFETCH_ABORT: case EXCP_DATA_ABORT: + /* + * FEAT_DoubleFault allows synchronous external aborts taken to EL3 + * to be taken to the SError vector entrypoint. + */ + if (new_el == 3 && (env->cp15.scr_el3 & SCR_EASE) && + syndrome_is_sync_extabt(env->exception.syndrome)) { + addr += 0x180; + } env->cp15.far_el[new_el] = env->exception.vaddress; qemu_log_mask(CPU_LOG_INT, "...with FAR 0x%" PRIx64 "\n", env->cp15.far_el[new_el]); From 9323e79f10e5f5d8fffc3b307776173ca11faeae Mon Sep 17 00:00:00 2001 From: Peter Maydell Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 06/90] Fix 'writeable' typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have about 30 instances of the typo/variant spelling 'writeable', and over 500 of the more common 'writable'. Standardize on the latter. Change produced with: sed -i -e 's/\([Ww][Rr][Ii][Tt]\)[Ee]\([Aa][Bb][Ll][Ee]\)/\1\2/g' $(git grep -il writeable) and then hand-undoing the instance in linux-headers/linux/kvm.h. Most of these changes are in comments or documentation; the exceptions are: * a local variable in accel/hvf/hvf-accel-ops.c * a local variable in accel/kvm/kvm-all.c * the PMCR_WRITABLE_MASK macro in target/arm/internals.h * the EPT_VIOLATION_GPA_WRITABLE macro in target/i386/hvf/vmcs.h (which is never used anywhere) * the AR_TYPE_WRITABLE_MASK macro in target/i386/hvf/vmx.h (which is never used anywhere) Signed-off-by: Peter Maydell Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Stefan Weil Message-id: 20220505095015.2714666-1-peter.maydell@linaro.org --- accel/hvf/hvf-accel-ops.c | 4 ++-- accel/kvm/kvm-all.c | 4 ++-- accel/tcg/user-exec.c | 6 +++--- docs/interop/vhost-user.rst | 2 +- docs/specs/vmgenid.txt | 4 ++-- hw/acpi/ghes.c | 2 +- hw/intc/arm_gicv3_cpuif.c | 2 +- hw/intc/arm_gicv3_dist.c | 2 +- hw/intc/arm_gicv3_redist.c | 4 ++-- hw/intc/riscv_aclint.c | 2 +- hw/intc/riscv_aplic.c | 2 +- hw/pci/shpc.c | 2 +- hw/scsi/mfi.h | 2 +- hw/sparc64/sun4u_iommu.c | 2 +- hw/timer/sse-timer.c | 2 +- python/qemu/machine/machine.py | 2 +- target/arm/gdbstub.c | 2 +- target/arm/helper.c | 4 ++-- target/arm/hvf/hvf.c | 4 ++-- target/arm/internals.h | 4 ++-- target/i386/cpu-sysemu.c | 2 +- target/i386/hvf/vmcs.h | 2 +- target/i386/hvf/vmx.h | 2 +- target/s390x/ioinst.c | 2 +- tests/tcg/x86_64/system/boot.S | 2 +- 25 files changed, 34 insertions(+), 34 deletions(-) diff --git a/accel/hvf/hvf-accel-ops.c b/accel/hvf/hvf-accel-ops.c index a70e2eb375dc..24913ca9c49f 100644 --- a/accel/hvf/hvf-accel-ops.c +++ b/accel/hvf/hvf-accel-ops.c @@ -120,12 +120,12 @@ static void hvf_set_phys_mem(MemoryRegionSection *section, bool add) { hvf_slot *mem; MemoryRegion *area = section->mr; - bool writeable = !area->readonly && !area->rom_device; + bool writable = !area->readonly && !area->rom_device; hv_memory_flags_t flags; uint64_t page_size = qemu_real_host_page_size(); if (!memory_region_is_ram(area)) { - if (writeable) { + if (writable) { return; } else if (!memory_region_is_romd(area)) { /* diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 32e177bd26b4..a4c4863f5329 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -1346,13 +1346,13 @@ static void kvm_set_phys_mem(KVMMemoryListener *kml, KVMSlot *mem; int err; MemoryRegion *mr = section->mr; - bool writeable = !mr->readonly && !mr->rom_device; + bool writable = !mr->readonly && !mr->rom_device; hwaddr start_addr, size, slot_size, mr_offset; ram_addr_t ram_start_offset; void *ram; if (!memory_region_is_ram(mr)) { - if (writeable || !kvm_readonly_mem_allowed) { + if (writable || !kvm_readonly_mem_allowed) { return; } else if (!mr->romd_mode) { /* If the memory device is not in romd_mode, then we actually want diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index ac57324d4f63..20ada5472b4c 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -101,10 +101,10 @@ MMUAccessType adjust_signal_pc(uintptr_t *pc, bool is_write) * Return true if the write fault has been handled, and should be re-tried. * * Note that it is important that we don't call page_unprotect() unless - * this is really a "write to nonwriteable page" fault, because + * this is really a "write to nonwritable page" fault, because * page_unprotect() assumes that if it is called for an access to - * a page that's writeable this means we had two threads racing and - * another thread got there first and already made the page writeable; + * a page that's writable this means we had two threads racing and + * another thread got there first and already made the page writable; * so we will retry the access. If we were to call page_unprotect() * for some other kind of fault that should really be passed to the * guest, we'd end up in an infinite loop of retrying the faulting access. diff --git a/docs/interop/vhost-user.rst b/docs/interop/vhost-user.rst index a99ba4433ce8..d7cf904f7fe0 100644 --- a/docs/interop/vhost-user.rst +++ b/docs/interop/vhost-user.rst @@ -222,7 +222,7 @@ Virtio device config space :size: a 32-bit configuration space access size in bytes :flags: a 32-bit value: - - 0: Vhost front-end messages used for writeable fields + - 0: Vhost front-end messages used for writable fields - 1: Vhost front-end messages used for live migration :payload: Size bytes array holding the contents of the virtio diff --git a/docs/specs/vmgenid.txt b/docs/specs/vmgenid.txt index aa9f5186767c..80ff69f31cc2 100644 --- a/docs/specs/vmgenid.txt +++ b/docs/specs/vmgenid.txt @@ -153,7 +153,7 @@ change the contents of the memory at runtime, specifically when starting a backed-up or snapshotted image. In order to do this, QEMU must know the address that has been allocated. -The mechanism chosen for this memory sharing is writeable fw_cfg blobs. +The mechanism chosen for this memory sharing is writable fw_cfg blobs. These are data object that are visible to both QEMU and guests, and are addressable as sequential files. @@ -164,7 +164,7 @@ Two fw_cfg blobs are used in this case: /etc/vmgenid_guid - contains the actual VM Generation ID GUID - read-only to the guest /etc/vmgenid_addr - contains the address of the downloaded vmgenid blob - - writeable by the guest + - writable by the guest QEMU sends the following commands to the guest at startup: diff --git a/hw/acpi/ghes.c b/hw/acpi/ghes.c index 45d9a809cc98..e9511d9b8f71 100644 --- a/hw/acpi/ghes.c +++ b/hw/acpi/ghes.c @@ -249,7 +249,7 @@ void build_ghes_error_table(GArray *hardware_errors, BIOSLinker *linker) for (i = 0; i < ACPI_GHES_ERROR_SOURCE_COUNT; i++) { /* * Initialize the value of read_ack_register to 1, so GHES can be - * writeable after (re)boot. + * writable after (re)boot. * ACPI 6.2: 18.3.2.8 Generic Hardware Error Source version 2 * (GHESv2 - Type 10) */ diff --git a/hw/intc/arm_gicv3_cpuif.c b/hw/intc/arm_gicv3_cpuif.c index 8867e2e496fd..8ca630e5ad1e 100644 --- a/hw/intc/arm_gicv3_cpuif.c +++ b/hw/intc/arm_gicv3_cpuif.c @@ -2047,7 +2047,7 @@ static void icc_ctlr_el3_write(CPUARMState *env, const ARMCPRegInfo *ri, cs->icc_ctlr_el1[GICV3_S] |= ICC_CTLR_EL1_CBPR; } - /* The only bit stored in icc_ctlr_el3 which is writeable is EOIMODE_EL3: */ + /* The only bit stored in icc_ctlr_el3 which is writable is EOIMODE_EL3: */ mask = ICC_CTLR_EL3_EOIMODE_EL3; cs->icc_ctlr_el3 &= ~mask; diff --git a/hw/intc/arm_gicv3_dist.c b/hw/intc/arm_gicv3_dist.c index b9ed955e36bb..eea03681187b 100644 --- a/hw/intc/arm_gicv3_dist.c +++ b/hw/intc/arm_gicv3_dist.c @@ -611,7 +611,7 @@ static bool gicd_writel(GICv3State *s, hwaddr offset, if (value & mask & GICD_CTLR_DS) { /* We just set DS, so the ARE_NS and EnG1S bits are now RES0. * Note that this is a one-way transition because if DS is set - * then it's not writeable, so it can only go back to 0 with a + * then it's not writable, so it can only go back to 0 with a * hardware reset. */ s->gicd_ctlr &= ~(GICD_CTLR_EN_GRP1S | GICD_CTLR_ARE_NS); diff --git a/hw/intc/arm_gicv3_redist.c b/hw/intc/arm_gicv3_redist.c index c3d4cdd66b74..f1ecb2502b1f 100644 --- a/hw/intc/arm_gicv3_redist.c +++ b/hw/intc/arm_gicv3_redist.c @@ -257,7 +257,7 @@ static void gicr_write_vpendbaser(GICv3CPUState *cs, uint64_t newval) /* * The DIRTY bit is read-only and for us is always zero; - * other fields are writeable. + * other fields are writable. */ newval &= R_GICR_VPENDBASER_INNERCACHE_MASK | R_GICR_VPENDBASER_SHAREABILITY_MASK | @@ -491,7 +491,7 @@ static MemTxResult gicr_writel(GICv3CPUState *cs, hwaddr offset, /* RAZ/WI for our implementation */ return MEMTX_OK; case GICR_WAKER: - /* Only the ProcessorSleep bit is writeable. When the guest sets + /* Only the ProcessorSleep bit is writable. When the guest sets * it it requests that we transition the channel between the * redistributor and the cpu interface to quiescent, and that * we set the ChildrenAsleep bit once the inteface has reached the diff --git a/hw/intc/riscv_aclint.c b/hw/intc/riscv_aclint.c index e6bceceefdbc..e7942c4e5a32 100644 --- a/hw/intc/riscv_aclint.c +++ b/hw/intc/riscv_aclint.c @@ -463,7 +463,7 @@ static void riscv_aclint_swi_realize(DeviceState *dev, Error **errp) /* Claim software interrupt bits */ for (i = 0; i < swi->num_harts; i++) { RISCVCPU *cpu = RISCV_CPU(qemu_get_cpu(swi->hartid_base + i)); - /* We don't claim mip.SSIP because it is writeable by software */ + /* We don't claim mip.SSIP because it is writable by software */ if (riscv_cpu_claim_interrupts(cpu, swi->sswi ? 0 : MIP_MSIP) < 0) { error_report("MSIP already claimed"); exit(1); diff --git a/hw/intc/riscv_aplic.c b/hw/intc/riscv_aplic.c index e7809fb6b2c7..cfd007e629ca 100644 --- a/hw/intc/riscv_aplic.c +++ b/hw/intc/riscv_aplic.c @@ -646,7 +646,7 @@ static void riscv_aplic_write(void *opaque, hwaddr addr, uint64_t value, } if (addr == APLIC_DOMAINCFG) { - /* Only IE bit writeable at the moment */ + /* Only IE bit writable at the moment */ value &= APLIC_DOMAINCFG_IE; aplic->domaincfg = value; } else if ((APLIC_SOURCECFG_BASE <= addr) && diff --git a/hw/pci/shpc.c b/hw/pci/shpc.c index 28e62174c425..f822f18b9806 100644 --- a/hw/pci/shpc.c +++ b/hw/pci/shpc.c @@ -456,7 +456,7 @@ static int shpc_cap_add_config(PCIDevice *d, Error **errp) pci_set_byte(config + SHPC_CAP_CxP, 0); pci_set_long(config + SHPC_CAP_DWORD_DATA, 0); d->shpc->cap = config_offset; - /* Make dword select and data writeable. */ + /* Make dword select and data writable. */ pci_set_byte(d->wmask + config_offset + SHPC_CAP_DWORD_SELECT, 0xff); pci_set_long(d->wmask + config_offset + SHPC_CAP_DWORD_DATA, 0xffffffff); return 0; diff --git a/hw/scsi/mfi.h b/hw/scsi/mfi.h index e67a5c0b477d..0b4ee53dfc0b 100644 --- a/hw/scsi/mfi.h +++ b/hw/scsi/mfi.h @@ -633,7 +633,7 @@ struct mfi_ctrl_props { * metadata and user data * 1=5%, 2=10%, 3=15% and so on */ - uint8_t viewSpace; /* snapshot writeable VIEWs + uint8_t viewSpace; /* snapshot writable VIEWs * capacity as a % of source LD * capacity. 0=READ only * 1=5%, 2=10%, 3=15% and so on diff --git a/hw/sparc64/sun4u_iommu.c b/hw/sparc64/sun4u_iommu.c index 9178277f8246..1c1dca712e32 100644 --- a/hw/sparc64/sun4u_iommu.c +++ b/hw/sparc64/sun4u_iommu.c @@ -165,7 +165,7 @@ static IOMMUTLBEntry sun4u_translate_iommu(IOMMUMemoryRegion *iommu, } if (tte & IOMMU_TTE_DATA_W) { - /* Writeable */ + /* Writable */ ret.perm = IOMMU_RW; } else { ret.perm = IOMMU_RO; diff --git a/hw/timer/sse-timer.c b/hw/timer/sse-timer.c index f959cb9d6033..e92e83747d2a 100644 --- a/hw/timer/sse-timer.c +++ b/hw/timer/sse-timer.c @@ -324,7 +324,7 @@ static void sse_timer_write(void *opaque, hwaddr offset, uint64_t value, { uint32_t old_ctl = s->cntp_aival_ctl; - /* EN bit is writeable; CLR bit is write-0-to-clear, write-1-ignored */ + /* EN bit is writable; CLR bit is write-0-to-clear, write-1-ignored */ s->cntp_aival_ctl &= ~R_CNTP_AIVAL_CTL_EN_MASK; s->cntp_aival_ctl |= value & R_CNTP_AIVAL_CTL_EN_MASK; if (!(value & R_CNTP_AIVAL_CTL_CLR_MASK)) { diff --git a/python/qemu/machine/machine.py b/python/qemu/machine/machine.py index 07ac5a710bea..37191f433b2d 100644 --- a/python/qemu/machine/machine.py +++ b/python/qemu/machine/machine.py @@ -495,7 +495,7 @@ def _early_cleanup(self) -> None: """ # If we keep the console socket open, we may deadlock waiting # for QEMU to exit, while QEMU is waiting for the socket to - # become writeable. + # become writable. if self._console_socket is not None: self._console_socket.close() self._console_socket = None diff --git a/target/arm/gdbstub.c b/target/arm/gdbstub.c index f5b35cd55f00..2f806512d0ad 100644 --- a/target/arm/gdbstub.c +++ b/target/arm/gdbstub.c @@ -118,7 +118,7 @@ int arm_cpu_gdb_write_register(CPUState *cs, uint8_t *mem_buf, int n) /* * Don't allow writing to XPSR.Exception as it can cause * a transition into or out of handler mode (it's not - * writeable via the MSR insn so this is a reasonable + * writable via the MSR insn so this is a reasonable * restriction). Other fields are safe to update. */ xpsr_write(env, tmp, ~XPSR_EXCP); diff --git a/target/arm/helper.c b/target/arm/helper.c index 7f2c14bea94d..5727ead5e4ce 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -1411,8 +1411,8 @@ static void pmcr_write(CPUARMState *env, const ARMCPRegInfo *ri, } } - env->cp15.c9_pmcr &= ~PMCR_WRITEABLE_MASK; - env->cp15.c9_pmcr |= (value & PMCR_WRITEABLE_MASK); + env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK; + env->cp15.c9_pmcr |= (value & PMCR_WRITABLE_MASK); pmu_op_finish(env); } diff --git a/target/arm/hvf/hvf.c b/target/arm/hvf/hvf.c index 1fdc5eef92bc..060aa0ccf4b0 100644 --- a/target/arm/hvf/hvf.c +++ b/target/arm/hvf/hvf.c @@ -978,8 +978,8 @@ static int hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val) } } - env->cp15.c9_pmcr &= ~PMCR_WRITEABLE_MASK; - env->cp15.c9_pmcr |= (val & PMCR_WRITEABLE_MASK); + env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK; + env->cp15.c9_pmcr |= (val & PMCR_WRITABLE_MASK); pmu_op_finish(env); break; diff --git a/target/arm/internals.h b/target/arm/internals.h index b654bee4682b..1e4887b2dd33 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1280,10 +1280,10 @@ enum MVEECIState { #define PMCRP 0x2 #define PMCRE 0x1 /* - * Mask of PMCR bits writeable by guest (not including WO bits like C, P, + * Mask of PMCR bits writable by guest (not including WO bits like C, P, * which can be written as 1 to trigger behaviour but which stay RAZ). */ -#define PMCR_WRITEABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE) +#define PMCR_WRITABLE_MASK (PMCRLC | PMCRDP | PMCRX | PMCRD | PMCRE) #define PMXEVTYPER_P 0x80000000 #define PMXEVTYPER_U 0x40000000 diff --git a/target/i386/cpu-sysemu.c b/target/i386/cpu-sysemu.c index e254d8ba10fb..a6f47b7d114d 100644 --- a/target/i386/cpu-sysemu.c +++ b/target/i386/cpu-sysemu.c @@ -103,7 +103,7 @@ static void x86_cpu_to_dict(X86CPU *cpu, QDict *props) /* Convert CPU model data from X86CPU object to a property dictionary * that can recreate exactly the same CPU model, including every - * writeable QOM property. + * writable QOM property. */ static void x86_cpu_to_dict_full(X86CPU *cpu, QDict *props) { diff --git a/target/i386/hvf/vmcs.h b/target/i386/hvf/vmcs.h index 42de7ebc3af2..b4692f63f653 100644 --- a/target/i386/hvf/vmcs.h +++ b/target/i386/hvf/vmcs.h @@ -330,7 +330,7 @@ #define EPT_VIOLATION_DATA_WRITE (1UL << 1) #define EPT_VIOLATION_INST_FETCH (1UL << 2) #define EPT_VIOLATION_GPA_READABLE (1UL << 3) -#define EPT_VIOLATION_GPA_WRITEABLE (1UL << 4) +#define EPT_VIOLATION_GPA_WRITABLE (1UL << 4) #define EPT_VIOLATION_GPA_EXECUTABLE (1UL << 5) #define EPT_VIOLATION_GLA_VALID (1UL << 7) #define EPT_VIOLATION_XLAT_VALID (1UL << 8) diff --git a/target/i386/hvf/vmx.h b/target/i386/hvf/vmx.h index 573ddc33c07b..fcd9a95e5b5c 100644 --- a/target/i386/hvf/vmx.h +++ b/target/i386/hvf/vmx.h @@ -80,7 +80,7 @@ static inline uint64_t cap2ctrl(uint64_t cap, uint64_t ctrl) #define AR_TYPE_ACCESSES_MASK 1 #define AR_TYPE_READABLE_MASK (1 << 1) -#define AR_TYPE_WRITEABLE_MASK (1 << 2) +#define AR_TYPE_WRITABLE_MASK (1 << 2) #define AR_TYPE_CODE_MASK (1 << 3) #define AR_TYPE_MASK 0x0f #define AR_TYPE_BUSY_64_TSS 11 diff --git a/target/s390x/ioinst.c b/target/s390x/ioinst.c index bdae5090bc8c..b12f18d346f7 100644 --- a/target/s390x/ioinst.c +++ b/target/s390x/ioinst.c @@ -284,7 +284,7 @@ void ioinst_handle_stsch(S390CPU *cpu, uint64_t reg1, uint32_t ipb, g_assert(!s390_is_pv()); /* * As operand exceptions have a lower priority than access exceptions, - * we check whether the memory area is writeable (injecting the + * we check whether the memory area is writable (injecting the * access execption if it is not) first. */ if (!s390_cpu_virt_mem_check_write(cpu, addr, ar, sizeof(schib))) { diff --git a/tests/tcg/x86_64/system/boot.S b/tests/tcg/x86_64/system/boot.S index f8a2fcc8395e..ed0f638406b1 100644 --- a/tests/tcg/x86_64/system/boot.S +++ b/tests/tcg/x86_64/system/boot.S @@ -56,7 +56,7 @@ * * - `ebx`: contains the physical memory address where the loader has placed * the boot start info structure. - * - `cr0`: bit 0 (PE) must be set. All the other writeable bits are cleared. + * - `cr0`: bit 0 (PE) must be set. All the other writable bits are cleared. * - `cr4`: all bits are cleared. * - `cs `: must be a 32-bit read/execute code segment with a base of ‘0’ * and a limit of ‘0xFFFFFFFF’. The selector value is unspecified. From d2008b3355061ab0cca823739cd98045e769a809 Mon Sep 17 00:00:00 2001 From: Frederic Konrad Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 07/90] xlnx_dp: fix the wrong register size The core and the vblend registers size are wrong, they should respectively be 0x3B0 and 0x1E0 according to: https://www.xilinx.com/htmldocs/registers/ug1087/ug1087-zynq-ultrascale-registers.html. Let's fix that and use macros when creating the mmio region. Fixes: 58ac482a66d ("introduce xlnx-dp") Signed-off-by: Frederic Konrad Reviewed-by: Edgar E. Iglesias Acked-by: Alistair Francis Message-id: 20220601172353.3220232-2-fkonrad@xilinx.com Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 17 ++++++++++------- include/hw/display/xlnx_dp.h | 9 +++++++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index 9bb781e31254..0378570459da 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -1219,19 +1219,22 @@ static void xlnx_dp_init(Object *obj) SysBusDevice *sbd = SYS_BUS_DEVICE(obj); XlnxDPState *s = XLNX_DP(obj); - memory_region_init(&s->container, obj, TYPE_XLNX_DP, 0xC050); + memory_region_init(&s->container, obj, TYPE_XLNX_DP, DP_CONTAINER_SIZE); memory_region_init_io(&s->core_iomem, obj, &dp_ops, s, TYPE_XLNX_DP - ".core", 0x3AF); - memory_region_add_subregion(&s->container, 0x0000, &s->core_iomem); + ".core", sizeof(s->core_registers)); + memory_region_add_subregion(&s->container, DP_CORE_REG_OFFSET, + &s->core_iomem); memory_region_init_io(&s->vblend_iomem, obj, &vblend_ops, s, TYPE_XLNX_DP - ".v_blend", 0x1DF); - memory_region_add_subregion(&s->container, 0xA000, &s->vblend_iomem); + ".v_blend", sizeof(s->vblend_registers)); + memory_region_add_subregion(&s->container, DP_VBLEND_REG_OFFSET, + &s->vblend_iomem); memory_region_init_io(&s->avbufm_iomem, obj, &avbufm_ops, s, TYPE_XLNX_DP - ".av_buffer_manager", 0x238); - memory_region_add_subregion(&s->container, 0xB000, &s->avbufm_iomem); + ".av_buffer_manager", sizeof(s->avbufm_registers)); + memory_region_add_subregion(&s->container, DP_AVBUF_REG_OFFSET, + &s->avbufm_iomem); memory_region_init_io(&s->audio_iomem, obj, &audio_ops, s, TYPE_XLNX_DP ".audio", sizeof(s->audio_registers)); diff --git a/include/hw/display/xlnx_dp.h b/include/hw/display/xlnx_dp.h index 8ab4733bb854..1ef5a89ee744 100644 --- a/include/hw/display/xlnx_dp.h +++ b/include/hw/display/xlnx_dp.h @@ -39,10 +39,15 @@ #define AUD_CHBUF_MAX_DEPTH (32 * KiB) #define MAX_QEMU_BUFFER_SIZE (4 * KiB) -#define DP_CORE_REG_ARRAY_SIZE (0x3AF >> 2) +#define DP_CORE_REG_OFFSET (0x0000) +#define DP_CORE_REG_ARRAY_SIZE (0x3B0 >> 2) +#define DP_AVBUF_REG_OFFSET (0xB000) #define DP_AVBUF_REG_ARRAY_SIZE (0x238 >> 2) -#define DP_VBLEND_REG_ARRAY_SIZE (0x1DF >> 2) +#define DP_VBLEND_REG_OFFSET (0xA000) +#define DP_VBLEND_REG_ARRAY_SIZE (0x1E0 >> 2) +#define DP_AUDIO_REG_OFFSET (0xC000) #define DP_AUDIO_REG_ARRAY_SIZE (0x50 >> 2) +#define DP_CONTAINER_SIZE (0xC050) struct PixmanPlane { pixman_format_code_t format; From 759ae1b47e7e0203eb6881304b4bbf0d935fd550 Mon Sep 17 00:00:00 2001 From: Sai Pavan Boddu Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 08/90] xlnx_dp: Introduce a vblank signal Add a periodic timer which raises vblank at a frequency of 30Hz. Note that this is a migration compatibility break for the xlnx-zcu102 board type. Signed-off-by: Sai Pavan Boddu Signed-off-by: Edgar E. Iglesias Signed-off-by: Frederic Konrad Acked-by: Alistair Francis Message-id: 20220601172353.3220232-3-fkonrad@xilinx.com Changes by fkonrad: - Switched to transaction-based ptimer API. - Added the DP_INT_VBLNK_START macro. Signed-off-by: Frederic Konrad [PMM: bump vmstate version, add commit message note about compat break] Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 30 ++++++++++++++++++++++++++---- include/hw/display/xlnx_dp.h | 3 +++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index 0378570459da..ed856b596dad 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -114,6 +114,7 @@ #define DP_TX_N_AUD (0x032C >> 2) #define DP_TX_AUDIO_EXT_DATA(n) ((0x0330 + 4 * n) >> 2) #define DP_INT_STATUS (0x03A0 >> 2) +#define DP_INT_VBLNK_START (1 << 13) #define DP_INT_MASK (0x03A4 >> 2) #define DP_INT_EN (0x03A8 >> 2) #define DP_INT_DS (0x03AC >> 2) @@ -260,7 +261,7 @@ typedef enum DPVideoFmt DPVideoFmt; static const VMStateDescription vmstate_dp = { .name = TYPE_XLNX_DP, - .version_id = 1, + .version_id = 2, .fields = (VMStateField[]){ VMSTATE_UINT32_ARRAY(core_registers, XlnxDPState, DP_CORE_REG_ARRAY_SIZE), @@ -270,10 +271,15 @@ static const VMStateDescription vmstate_dp = { DP_VBLEND_REG_ARRAY_SIZE), VMSTATE_UINT32_ARRAY(audio_registers, XlnxDPState, DP_AUDIO_REG_ARRAY_SIZE), + VMSTATE_PTIMER(vblank, XlnxDPState), VMSTATE_END_OF_LIST() } }; +#define DP_VBLANK_PTIMER_POLICY (PTIMER_POLICY_WRAP_AFTER_ONE_PERIOD | \ + PTIMER_POLICY_CONTINUOUS_TRIGGER | \ + PTIMER_POLICY_NO_IMMEDIATE_TRIGGER) + static void xlnx_dp_update_irq(XlnxDPState *s); static uint64_t xlnx_dp_audio_read(void *opaque, hwaddr offset, unsigned size) @@ -773,6 +779,13 @@ static void xlnx_dp_write(void *opaque, hwaddr offset, uint64_t value, break; case DP_TRANSMITTER_ENABLE: s->core_registers[offset] = value & 0x01; + ptimer_transaction_begin(s->vblank); + if (value & 0x1) { + ptimer_run(s->vblank, 0); + } else { + ptimer_stop(s->vblank); + } + ptimer_transaction_commit(s->vblank); break; case DP_FORCE_SCRAMBLER_RESET: /* @@ -1177,9 +1190,6 @@ static void xlnx_dp_update_display(void *opaque) return; } - s->core_registers[DP_INT_STATUS] |= (1 << 13); - xlnx_dp_update_irq(s); - xlnx_dpdma_trigger_vsync_irq(s->dpdma); /* @@ -1275,6 +1285,14 @@ static void xlnx_dp_finalize(Object *obj) fifo8_destroy(&s->rx_fifo); } +static void vblank_hit(void *opaque) +{ + XlnxDPState *s = XLNX_DP(opaque); + + s->core_registers[DP_INT_STATUS] |= DP_INT_VBLNK_START; + xlnx_dp_update_irq(s); +} + static void xlnx_dp_realize(DeviceState *dev, Error **errp) { XlnxDPState *s = XLNX_DP(dev); @@ -1309,6 +1327,10 @@ static void xlnx_dp_realize(DeviceState *dev, Error **errp) &as); AUD_set_volume_out(s->amixer_output_stream, 0, 255, 255); xlnx_dp_audio_activate(s); + s->vblank = ptimer_init(vblank_hit, s, DP_VBLANK_PTIMER_POLICY); + ptimer_transaction_begin(s->vblank); + ptimer_set_freq(s->vblank, 30); + ptimer_transaction_commit(s->vblank); } static void xlnx_dp_reset(DeviceState *dev) diff --git a/include/hw/display/xlnx_dp.h b/include/hw/display/xlnx_dp.h index 1ef5a89ee744..e86a87f235e1 100644 --- a/include/hw/display/xlnx_dp.h +++ b/include/hw/display/xlnx_dp.h @@ -35,6 +35,7 @@ #include "hw/dma/xlnx_dpdma.h" #include "audio/audio.h" #include "qom/object.h" +#include "hw/ptimer.h" #define AUD_CHBUF_MAX_DEPTH (32 * KiB) #define MAX_QEMU_BUFFER_SIZE (4 * KiB) @@ -107,6 +108,8 @@ struct XlnxDPState { */ DPCDState *dpcd; I2CDDCState *edid; + + ptimer_state *vblank; }; #define TYPE_XLNX_DP "xlnx.v-dp" From 39f40d02f6af8e4718100b6a3854e963c962cfd7 Mon Sep 17 00:00:00 2001 From: Sai Pavan Boddu Date: Wed, 8 Jun 2022 19:38:47 +0100 Subject: [PATCH 09/90] xlnx_dp: Fix the interrupt disable logic Fix interrupt disable logic. Mask value 1 indicates that interrupts are disabled. Signed-off-by: Sai Pavan Boddu Reviewed-by: Edgar E. Iglesias Signed-off-by: Frederic Konrad Acked-by: Alistair Francis Message-id: 20220601172353.3220232-4-fkonrad@xilinx.com Signed-off-by: Peter Maydell --- hw/display/xlnx_dp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw/display/xlnx_dp.c b/hw/display/xlnx_dp.c index ed856b596dad..a071c818833b 100644 --- a/hw/display/xlnx_dp.c +++ b/hw/display/xlnx_dp.c @@ -889,7 +889,7 @@ static void xlnx_dp_write(void *opaque, hwaddr offset, uint64_t value, xlnx_dp_update_irq(s); break; case DP_INT_DS: - s->core_registers[DP_INT_MASK] |= ~value; + s->core_registers[DP_INT_MASK] |= value; xlnx_dp_update_irq(s); break; default: From b3f5cc3fda5775b34aa52e111d50c9d10911b352 Mon Sep 17 00:00:00 2001 From: Frederic Konrad Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 10/90] xlnx-zynqmp: fix the irq mapping for the display port and its dma When the display port has been initially implemented the device driver wasn't using interrupts. Now that the display port driver waits for vblank interrupt it has been noticed that the irq mapping is wrong. So use the value from the linux device tree and the ultrascale+ reference manual. Signed-off-by: Frederic Konrad Reviewed-by: Edgar E. Iglesias Acked-by: Alistair Francis Message-id: 20220601172353.3220232-5-fkonrad@xilinx.com [PMM: refold lines in commit message] Signed-off-by: Peter Maydell --- hw/arm/xlnx-zynqmp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hw/arm/xlnx-zynqmp.c b/hw/arm/xlnx-zynqmp.c index 375309e68ebf..383e177a0019 100644 --- a/hw/arm/xlnx-zynqmp.c +++ b/hw/arm/xlnx-zynqmp.c @@ -60,10 +60,10 @@ #define SERDES_SIZE 0x20000 #define DP_ADDR 0xfd4a0000 -#define DP_IRQ 113 +#define DP_IRQ 0x77 #define DPDMA_ADDR 0xfd4c0000 -#define DPDMA_IRQ 116 +#define DPDMA_IRQ 0x7a #define APU_ADDR 0xfd5c0000 #define APU_IRQ 153 From d8cca960a9a5dbd216d2331cef3cc72f2c7338e0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 11/90] target/arm: Move stage_1_mmu_idx decl to internals.h Move the decl from ptw.h to internals.h. Provide an inline version for user-only, just as we do for arm_stage1_mmu_idx. Move an endif down to make the definition in helper.c be system only. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-2-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 5 ++--- target/arm/internals.h | 5 +++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 5727ead5e4ce..829b660db92f 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10517,12 +10517,10 @@ static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, } } -#endif /* !CONFIG_USER_ONLY */ - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ -static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) { switch (mmu_idx) { case ARMMMUIdx_SE10_0: @@ -10541,6 +10539,7 @@ static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) return mmu_idx; } } +#endif /* !CONFIG_USER_ONLY */ /* Return true if the translation regime is using LPAE format page tables */ static inline bool regime_using_lpae_format(CPUARMState *env, diff --git a/target/arm/internals.h b/target/arm/internals.h index 1e4887b2dd33..049edce946c5 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -979,11 +979,16 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env); * Return the ARMMMUIdx for the stage1 traversal for the current regime. */ #ifdef CONFIG_USER_ONLY +static inline ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +{ + return ARMMMUIdx_Stage1_E0; +} static inline ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) { return ARMMMUIdx_Stage1_E0; } #else +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx); ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env); #endif From 8ae0886002aaf917601701df933617b2449e94b5 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 12/90] target/arm: Move get_phys_addr to ptw.c Begin moving all of the page table walking functions out of helper.c, starting with get_phys_addr(). Create a temporary header file, "ptw.h", in which to share declarations between the two C files while we are moving functions. Move a few declarations to "internals.h", which will remain used by multiple C files. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-3-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 344 +++++------------------------------------ target/arm/internals.h | 18 ++- target/arm/meson.build | 1 + target/arm/ptw.c | 267 ++++++++++++++++++++++++++++++++ target/arm/ptw.h | 51 ++++++ 5 files changed, 372 insertions(+), 309 deletions(-) create mode 100644 target/arm/ptw.c create mode 100644 target/arm/ptw.h diff --git a/target/arm/helper.c b/target/arm/helper.c index 829b660db92f..3ffd122178d8 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -37,22 +37,11 @@ #include "semihosting/common-semi.h" #endif #include "cpregs.h" +#include "ptw.h" #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */ -#ifndef CONFIG_USER_ONLY - -static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) - __attribute__((nonnull)); -#endif - static void switch_mode(CPUARMState *env, int mode); -static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx); static uint64_t raw_read(CPUARMState *env, const ARMCPRegInfo *ri) { @@ -10440,17 +10429,10 @@ uint64_t arm_sctlr(CPUARMState *env, int el) return env->cp15.sctlr_el[el]; } -/* Return the SCTLR value which controls this address translation regime */ -static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; -} - #ifndef CONFIG_USER_ONLY /* Return true if the specified stage of address translation is disabled */ -static inline bool regime_translation_disabled(CPUARMState *env, - ARMMMUIdx mmu_idx) +bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) { uint64_t hcr_el2; @@ -10542,8 +10524,7 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) #endif /* !CONFIG_USER_ONLY */ /* Return true if the translation regime is using LPAE format page tables */ -static inline bool regime_using_lpae_format(CPUARMState *env, - ARMMMUIdx mmu_idx) +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) { int el = regime_el(env, mmu_idx); if (el == 2 || arm_el_is_aa64(env, el)) { @@ -10567,7 +10548,7 @@ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) } #ifndef CONFIG_USER_ONLY -static inline bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) +bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { switch (mmu_idx) { case ARMMMUIdx_SE10_0: @@ -10959,11 +10940,11 @@ static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) { CPUState *cs = env_cpu(env); int level = 1; @@ -11081,10 +11062,10 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, return true; } -static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi) +bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi) { CPUState *cs = env_cpu(env); ARMCPU *cpu = env_archcpu(env); @@ -11360,7 +11341,7 @@ unsigned int arm_pamax(ARMCPU *cpu) return pamax_map[parange]; } -static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) +int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { return extract64(tcr, 37, 2); @@ -11372,7 +11353,7 @@ static int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) } } -static int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx) +int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { return extract64(tcr, 51, 2); @@ -11602,12 +11583,12 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, * @fi: set to fault info if the translation fails * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes */ -static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); @@ -12055,11 +12036,11 @@ static inline bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) { ARMCPU *cpu = env_archcpu(env); int n; @@ -12501,11 +12482,11 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, } -static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi) { uint32_t secure = regime_is_secure(env, mmu_idx); V8M_SAttributes sattrs = {}; @@ -12575,10 +12556,10 @@ static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } -static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi) +bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi) { int n; uint32_t mask; @@ -12795,8 +12776,8 @@ static uint8_t combined_attrs_fwb(CPUARMState *env, * @s1: Attributes from stage 1 walk * @s2: Attributes from stage 2 walk */ -static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) +ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) { ARMCacheAttrs ret; bool tagged = false; @@ -12848,256 +12829,6 @@ static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, return ret; } - -/* get_phys_addr - get the physical address for this virtual address - * - * Find the physical address corresponding to the given virtual address, - * by doing a translation table walk on MMU based systems or using the - * MPU state on MPU based systems. - * - * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, - * prot and page_size may not be filled in, and the populated fsr value provides - * information on why the translation aborted, in the format of a - * DFSR/IFSR fault register, with the following caveats: - * * we honour the short vs long DFSR format differences. - * * the WnR bit is never set (the caller must do this). - * * for PSMAv5 based systems we don't bother to return a full FSR format - * value. - * - * @env: CPUARMState - * @address: virtual address to get physical address for - * @access_type: 0 for read, 1 for write, 2 for execute - * @mmu_idx: MMU index indicating required translation regime - * @phys_ptr: set to the physical address corresponding to the virtual address - * @attrs: set to the memory transaction attributes to use - * @prot: set to the permissions for the page containing phys_ptr - * @page_size: set to the size of the page containing phys_ptr - * @fi: set to fault info if the translation fails - * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes - */ -bool get_phys_addr(CPUARMState *env, target_ulong address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) -{ - ARMMMUIdx s1_mmu_idx = stage_1_mmu_idx(mmu_idx); - - if (mmu_idx != s1_mmu_idx) { - /* Call ourselves recursively to do the stage 1 and then stage 2 - * translations if mmu_idx is a two-stage regime. - */ - if (arm_feature(env, ARM_FEATURE_EL2)) { - hwaddr ipa; - int s2_prot; - int ret; - bool ipa_secure; - ARMCacheAttrs cacheattrs2 = {}; - ARMMMUIdx s2_mmu_idx; - bool is_el0; - - ret = get_phys_addr(env, address, access_type, s1_mmu_idx, &ipa, - attrs, prot, page_size, fi, cacheattrs); - - /* If S1 fails or S2 is disabled, return early. */ - if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { - *phys_ptr = ipa; - return ret; - } - - ipa_secure = attrs->secure; - if (arm_is_secure_below_el3(env)) { - if (ipa_secure) { - attrs->secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); - } else { - attrs->secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); - } - } else { - assert(!ipa_secure); - } - - s2_mmu_idx = attrs->secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; - is_el0 = mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_SE10_0; - - /* S1 is done. Now do S2 translation. */ - ret = get_phys_addr_lpae(env, ipa, access_type, s2_mmu_idx, is_el0, - phys_ptr, attrs, &s2_prot, - page_size, fi, &cacheattrs2); - fi->s2addr = ipa; - /* Combine the S1 and S2 perms. */ - *prot &= s2_prot; - - /* If S2 fails, return early. */ - if (ret) { - return ret; - } - - /* Combine the S1 and S2 cache attributes. */ - if (arm_hcr_el2_eff(env) & HCR_DC) { - /* - * HCR.DC forces the first stage attributes to - * Normal Non-Shareable, - * Inner Write-Back Read-Allocate Write-Allocate, - * Outer Write-Back Read-Allocate Write-Allocate. - * Do not overwrite Tagged within attrs. - */ - if (cacheattrs->attrs != 0xf0) { - cacheattrs->attrs = 0xff; - } - cacheattrs->shareability = 0; - } - *cacheattrs = combine_cacheattrs(env, *cacheattrs, cacheattrs2); - - /* Check if IPA translates to secure or non-secure PA space. */ - if (arm_is_secure_below_el3(env)) { - if (ipa_secure) { - attrs->secure = - !(env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW)); - } else { - attrs->secure = - !((env->cp15.vtcr_el2.raw_tcr & (VTCR_NSA | VTCR_NSW)) - || (env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW))); - } - } - return 0; - } else { - /* - * For non-EL2 CPUs a stage1+stage2 translation is just stage 1. - */ - mmu_idx = stage_1_mmu_idx(mmu_idx); - } - } - - /* The page table entries may downgrade secure to non-secure, but - * cannot upgrade an non-secure translation regime's attributes - * to secure. - */ - attrs->secure = regime_is_secure(env, mmu_idx); - attrs->user = regime_is_user(env, mmu_idx); - - /* Fast Context Switch Extension. This doesn't exist at all in v8. - * In v7 and earlier it affects all stage 1 translations. - */ - if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 - && !arm_feature(env, ARM_FEATURE_V8)) { - if (regime_el(env, mmu_idx) == 3) { - address += env->cp15.fcseidr_s; - } else { - address += env->cp15.fcseidr_ns; - } - } - - if (arm_feature(env, ARM_FEATURE_PMSA)) { - bool ret; - *page_size = TARGET_PAGE_SIZE; - - if (arm_feature(env, ARM_FEATURE_V8)) { - /* PMSAv8 */ - ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx, - phys_ptr, attrs, prot, page_size, fi); - } else if (arm_feature(env, ARM_FEATURE_V7)) { - /* PMSAv7 */ - ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx, - phys_ptr, prot, page_size, fi); - } else { - /* Pre-v7 MPU */ - ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx, - phys_ptr, prot, fi); - } - qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32 - " mmu_idx %u -> %s (prot %c%c%c)\n", - access_type == MMU_DATA_LOAD ? "reading" : - (access_type == MMU_DATA_STORE ? "writing" : "execute"), - (uint32_t)address, mmu_idx, - ret ? "Miss" : "Hit", - *prot & PAGE_READ ? 'r' : '-', - *prot & PAGE_WRITE ? 'w' : '-', - *prot & PAGE_EXEC ? 'x' : '-'); - - return ret; - } - - /* Definitely a real MMU, not an MPU */ - - if (regime_translation_disabled(env, mmu_idx)) { - uint64_t hcr; - uint8_t memattr; - - /* - * MMU disabled. S1 addresses within aa64 translation regimes are - * still checked for bounds -- see AArch64.TranslateAddressS1Off. - */ - if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { - int r_el = regime_el(env, mmu_idx); - if (arm_el_is_aa64(env, r_el)) { - int pamax = arm_pamax(env_archcpu(env)); - uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr; - int addrtop, tbi; - - tbi = aa64_va_parameter_tbi(tcr, mmu_idx); - if (access_type == MMU_INST_FETCH) { - tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx); - } - tbi = (tbi >> extract64(address, 55, 1)) & 1; - addrtop = (tbi ? 55 : 63); - - if (extract64(address, pamax, addrtop - pamax + 1) != 0) { - fi->type = ARMFault_AddressSize; - fi->level = 0; - fi->stage2 = false; - return 1; - } - - /* - * When TBI is disabled, we've just validated that all of the - * bits above PAMax are zero, so logically we only need to - * clear the top byte for TBI. But it's clearer to follow - * the pseudocode set of addrdesc.paddress. - */ - address = extract64(address, 0, 52); - } - } - *phys_ptr = address; - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - *page_size = TARGET_PAGE_SIZE; - - /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */ - hcr = arm_hcr_el2_eff(env); - cacheattrs->shareability = 0; - cacheattrs->is_s2_format = false; - if (hcr & HCR_DC) { - if (hcr & HCR_DCT) { - memattr = 0xf0; /* Tagged, Normal, WB, RWA */ - } else { - memattr = 0xff; /* Normal, WB, RWA */ - } - } else if (access_type == MMU_INST_FETCH) { - if (regime_sctlr(env, mmu_idx) & SCTLR_I) { - memattr = 0xee; /* Normal, WT, RA, NT */ - } else { - memattr = 0x44; /* Normal, NC, No */ - } - cacheattrs->shareability = 2; /* outer sharable */ - } else { - memattr = 0x00; /* Device, nGnRnE */ - } - cacheattrs->attrs = memattr; - return 0; - } - - if (regime_using_lpae_format(env, mmu_idx)) { - return get_phys_addr_lpae(env, address, access_type, mmu_idx, false, - phys_ptr, attrs, prot, page_size, - fi, cacheattrs); - } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) { - return get_phys_addr_v6(env, address, access_type, mmu_idx, - phys_ptr, attrs, prot, page_size, fi); - } else { - return get_phys_addr_v5(env, address, access_type, mmu_idx, - phys_ptr, prot, page_size, fi); - } -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { @@ -13121,7 +12852,6 @@ hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, } return phys_addr; } - #endif /* Note that signed overflow is undefined in C. The following routines are diff --git a/target/arm/internals.h b/target/arm/internals.h index 049edce946c5..1d83146d565e 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -613,8 +613,13 @@ ARMMMUIdx arm_v7m_mmu_idx_for_secstate_and_priv(CPUARMState *env, /* Return the MMU index for a v7M CPU in the specified security state */ ARMMMUIdx arm_v7m_mmu_idx_for_secstate(CPUARMState *env, bool secstate); -/* Return true if the stage 1 translation regime is using LPAE format page - * tables */ +/* Return true if the translation regime is using LPAE format page tables */ +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); + +/* + * Return true if the stage 1 translation regime is using LPAE + * format page tables + */ bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx); /* Raise a data fault alignment exception for the specified virtual address */ @@ -777,6 +782,12 @@ static inline uint32_t regime_el(CPUARMState *env, ARMMMUIdx mmu_idx) } } +/* Return the SCTLR value which controls this address translation regime */ +static inline uint64_t regime_sctlr(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + return env->cp15.sctlr_el[regime_el(env, mmu_idx)]; +} + /* Return the TCR controlling this translation regime */ static inline TCR *regime_tcr(CPUARMState *env, ARMMMUIdx mmu_idx) { @@ -1095,6 +1106,9 @@ typedef struct ARMVAParameters { ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, ARMMMUIdx mmu_idx, bool data); +int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx); +int aa64_va_parameter_tbid(uint64_t tcr, ARMMMUIdx mmu_idx); + static inline int exception_target_el(CPUARMState *env) { int target_el = MAX(1, arm_current_el(env)); diff --git a/target/arm/meson.build b/target/arm/meson.build index 50f152214afe..ac571fc45dbd 100644 --- a/target/arm/meson.build +++ b/target/arm/meson.build @@ -58,6 +58,7 @@ arm_softmmu_ss.add(files( 'machine.c', 'monitor.c', 'psci.c', + 'ptw.c', )) subdir('hvf') diff --git a/target/arm/ptw.c b/target/arm/ptw.c new file mode 100644 index 000000000000..318000f6d941 --- /dev/null +++ b/target/arm/ptw.c @@ -0,0 +1,267 @@ +/* + * ARM page table walking. + * + * This code is licensed under the GNU GPL v2 or later. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#include "qemu/osdep.h" +#include "qemu/log.h" +#include "cpu.h" +#include "internals.h" +#include "ptw.h" + + +/** + * get_phys_addr - get the physical address for this virtual address + * + * Find the physical address corresponding to the given virtual address, + * by doing a translation table walk on MMU based systems or using the + * MPU state on MPU based systems. + * + * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, + * prot and page_size may not be filled in, and the populated fsr value provides + * information on why the translation aborted, in the format of a + * DFSR/IFSR fault register, with the following caveats: + * * we honour the short vs long DFSR format differences. + * * the WnR bit is never set (the caller must do this). + * * for PSMAv5 based systems we don't bother to return a full FSR format + * value. + * + * @env: CPUARMState + * @address: virtual address to get physical address for + * @access_type: 0 for read, 1 for write, 2 for execute + * @mmu_idx: MMU index indicating required translation regime + * @phys_ptr: set to the physical address corresponding to the virtual address + * @attrs: set to the memory transaction attributes to use + * @prot: set to the permissions for the page containing phys_ptr + * @page_size: set to the size of the page containing phys_ptr + * @fi: set to fault info if the translation fails + * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes + */ +bool get_phys_addr(CPUARMState *env, target_ulong address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +{ + ARMMMUIdx s1_mmu_idx = stage_1_mmu_idx(mmu_idx); + + if (mmu_idx != s1_mmu_idx) { + /* + * Call ourselves recursively to do the stage 1 and then stage 2 + * translations if mmu_idx is a two-stage regime. + */ + if (arm_feature(env, ARM_FEATURE_EL2)) { + hwaddr ipa; + int s2_prot; + int ret; + bool ipa_secure; + ARMCacheAttrs cacheattrs2 = {}; + ARMMMUIdx s2_mmu_idx; + bool is_el0; + + ret = get_phys_addr(env, address, access_type, s1_mmu_idx, &ipa, + attrs, prot, page_size, fi, cacheattrs); + + /* If S1 fails or S2 is disabled, return early. */ + if (ret || regime_translation_disabled(env, ARMMMUIdx_Stage2)) { + *phys_ptr = ipa; + return ret; + } + + ipa_secure = attrs->secure; + if (arm_is_secure_below_el3(env)) { + if (ipa_secure) { + attrs->secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); + } else { + attrs->secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); + } + } else { + assert(!ipa_secure); + } + + s2_mmu_idx = attrs->secure ? ARMMMUIdx_Stage2_S : ARMMMUIdx_Stage2; + is_el0 = mmu_idx == ARMMMUIdx_E10_0 || mmu_idx == ARMMMUIdx_SE10_0; + + /* S1 is done. Now do S2 translation. */ + ret = get_phys_addr_lpae(env, ipa, access_type, s2_mmu_idx, is_el0, + phys_ptr, attrs, &s2_prot, + page_size, fi, &cacheattrs2); + fi->s2addr = ipa; + /* Combine the S1 and S2 perms. */ + *prot &= s2_prot; + + /* If S2 fails, return early. */ + if (ret) { + return ret; + } + + /* Combine the S1 and S2 cache attributes. */ + if (arm_hcr_el2_eff(env) & HCR_DC) { + /* + * HCR.DC forces the first stage attributes to + * Normal Non-Shareable, + * Inner Write-Back Read-Allocate Write-Allocate, + * Outer Write-Back Read-Allocate Write-Allocate. + * Do not overwrite Tagged within attrs. + */ + if (cacheattrs->attrs != 0xf0) { + cacheattrs->attrs = 0xff; + } + cacheattrs->shareability = 0; + } + *cacheattrs = combine_cacheattrs(env, *cacheattrs, cacheattrs2); + + /* Check if IPA translates to secure or non-secure PA space. */ + if (arm_is_secure_below_el3(env)) { + if (ipa_secure) { + attrs->secure = + !(env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW)); + } else { + attrs->secure = + !((env->cp15.vtcr_el2.raw_tcr & (VTCR_NSA | VTCR_NSW)) + || (env->cp15.vstcr_el2.raw_tcr & (VSTCR_SA | VSTCR_SW))); + } + } + return 0; + } else { + /* + * For non-EL2 CPUs a stage1+stage2 translation is just stage 1. + */ + mmu_idx = stage_1_mmu_idx(mmu_idx); + } + } + + /* + * The page table entries may downgrade secure to non-secure, but + * cannot upgrade an non-secure translation regime's attributes + * to secure. + */ + attrs->secure = regime_is_secure(env, mmu_idx); + attrs->user = regime_is_user(env, mmu_idx); + + /* + * Fast Context Switch Extension. This doesn't exist at all in v8. + * In v7 and earlier it affects all stage 1 translations. + */ + if (address < 0x02000000 && mmu_idx != ARMMMUIdx_Stage2 + && !arm_feature(env, ARM_FEATURE_V8)) { + if (regime_el(env, mmu_idx) == 3) { + address += env->cp15.fcseidr_s; + } else { + address += env->cp15.fcseidr_ns; + } + } + + if (arm_feature(env, ARM_FEATURE_PMSA)) { + bool ret; + *page_size = TARGET_PAGE_SIZE; + + if (arm_feature(env, ARM_FEATURE_V8)) { + /* PMSAv8 */ + ret = get_phys_addr_pmsav8(env, address, access_type, mmu_idx, + phys_ptr, attrs, prot, page_size, fi); + } else if (arm_feature(env, ARM_FEATURE_V7)) { + /* PMSAv7 */ + ret = get_phys_addr_pmsav7(env, address, access_type, mmu_idx, + phys_ptr, prot, page_size, fi); + } else { + /* Pre-v7 MPU */ + ret = get_phys_addr_pmsav5(env, address, access_type, mmu_idx, + phys_ptr, prot, fi); + } + qemu_log_mask(CPU_LOG_MMU, "PMSA MPU lookup for %s at 0x%08" PRIx32 + " mmu_idx %u -> %s (prot %c%c%c)\n", + access_type == MMU_DATA_LOAD ? "reading" : + (access_type == MMU_DATA_STORE ? "writing" : "execute"), + (uint32_t)address, mmu_idx, + ret ? "Miss" : "Hit", + *prot & PAGE_READ ? 'r' : '-', + *prot & PAGE_WRITE ? 'w' : '-', + *prot & PAGE_EXEC ? 'x' : '-'); + + return ret; + } + + /* Definitely a real MMU, not an MPU */ + + if (regime_translation_disabled(env, mmu_idx)) { + uint64_t hcr; + uint8_t memattr; + + /* + * MMU disabled. S1 addresses within aa64 translation regimes are + * still checked for bounds -- see AArch64.TranslateAddressS1Off. + */ + if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { + int r_el = regime_el(env, mmu_idx); + if (arm_el_is_aa64(env, r_el)) { + int pamax = arm_pamax(env_archcpu(env)); + uint64_t tcr = env->cp15.tcr_el[r_el].raw_tcr; + int addrtop, tbi; + + tbi = aa64_va_parameter_tbi(tcr, mmu_idx); + if (access_type == MMU_INST_FETCH) { + tbi &= ~aa64_va_parameter_tbid(tcr, mmu_idx); + } + tbi = (tbi >> extract64(address, 55, 1)) & 1; + addrtop = (tbi ? 55 : 63); + + if (extract64(address, pamax, addrtop - pamax + 1) != 0) { + fi->type = ARMFault_AddressSize; + fi->level = 0; + fi->stage2 = false; + return 1; + } + + /* + * When TBI is disabled, we've just validated that all of the + * bits above PAMax are zero, so logically we only need to + * clear the top byte for TBI. But it's clearer to follow + * the pseudocode set of addrdesc.paddress. + */ + address = extract64(address, 0, 52); + } + } + *phys_ptr = address; + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + *page_size = TARGET_PAGE_SIZE; + + /* Fill in cacheattr a-la AArch64.TranslateAddressS1Off. */ + hcr = arm_hcr_el2_eff(env); + cacheattrs->shareability = 0; + cacheattrs->is_s2_format = false; + if (hcr & HCR_DC) { + if (hcr & HCR_DCT) { + memattr = 0xf0; /* Tagged, Normal, WB, RWA */ + } else { + memattr = 0xff; /* Normal, WB, RWA */ + } + } else if (access_type == MMU_INST_FETCH) { + if (regime_sctlr(env, mmu_idx) & SCTLR_I) { + memattr = 0xee; /* Normal, WT, RA, NT */ + } else { + memattr = 0x44; /* Normal, NC, No */ + } + cacheattrs->shareability = 2; /* outer sharable */ + } else { + memattr = 0x00; /* Device, nGnRnE */ + } + cacheattrs->attrs = memattr; + return 0; + } + + if (regime_using_lpae_format(env, mmu_idx)) { + return get_phys_addr_lpae(env, address, access_type, mmu_idx, false, + phys_ptr, attrs, prot, page_size, + fi, cacheattrs); + } else if (regime_sctlr(env, mmu_idx) & SCTLR_XP) { + return get_phys_addr_v6(env, address, access_type, mmu_idx, + phys_ptr, attrs, prot, page_size, fi); + } else { + return get_phys_addr_v5(env, address, access_type, mmu_idx, + phys_ptr, prot, page_size, fi); + } +} diff --git a/target/arm/ptw.h b/target/arm/ptw.h new file mode 100644 index 000000000000..e2023ae7508f --- /dev/null +++ b/target/arm/ptw.h @@ -0,0 +1,51 @@ +/* + * ARM page table walking. + * + * This code is licensed under the GNU GPL v2 or later. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +#ifndef TARGET_ARM_PTW_H +#define TARGET_ARM_PTW_H + +#ifndef CONFIG_USER_ONLY + +bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); +bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); +ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2); + +bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi); +bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi); +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) + __attribute__((nonnull)); + +#endif /* !CONFIG_USER_ONLY */ +#endif /* TARGET_ARM_PTW_H */ From f2d2f5ceb87e828546f7b544f1289193f3b920ce Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 13/90] target/arm: Move get_phys_addr_v5 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-4-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 137 +++----------------------------------------- target/arm/ptw.c | 123 +++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 15 +++-- 3 files changed, 140 insertions(+), 135 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 3ffd122178d8..321716914b11 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10578,8 +10578,7 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) * @ap: The 3-bit access permissions (AP[2:0]) * @domain_prot: The 2-bit domain access permissions */ -static inline int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, - int ap, int domain_prot) +int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) { bool is_user = regime_is_user(env, mmu_idx); @@ -10782,8 +10781,8 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address) +bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address) { /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ TCR *tcr = regime_tcr(env, mmu_idx); @@ -10882,8 +10881,8 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, } /* All loads done in the course of a page table walk go through here. */ -static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; @@ -10911,8 +10910,8 @@ static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { ARMCPU *cpu = ARM_CPU(cs); CPUARMState *env = &cpu->env; @@ -10940,128 +10939,6 @@ static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - CPUState *cs = env_cpu(env); - int level = 1; - uint32_t table; - uint32_t desc; - int type; - int ap; - int domain = 0; - int domain_prot; - hwaddr phys_addr; - uint32_t dacr; - - /* Pagetable walk. */ - /* Lookup l1 descriptor. */ - if (!get_level1_table_address(env, mmu_idx, &table, address)) { - /* Section translation fault if page walk is disabled by PD0 or PD1 */ - fi->type = ARMFault_Translation; - goto do_fault; - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - type = (desc & 3); - domain = (desc >> 5) & 0x0f; - if (regime_el(env, mmu_idx) == 1) { - dacr = env->cp15.dacr_ns; - } else { - dacr = env->cp15.dacr_s; - } - domain_prot = (dacr >> (domain * 2)) & 3; - if (type == 0) { - /* Section translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - } - if (type != 2) { - level = 2; - } - if (domain_prot == 0 || domain_prot == 2) { - fi->type = ARMFault_Domain; - goto do_fault; - } - if (type == 2) { - /* 1Mb section. */ - phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); - ap = (desc >> 10) & 3; - *page_size = 1024 * 1024; - } else { - /* Lookup l2 entry. */ - if (type == 1) { - /* Coarse pagetable. */ - table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - } else { - /* Fine pagetable. */ - table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - switch (desc & 3) { - case 0: /* Page translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - case 1: /* 64k page. */ - phys_addr = (desc & 0xffff0000) | (address & 0xffff); - ap = (desc >> (4 + ((address >> 13) & 6))) & 3; - *page_size = 0x10000; - break; - case 2: /* 4k page. */ - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - ap = (desc >> (4 + ((address >> 9) & 6))) & 3; - *page_size = 0x1000; - break; - case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */ - if (type == 1) { - /* ARMv6/XScale extended small page format */ - if (arm_feature(env, ARM_FEATURE_XSCALE) - || arm_feature(env, ARM_FEATURE_V6)) { - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - *page_size = 0x1000; - } else { - /* UNPREDICTABLE in ARMv5; we choose to take a - * page translation fault. - */ - fi->type = ARMFault_Translation; - goto do_fault; - } - } else { - phys_addr = (desc & 0xfffffc00) | (address & 0x3ff); - *page_size = 0x400; - } - ap = (desc >> 4) & 3; - break; - default: - /* Never happens, but compiler isn't smart enough to tell. */ - g_assert_not_reached(); - } - } - *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); - *prot |= *prot ? PAGE_EXEC : 0; - if (!(*prot & (1 << access_type))) { - /* Access permission fault. */ - fi->type = ARMFault_Permission; - goto do_fault; - } - *phys_ptr = phys_addr; - return false; -do_fault: - fi->domain = domain; - fi->level = level; - return true; -} - bool get_phys_addr_v6(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 318000f6d941..09c447262876 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -13,6 +13,129 @@ #include "ptw.h" +static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + CPUState *cs = env_cpu(env); + int level = 1; + uint32_t table; + uint32_t desc; + int type; + int ap; + int domain = 0; + int domain_prot; + hwaddr phys_addr; + uint32_t dacr; + + /* Pagetable walk. */ + /* Lookup l1 descriptor. */ + if (!get_level1_table_address(env, mmu_idx, &table, address)) { + /* Section translation fault if page walk is disabled by PD0 or PD1 */ + fi->type = ARMFault_Translation; + goto do_fault; + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + type = (desc & 3); + domain = (desc >> 5) & 0x0f; + if (regime_el(env, mmu_idx) == 1) { + dacr = env->cp15.dacr_ns; + } else { + dacr = env->cp15.dacr_s; + } + domain_prot = (dacr >> (domain * 2)) & 3; + if (type == 0) { + /* Section translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + } + if (type != 2) { + level = 2; + } + if (domain_prot == 0 || domain_prot == 2) { + fi->type = ARMFault_Domain; + goto do_fault; + } + if (type == 2) { + /* 1Mb section. */ + phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); + ap = (desc >> 10) & 3; + *page_size = 1024 * 1024; + } else { + /* Lookup l2 entry. */ + if (type == 1) { + /* Coarse pagetable. */ + table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); + } else { + /* Fine pagetable. */ + table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + switch (desc & 3) { + case 0: /* Page translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + case 1: /* 64k page. */ + phys_addr = (desc & 0xffff0000) | (address & 0xffff); + ap = (desc >> (4 + ((address >> 13) & 6))) & 3; + *page_size = 0x10000; + break; + case 2: /* 4k page. */ + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + ap = (desc >> (4 + ((address >> 9) & 6))) & 3; + *page_size = 0x1000; + break; + case 3: /* 1k page, or ARMv6/XScale "extended small (4k) page" */ + if (type == 1) { + /* ARMv6/XScale extended small page format */ + if (arm_feature(env, ARM_FEATURE_XSCALE) + || arm_feature(env, ARM_FEATURE_V6)) { + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + *page_size = 0x1000; + } else { + /* + * UNPREDICTABLE in ARMv5; we choose to take a + * page translation fault. + */ + fi->type = ARMFault_Translation; + goto do_fault; + } + } else { + phys_addr = (desc & 0xfffffc00) | (address & 0x3ff); + *page_size = 0x400; + } + ap = (desc >> 4) & 3; + break; + default: + /* Never happens, but compiler isn't smart enough to tell. */ + g_assert_not_reached(); + } + } + *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); + *prot |= *prot ? PAGE_EXEC : 0; + if (!(*prot & (1 << access_type))) { + /* Access permission fault. */ + fi->type = ARMFault_Permission; + goto do_fault; + } + *phys_ptr = phys_addr; + return false; +do_fault: + fi->domain = domain; + fi->level = level; + return true; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index e2023ae7508f..2dbd97b8cbf9 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,16 +11,21 @@ #ifndef CONFIG_USER_ONLY +uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); +uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); + bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); ARMCacheAttrs combine_cacheattrs(CPUARMState *env, ARMCacheAttrs s1, ARMCacheAttrs s2); -bool get_phys_addr_v5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi); +bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address); +int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, + int ap, int domain_prot); + bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 53c038efb7a634dd5b6ede8eb034791c0dc83f43 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:48 +0100 Subject: [PATCH 14/90] target/arm: Move get_phys_addr_v6 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-5-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 161 +------------------------------------------- target/arm/ptw.c | 153 +++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 11 +-- 3 files changed, 161 insertions(+), 164 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 321716914b11..4a588220250a 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10631,7 +10631,7 @@ int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) * @ap: The 2-bit simple AP (AP[2:1]) * @is_user: TRUE if accessing from PL0 */ -static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user) +int simple_ap_to_rw_prot_is_user(int ap, bool is_user) { switch (ap) { case 0: @@ -10647,12 +10647,6 @@ static inline int simple_ap_to_rw_prot_is_user(int ap, bool is_user) } } -static inline int -simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) -{ - return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); -} - /* Translate S2 section/page access permissions to protection flags * * @env: CPUARMState @@ -10939,159 +10933,6 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi) -{ - CPUState *cs = env_cpu(env); - ARMCPU *cpu = env_archcpu(env); - int level = 1; - uint32_t table; - uint32_t desc; - uint32_t xn; - uint32_t pxn = 0; - int type; - int ap; - int domain = 0; - int domain_prot; - hwaddr phys_addr; - uint32_t dacr; - bool ns; - - /* Pagetable walk. */ - /* Lookup l1 descriptor. */ - if (!get_level1_table_address(env, mmu_idx, &table, address)) { - /* Section translation fault if page walk is disabled by PD0 or PD1 */ - fi->type = ARMFault_Translation; - goto do_fault; - } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - type = (desc & 3); - if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) { - /* Section translation fault, or attempt to use the encoding - * which is Reserved on implementations without PXN. - */ - fi->type = ARMFault_Translation; - goto do_fault; - } - if ((type == 1) || !(desc & (1 << 18))) { - /* Page or Section. */ - domain = (desc >> 5) & 0x0f; - } - if (regime_el(env, mmu_idx) == 1) { - dacr = env->cp15.dacr_ns; - } else { - dacr = env->cp15.dacr_s; - } - if (type == 1) { - level = 2; - } - domain_prot = (dacr >> (domain * 2)) & 3; - if (domain_prot == 0 || domain_prot == 2) { - /* Section or Page domain fault */ - fi->type = ARMFault_Domain; - goto do_fault; - } - if (type != 1) { - if (desc & (1 << 18)) { - /* Supersection. */ - phys_addr = (desc & 0xff000000) | (address & 0x00ffffff); - phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32; - phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36; - *page_size = 0x1000000; - } else { - /* Section. */ - phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); - *page_size = 0x100000; - } - ap = ((desc >> 10) & 3) | ((desc >> 13) & 4); - xn = desc & (1 << 4); - pxn = desc & 1; - ns = extract32(desc, 19, 1); - } else { - if (cpu_isar_feature(aa32_pxn, cpu)) { - pxn = (desc >> 2) & 1; - } - ns = extract32(desc, 3, 1); - /* Lookup l2 entry. */ - table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), - mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - ap = ((desc >> 4) & 3) | ((desc >> 7) & 4); - switch (desc & 3) { - case 0: /* Page translation fault. */ - fi->type = ARMFault_Translation; - goto do_fault; - case 1: /* 64k page. */ - phys_addr = (desc & 0xffff0000) | (address & 0xffff); - xn = desc & (1 << 15); - *page_size = 0x10000; - break; - case 2: case 3: /* 4k page. */ - phys_addr = (desc & 0xfffff000) | (address & 0xfff); - xn = desc & 1; - *page_size = 0x1000; - break; - default: - /* Never happens, but compiler isn't smart enough to tell. */ - g_assert_not_reached(); - } - } - if (domain_prot == 3) { - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - } else { - if (pxn && !regime_is_user(env, mmu_idx)) { - xn = 1; - } - if (xn && access_type == MMU_INST_FETCH) { - fi->type = ARMFault_Permission; - goto do_fault; - } - - if (arm_feature(env, ARM_FEATURE_V6K) && - (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) { - /* The simplified model uses AP[0] as an access control bit. */ - if ((ap & 1) == 0) { - /* Access flag fault. */ - fi->type = ARMFault_AccessFlag; - goto do_fault; - } - *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1); - } else { - *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); - } - if (*prot && !xn) { - *prot |= PAGE_EXEC; - } - if (!(*prot & (1 << access_type))) { - /* Access permission fault. */ - fi->type = ARMFault_Permission; - goto do_fault; - } - } - if (ns) { - /* The NS bit will (as required by the architecture) have no effect if - * the CPU doesn't support TZ or this is a non-secure translation - * regime, because the attribute will already be non-secure. - */ - attrs->secure = false; - } - *phys_ptr = phys_addr; - return false; -do_fault: - fi->domain = domain; - fi->level = level; - return true; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 09c447262876..6a1f4b549d88 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -136,6 +136,159 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, return true; } +static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, + target_ulong *page_size, ARMMMUFaultInfo *fi) +{ + CPUState *cs = env_cpu(env); + ARMCPU *cpu = env_archcpu(env); + int level = 1; + uint32_t table; + uint32_t desc; + uint32_t xn; + uint32_t pxn = 0; + int type; + int ap; + int domain = 0; + int domain_prot; + hwaddr phys_addr; + uint32_t dacr; + bool ns; + + /* Pagetable walk. */ + /* Lookup l1 descriptor. */ + if (!get_level1_table_address(env, mmu_idx, &table, address)) { + /* Section translation fault if page walk is disabled by PD0 or PD1 */ + fi->type = ARMFault_Translation; + goto do_fault; + } + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + type = (desc & 3); + if (type == 0 || (type == 3 && !cpu_isar_feature(aa32_pxn, cpu))) { + /* Section translation fault, or attempt to use the encoding + * which is Reserved on implementations without PXN. + */ + fi->type = ARMFault_Translation; + goto do_fault; + } + if ((type == 1) || !(desc & (1 << 18))) { + /* Page or Section. */ + domain = (desc >> 5) & 0x0f; + } + if (regime_el(env, mmu_idx) == 1) { + dacr = env->cp15.dacr_ns; + } else { + dacr = env->cp15.dacr_s; + } + if (type == 1) { + level = 2; + } + domain_prot = (dacr >> (domain * 2)) & 3; + if (domain_prot == 0 || domain_prot == 2) { + /* Section or Page domain fault */ + fi->type = ARMFault_Domain; + goto do_fault; + } + if (type != 1) { + if (desc & (1 << 18)) { + /* Supersection. */ + phys_addr = (desc & 0xff000000) | (address & 0x00ffffff); + phys_addr |= (uint64_t)extract32(desc, 20, 4) << 32; + phys_addr |= (uint64_t)extract32(desc, 5, 4) << 36; + *page_size = 0x1000000; + } else { + /* Section. */ + phys_addr = (desc & 0xfff00000) | (address & 0x000fffff); + *page_size = 0x100000; + } + ap = ((desc >> 10) & 3) | ((desc >> 13) & 4); + xn = desc & (1 << 4); + pxn = desc & 1; + ns = extract32(desc, 19, 1); + } else { + if (cpu_isar_feature(aa32_pxn, cpu)) { + pxn = (desc >> 2) & 1; + } + ns = extract32(desc, 3, 1); + /* Lookup l2 entry. */ + table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); + desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + ap = ((desc >> 4) & 3) | ((desc >> 7) & 4); + switch (desc & 3) { + case 0: /* Page translation fault. */ + fi->type = ARMFault_Translation; + goto do_fault; + case 1: /* 64k page. */ + phys_addr = (desc & 0xffff0000) | (address & 0xffff); + xn = desc & (1 << 15); + *page_size = 0x10000; + break; + case 2: case 3: /* 4k page. */ + phys_addr = (desc & 0xfffff000) | (address & 0xfff); + xn = desc & 1; + *page_size = 0x1000; + break; + default: + /* Never happens, but compiler isn't smart enough to tell. */ + g_assert_not_reached(); + } + } + if (domain_prot == 3) { + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + } else { + if (pxn && !regime_is_user(env, mmu_idx)) { + xn = 1; + } + if (xn && access_type == MMU_INST_FETCH) { + fi->type = ARMFault_Permission; + goto do_fault; + } + + if (arm_feature(env, ARM_FEATURE_V6K) && + (regime_sctlr(env, mmu_idx) & SCTLR_AFE)) { + /* The simplified model uses AP[0] as an access control bit. */ + if ((ap & 1) == 0) { + /* Access flag fault. */ + fi->type = ARMFault_AccessFlag; + goto do_fault; + } + *prot = simple_ap_to_rw_prot(env, mmu_idx, ap >> 1); + } else { + *prot = ap_to_rw_prot(env, mmu_idx, ap, domain_prot); + } + if (*prot && !xn) { + *prot |= PAGE_EXEC; + } + if (!(*prot & (1 << access_type))) { + /* Access permission fault. */ + fi->type = ARMFault_Permission; + goto do_fault; + } + } + if (ns) { + /* The NS bit will (as required by the architecture) have no effect if + * the CPU doesn't support TZ or this is a non-secure translation + * regime, because the attribute will already be non-secure. + */ + attrs->secure = false; + } + *phys_ptr = phys_addr; + return false; +do_fault: + fi->domain = domain; + fi->level = level; + return true; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 2dbd97b8cbf9..349b842d3cee 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -25,15 +25,18 @@ bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, uint32_t *table, uint32_t address); int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); +int simple_ap_to_rw_prot_is_user(int ap, bool is_user); + +static inline int +simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) +{ + return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); +} bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, ARMMMUFaultInfo *fi); -bool get_phys_addr_v6(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, - target_ulong *page_size, ARMMMUFaultInfo *fi); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 9a12fb366d442751603dc91ab5c5f9b7c828c783 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 15/90] target/arm: Move get_phys_addr_pmsav5 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-6-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 85 --------------------------------------------- target/arm/ptw.c | 85 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 4 --- 3 files changed, 85 insertions(+), 89 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 4a588220250a..5d0101901082 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -12274,91 +12274,6 @@ bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } -bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi) -{ - int n; - uint32_t mask; - uint32_t base; - bool is_user = regime_is_user(env, mmu_idx); - - if (regime_translation_disabled(env, mmu_idx)) { - /* MPU disabled. */ - *phys_ptr = address; - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - return false; - } - - *phys_ptr = address; - for (n = 7; n >= 0; n--) { - base = env->cp15.c6_region[n]; - if ((base & 1) == 0) { - continue; - } - mask = 1 << ((base >> 1) & 0x1f); - /* Keep this shift separate from the above to avoid an - (undefined) << 32. */ - mask = (mask << 1) - 1; - if (((base ^ address) & ~mask) == 0) { - break; - } - } - if (n < 0) { - fi->type = ARMFault_Background; - return true; - } - - if (access_type == MMU_INST_FETCH) { - mask = env->cp15.pmsav5_insn_ap; - } else { - mask = env->cp15.pmsav5_data_ap; - } - mask = (mask >> (n * 4)) & 0xf; - switch (mask) { - case 0: - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - case 1: - if (is_user) { - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot = PAGE_READ | PAGE_WRITE; - break; - case 2: - *prot = PAGE_READ; - if (!is_user) { - *prot |= PAGE_WRITE; - } - break; - case 3: - *prot = PAGE_READ | PAGE_WRITE; - break; - case 5: - if (is_user) { - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot = PAGE_READ; - break; - case 6: - *prot = PAGE_READ; - break; - default: - /* Bad permission. */ - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - *prot |= PAGE_EXEC; - return false; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 6a1f4b549d88..5c32648a16a9 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -289,6 +289,91 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, return true; } +static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + ARMMMUFaultInfo *fi) +{ + int n; + uint32_t mask; + uint32_t base; + bool is_user = regime_is_user(env, mmu_idx); + + if (regime_translation_disabled(env, mmu_idx)) { + /* MPU disabled. */ + *phys_ptr = address; + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + return false; + } + + *phys_ptr = address; + for (n = 7; n >= 0; n--) { + base = env->cp15.c6_region[n]; + if ((base & 1) == 0) { + continue; + } + mask = 1 << ((base >> 1) & 0x1f); + /* Keep this shift separate from the above to avoid an + (undefined) << 32. */ + mask = (mask << 1) - 1; + if (((base ^ address) & ~mask) == 0) { + break; + } + } + if (n < 0) { + fi->type = ARMFault_Background; + return true; + } + + if (access_type == MMU_INST_FETCH) { + mask = env->cp15.pmsav5_insn_ap; + } else { + mask = env->cp15.pmsav5_data_ap; + } + mask = (mask >> (n * 4)) & 0xf; + switch (mask) { + case 0: + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + case 1: + if (is_user) { + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot = PAGE_READ | PAGE_WRITE; + break; + case 2: + *prot = PAGE_READ; + if (!is_user) { + *prot |= PAGE_WRITE; + } + break; + case 3: + *prot = PAGE_READ | PAGE_WRITE; + break; + case 5: + if (is_user) { + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot = PAGE_READ; + break; + case 6: + *prot = PAGE_READ; + break; + default: + /* Bad permission. */ + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + *prot |= PAGE_EXEC; + return false; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 349b842d3cee..324a9dde140c 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,10 +33,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - ARMMMUFaultInfo *fi); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 7d2e08c96077bcf7ed31b6b7c7b7066801b4b89d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 16/90] target/arm: Move get_phys_addr_pmsav7_default to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-7-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 41 ----------------------------------------- target/arm/ptw.c | 41 +++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 +++ 3 files changed, 44 insertions(+), 41 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 5d0101901082..d4f7c05625c6 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,47 +11678,6 @@ bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, return true; } -static inline void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot) -{ - if (!arm_feature(env, ARM_FEATURE_M)) { - *prot = PAGE_READ | PAGE_WRITE; - switch (address) { - case 0xF0000000 ... 0xFFFFFFFF: - if (regime_sctlr(env, mmu_idx) & SCTLR_V) { - /* hivecs execing is ok */ - *prot |= PAGE_EXEC; - } - break; - case 0x00000000 ... 0x7FFFFFFF: - *prot |= PAGE_EXEC; - break; - } - } else { - /* Default system address map for M profile cores. - * The architecture specifies which regions are execute-never; - * at the MPU level no other checks are defined. - */ - switch (address) { - case 0x00000000 ... 0x1fffffff: /* ROM */ - case 0x20000000 ... 0x3fffffff: /* SRAM */ - case 0x60000000 ... 0x7fffffff: /* RAM */ - case 0x80000000 ... 0x9fffffff: /* RAM */ - *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; - break; - case 0x40000000 ... 0x5fffffff: /* Peripheral */ - case 0xa0000000 ... 0xbfffffff: /* Device */ - case 0xc0000000 ... 0xdfffffff: /* Device */ - case 0xe0000000 ... 0xffffffff: /* System */ - *prot = PAGE_READ | PAGE_WRITE; - break; - default: - g_assert_not_reached(); - } - } -} - static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 5c32648a16a9..74650c6c5253 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -374,6 +374,47 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, return false; } +void get_phys_addr_pmsav7_default(CPUARMState *env, + ARMMMUIdx mmu_idx, + int32_t address, int *prot) +{ + if (!arm_feature(env, ARM_FEATURE_M)) { + *prot = PAGE_READ | PAGE_WRITE; + switch (address) { + case 0xF0000000 ... 0xFFFFFFFF: + if (regime_sctlr(env, mmu_idx) & SCTLR_V) { + /* hivecs execing is ok */ + *prot |= PAGE_EXEC; + } + break; + case 0x00000000 ... 0x7FFFFFFF: + *prot |= PAGE_EXEC; + break; + } + } else { + /* Default system address map for M profile cores. + * The architecture specifies which regions are execute-never; + * at the MPU level no other checks are defined. + */ + switch (address) { + case 0x00000000 ... 0x1fffffff: /* ROM */ + case 0x20000000 ... 0x3fffffff: /* SRAM */ + case 0x60000000 ... 0x7fffffff: /* RAM */ + case 0x80000000 ... 0x9fffffff: /* RAM */ + *prot = PAGE_READ | PAGE_WRITE | PAGE_EXEC; + break; + case 0x40000000 ... 0x5fffffff: /* Peripheral */ + case 0xa0000000 ... 0xbfffffff: /* Device */ + case 0xc0000000 ... 0xdfffffff: /* Device */ + case 0xe0000000 ... 0xffffffff: /* System */ + *prot = PAGE_READ | PAGE_WRITE; + break; + default: + g_assert_not_reached(); + } + } +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 324a9dde140c..d6e3fee15230 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,6 +33,9 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +void get_phys_addr_pmsav7_default(CPUARMState *env, + ARMMMUIdx mmu_idx, + int32_t address, int *prot); bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, From 1f2e87e5aba133ecf4c85000a61d5d760176dd6c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 17/90] target/arm: Move get_phys_addr_pmsav7 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-8-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 194 +------------------------------------------- target/arm/ptw.c | 190 +++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 +-- 3 files changed, 198 insertions(+), 196 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d4f7c05625c6..2ebaf694075a 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,8 +11678,7 @@ bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, return true; } -static bool pmsav7_use_background_region(ARMCPU *cpu, - ARMMMUIdx mmu_idx, bool is_user) +bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { /* Return true if we should use the default memory map as a * "background" region if there are no hits against any MPU regions. @@ -11698,14 +11697,14 @@ static bool pmsav7_use_background_region(ARMCPU *cpu, } } -static inline bool m_is_ppb_region(CPUARMState *env, uint32_t address) +bool m_is_ppb_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ return arm_feature(env, ARM_FEATURE_M) && extract32(address, 20, 12) == 0xe00; } -static inline bool m_is_system_region(CPUARMState *env, uint32_t address) +bool m_is_system_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile system region * 0xe0000000 - 0xffffffff @@ -11713,193 +11712,6 @@ static inline bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = env_archcpu(env); - int n; - bool is_user = regime_is_user(env, mmu_idx); - - *phys_ptr = address; - *page_size = TARGET_PAGE_SIZE; - *prot = 0; - - if (regime_translation_disabled(env, mmu_idx) || - m_is_ppb_region(env, address)) { - /* MPU disabled or M profile PPB access: use default memory map. - * The other case which uses the default memory map in the - * v7M ARM ARM pseudocode is exception vector reads from the vector - * table. In QEMU those accesses are done in arm_v7m_load_vector(), - * which always does a direct read using address_space_ldl(), rather - * than going via this function, so we don't need to check that here. - */ - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { /* MPU enabled */ - for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { - /* region search */ - uint32_t base = env->pmsav7.drbar[n]; - uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5); - uint32_t rmask; - bool srdis = false; - - if (!(env->pmsav7.drsr[n] & 0x1)) { - continue; - } - - if (!rsize) { - qemu_log_mask(LOG_GUEST_ERROR, - "DRSR[%d]: Rsize field cannot be 0\n", n); - continue; - } - rsize++; - rmask = (1ull << rsize) - 1; - - if (base & rmask) { - qemu_log_mask(LOG_GUEST_ERROR, - "DRBAR[%d]: 0x%" PRIx32 " misaligned " - "to DRSR region size, mask = 0x%" PRIx32 "\n", - n, base, rmask); - continue; - } - - if (address < base || address > base + rmask) { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result in - * incorrect TLB hits for subsequent accesses to addresses that - * are in this MPU region. - */ - if (ranges_overlap(base, rmask, - address & TARGET_PAGE_MASK, - TARGET_PAGE_SIZE)) { - *page_size = 1; - } - continue; - } - - /* Region matched */ - - if (rsize >= 8) { /* no subregions for regions < 256 bytes */ - int i, snd; - uint32_t srdis_mask; - - rsize -= 3; /* sub region size (power of 2) */ - snd = ((address - base) >> rsize) & 0x7; - srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1); - - srdis_mask = srdis ? 0x3 : 0x0; - for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) { - /* This will check in groups of 2, 4 and then 8, whether - * the subregion bits are consistent. rsize is incremented - * back up to give the region size, considering consistent - * adjacent subregions as one region. Stop testing if rsize - * is already big enough for an entire QEMU page. - */ - int snd_rounded = snd & ~(i - 1); - uint32_t srdis_multi = extract32(env->pmsav7.drsr[n], - snd_rounded + 8, i); - if (srdis_mask ^ srdis_multi) { - break; - } - srdis_mask = (srdis_mask << i) | srdis_mask; - rsize++; - } - } - if (srdis) { - continue; - } - if (rsize < TARGET_PAGE_BITS) { - *page_size = 1 << rsize; - } - break; - } - - if (n == -1) { /* no hits */ - if (!pmsav7_use_background_region(cpu, mmu_idx, is_user)) { - /* background fault */ - fi->type = ARMFault_Background; - return true; - } - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { /* a MPU hit! */ - uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3); - uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1); - - if (m_is_system_region(env, address)) { - /* System space is always execute never */ - xn = 1; - } - - if (is_user) { /* User mode AP bit decoding */ - switch (ap) { - case 0: - case 1: - case 5: - break; /* no access */ - case 3: - *prot |= PAGE_WRITE; - /* fall through */ - case 2: - case 6: - *prot |= PAGE_READ | PAGE_EXEC; - break; - case 7: - /* for v7M, same as 6; for R profile a reserved value */ - if (arm_feature(env, ARM_FEATURE_M)) { - *prot |= PAGE_READ | PAGE_EXEC; - break; - } - /* fall through */ - default: - qemu_log_mask(LOG_GUEST_ERROR, - "DRACR[%d]: Bad value for AP bits: 0x%" - PRIx32 "\n", n, ap); - } - } else { /* Priv. mode AP bits decoding */ - switch (ap) { - case 0: - break; /* no access */ - case 1: - case 2: - case 3: - *prot |= PAGE_WRITE; - /* fall through */ - case 5: - case 6: - *prot |= PAGE_READ | PAGE_EXEC; - break; - case 7: - /* for v7M, same as 6; for R profile a reserved value */ - if (arm_feature(env, ARM_FEATURE_M)) { - *prot |= PAGE_READ | PAGE_EXEC; - break; - } - /* fall through */ - default: - qemu_log_mask(LOG_GUEST_ERROR, - "DRACR[%d]: Bad value for AP bits: 0x%" - PRIx32 "\n", n, ap); - } - } - - /* execute never */ - if (xn) { - *prot &= ~PAGE_EXEC; - } - } - } - - fi->type = ARMFault_Permission; - fi->level = 1; - return !(*prot & (1 << access_type)); -} - static bool v8m_is_sau_exempt(CPUARMState *env, uint32_t address, MMUAccessType access_type) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 74650c6c5253..27715dbfa8c5 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -8,6 +8,7 @@ #include "qemu/osdep.h" #include "qemu/log.h" +#include "qemu/range.h" #include "cpu.h" #include "internals.h" #include "ptw.h" @@ -415,6 +416,195 @@ void get_phys_addr_pmsav7_default(CPUARMState *env, } } +static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, int *prot, + target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = env_archcpu(env); + int n; + bool is_user = regime_is_user(env, mmu_idx); + + *phys_ptr = address; + *page_size = TARGET_PAGE_SIZE; + *prot = 0; + + if (regime_translation_disabled(env, mmu_idx) || + m_is_ppb_region(env, address)) { + /* + * MPU disabled or M profile PPB access: use default memory map. + * The other case which uses the default memory map in the + * v7M ARM ARM pseudocode is exception vector reads from the vector + * table. In QEMU those accesses are done in arm_v7m_load_vector(), + * which always does a direct read using address_space_ldl(), rather + * than going via this function, so we don't need to check that here. + */ + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { /* MPU enabled */ + for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { + /* region search */ + uint32_t base = env->pmsav7.drbar[n]; + uint32_t rsize = extract32(env->pmsav7.drsr[n], 1, 5); + uint32_t rmask; + bool srdis = false; + + if (!(env->pmsav7.drsr[n] & 0x1)) { + continue; + } + + if (!rsize) { + qemu_log_mask(LOG_GUEST_ERROR, + "DRSR[%d]: Rsize field cannot be 0\n", n); + continue; + } + rsize++; + rmask = (1ull << rsize) - 1; + + if (base & rmask) { + qemu_log_mask(LOG_GUEST_ERROR, + "DRBAR[%d]: 0x%" PRIx32 " misaligned " + "to DRSR region size, mask = 0x%" PRIx32 "\n", + n, base, rmask); + continue; + } + + if (address < base || address > base + rmask) { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result in + * incorrect TLB hits for subsequent accesses to addresses that + * are in this MPU region. + */ + if (ranges_overlap(base, rmask, + address & TARGET_PAGE_MASK, + TARGET_PAGE_SIZE)) { + *page_size = 1; + } + continue; + } + + /* Region matched */ + + if (rsize >= 8) { /* no subregions for regions < 256 bytes */ + int i, snd; + uint32_t srdis_mask; + + rsize -= 3; /* sub region size (power of 2) */ + snd = ((address - base) >> rsize) & 0x7; + srdis = extract32(env->pmsav7.drsr[n], snd + 8, 1); + + srdis_mask = srdis ? 0x3 : 0x0; + for (i = 2; i <= 8 && rsize < TARGET_PAGE_BITS; i *= 2) { + /* + * This will check in groups of 2, 4 and then 8, whether + * the subregion bits are consistent. rsize is incremented + * back up to give the region size, considering consistent + * adjacent subregions as one region. Stop testing if rsize + * is already big enough for an entire QEMU page. + */ + int snd_rounded = snd & ~(i - 1); + uint32_t srdis_multi = extract32(env->pmsav7.drsr[n], + snd_rounded + 8, i); + if (srdis_mask ^ srdis_multi) { + break; + } + srdis_mask = (srdis_mask << i) | srdis_mask; + rsize++; + } + } + if (srdis) { + continue; + } + if (rsize < TARGET_PAGE_BITS) { + *page_size = 1 << rsize; + } + break; + } + + if (n == -1) { /* no hits */ + if (!pmsav7_use_background_region(cpu, mmu_idx, is_user)) { + /* background fault */ + fi->type = ARMFault_Background; + return true; + } + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { /* a MPU hit! */ + uint32_t ap = extract32(env->pmsav7.dracr[n], 8, 3); + uint32_t xn = extract32(env->pmsav7.dracr[n], 12, 1); + + if (m_is_system_region(env, address)) { + /* System space is always execute never */ + xn = 1; + } + + if (is_user) { /* User mode AP bit decoding */ + switch (ap) { + case 0: + case 1: + case 5: + break; /* no access */ + case 3: + *prot |= PAGE_WRITE; + /* fall through */ + case 2: + case 6: + *prot |= PAGE_READ | PAGE_EXEC; + break; + case 7: + /* for v7M, same as 6; for R profile a reserved value */ + if (arm_feature(env, ARM_FEATURE_M)) { + *prot |= PAGE_READ | PAGE_EXEC; + break; + } + /* fall through */ + default: + qemu_log_mask(LOG_GUEST_ERROR, + "DRACR[%d]: Bad value for AP bits: 0x%" + PRIx32 "\n", n, ap); + } + } else { /* Priv. mode AP bits decoding */ + switch (ap) { + case 0: + break; /* no access */ + case 1: + case 2: + case 3: + *prot |= PAGE_WRITE; + /* fall through */ + case 5: + case 6: + *prot |= PAGE_READ | PAGE_EXEC; + break; + case 7: + /* for v7M, same as 6; for R profile a reserved value */ + if (arm_feature(env, ARM_FEATURE_M)) { + *prot |= PAGE_READ | PAGE_EXEC; + break; + } + /* fall through */ + default: + qemu_log_mask(LOG_GUEST_ERROR, + "DRACR[%d]: Bad value for AP bits: 0x%" + PRIx32 "\n", n, ap); + } + } + + /* execute never */ + if (xn) { + *prot &= ~PAGE_EXEC; + } + } + } + + fi->type = ARMFault_Permission; + fi->level = 1; + return !(*prot & (1 << access_type)); +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d6e3fee15230..d24b7c263a85 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,14 +33,14 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +bool m_is_ppb_region(CPUARMState *env, uint32_t address); +bool m_is_system_region(CPUARMState *env, uint32_t address); + void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, int32_t address, int *prot); -bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, int *prot, - target_ulong *page_size, - ARMMMUFaultInfo *fi); +bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); + bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, From 730d5c31d8872e94d08877ad62f44cc49bc01c90 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:49 +0100 Subject: [PATCH 18/90] target/arm: Move get_phys_addr_pmsav8 to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-9-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 75 ------------------------------------------- target/arm/ptw.c | 77 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 5 --- 3 files changed, 77 insertions(+), 80 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 2ebaf694075a..44997fd179d4 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11970,81 +11970,6 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } - -bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi) -{ - uint32_t secure = regime_is_secure(env, mmu_idx); - V8M_SAttributes sattrs = {}; - bool ret; - bool mpu_is_subpage; - - if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { - v8m_security_lookup(env, address, access_type, mmu_idx, &sattrs); - if (access_type == MMU_INST_FETCH) { - /* Instruction fetches always use the MMU bank and the - * transaction attribute determined by the fetch address, - * regardless of CPU state. This is painful for QEMU - * to handle, because it would mean we need to encode - * into the mmu_idx not just the (user, negpri) information - * for the current security state but also that for the - * other security state, which would balloon the number - * of mmu_idx values needed alarmingly. - * Fortunately we can avoid this because it's not actually - * possible to arbitrarily execute code from memory with - * the wrong security attribute: it will always generate - * an exception of some kind or another, apart from the - * special case of an NS CPU executing an SG instruction - * in S&NSC memory. So we always just fail the translation - * here and sort things out in the exception handler - * (including possibly emulating an SG instruction). - */ - if (sattrs.ns != !secure) { - if (sattrs.nsc) { - fi->type = ARMFault_QEMU_NSCExec; - } else { - fi->type = ARMFault_QEMU_SFault; - } - *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; - *phys_ptr = address; - *prot = 0; - return true; - } - } else { - /* For data accesses we always use the MMU bank indicated - * by the current CPU state, but the security attributes - * might downgrade a secure access to nonsecure. - */ - if (sattrs.ns) { - txattrs->secure = false; - } else if (!secure) { - /* NS access to S memory must fault. - * Architecturally we should first check whether the - * MPU information for this address indicates that we - * are doing an unaligned access to Device memory, which - * should generate a UsageFault instead. QEMU does not - * currently check for that kind of unaligned access though. - * If we added it we would need to do so as a special case - * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt(). - */ - fi->type = ARMFault_QEMU_SFault; - *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; - *phys_ptr = address; - *prot = 0; - return true; - } - } - } - - ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, phys_ptr, - txattrs, prot, &mpu_is_subpage, fi, NULL); - *page_size = sattrs.subpage || mpu_is_subpage ? 1 : TARGET_PAGE_SIZE; - return ret; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 27715dbfa8c5..28caa7a7ae05 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -605,6 +605,83 @@ static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, target_ulong *page_size, + ARMMMUFaultInfo *fi) +{ + uint32_t secure = regime_is_secure(env, mmu_idx); + V8M_SAttributes sattrs = {}; + bool ret; + bool mpu_is_subpage; + + if (arm_feature(env, ARM_FEATURE_M_SECURITY)) { + v8m_security_lookup(env, address, access_type, mmu_idx, &sattrs); + if (access_type == MMU_INST_FETCH) { + /* + * Instruction fetches always use the MMU bank and the + * transaction attribute determined by the fetch address, + * regardless of CPU state. This is painful for QEMU + * to handle, because it would mean we need to encode + * into the mmu_idx not just the (user, negpri) information + * for the current security state but also that for the + * other security state, which would balloon the number + * of mmu_idx values needed alarmingly. + * Fortunately we can avoid this because it's not actually + * possible to arbitrarily execute code from memory with + * the wrong security attribute: it will always generate + * an exception of some kind or another, apart from the + * special case of an NS CPU executing an SG instruction + * in S&NSC memory. So we always just fail the translation + * here and sort things out in the exception handler + * (including possibly emulating an SG instruction). + */ + if (sattrs.ns != !secure) { + if (sattrs.nsc) { + fi->type = ARMFault_QEMU_NSCExec; + } else { + fi->type = ARMFault_QEMU_SFault; + } + *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; + *phys_ptr = address; + *prot = 0; + return true; + } + } else { + /* + * For data accesses we always use the MMU bank indicated + * by the current CPU state, but the security attributes + * might downgrade a secure access to nonsecure. + */ + if (sattrs.ns) { + txattrs->secure = false; + } else if (!secure) { + /* + * NS access to S memory must fault. + * Architecturally we should first check whether the + * MPU information for this address indicates that we + * are doing an unaligned access to Device memory, which + * should generate a UsageFault instead. QEMU does not + * currently check for that kind of unaligned access though. + * If we added it we would need to do so as a special case + * for M_FAKE_FSR_SFAULT in arm_v7m_cpu_do_interrupt(). + */ + fi->type = ARMFault_QEMU_SFault; + *page_size = sattrs.subpage ? 1 : TARGET_PAGE_SIZE; + *phys_ptr = address; + *prot = 0; + return true; + } + } + } + + ret = pmsav8_mpu_lookup(env, address, access_type, mmu_idx, phys_ptr, + txattrs, prot, &mpu_is_subpage, fi, NULL); + *page_size = sattrs.subpage || mpu_is_subpage ? 1 : TARGET_PAGE_SIZE; + return ret; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d24b7c263a85..d569507951f5 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -41,11 +41,6 @@ void get_phys_addr_pmsav7_default(CPUARMState *env, int32_t address, int *prot); bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); -bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, target_ulong *page_size, - ARMMMUFaultInfo *fi); bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From fedbaa0503d2c56d4ef58ca9a08121e2af7f5e54 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 19/90] target/arm: Move pmsav8_mpu_lookup to ptw.c This is the final user of get_phys_addr_pmsav7_default within helper.c, so make it static within ptw.c. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-10-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 136 ----------------------------------------- target/arm/ptw.c | 146 +++++++++++++++++++++++++++++++++++++++++++- target/arm/ptw.h | 3 - 3 files changed, 143 insertions(+), 142 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 44997fd179d4..cb23413d8e5e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11834,142 +11834,6 @@ void v8m_security_lookup(CPUARMState *env, uint32_t address, } } -bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - hwaddr *phys_ptr, MemTxAttrs *txattrs, - int *prot, bool *is_subpage, - ARMMMUFaultInfo *fi, uint32_t *mregion) -{ - /* Perform a PMSAv8 MPU lookup (without also doing the SAU check - * that a full phys-to-virt translation does). - * mregion is (if not NULL) set to the region number which matched, - * or -1 if no region number is returned (MPU off, address did not - * hit a region, address hit in multiple regions). - * We set is_subpage to true if the region hit doesn't cover the - * entire TARGET_PAGE the address is within. - */ - ARMCPU *cpu = env_archcpu(env); - bool is_user = regime_is_user(env, mmu_idx); - uint32_t secure = regime_is_secure(env, mmu_idx); - int n; - int matchregion = -1; - bool hit = false; - uint32_t addr_page_base = address & TARGET_PAGE_MASK; - uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); - - *is_subpage = false; - *phys_ptr = address; - *prot = 0; - if (mregion) { - *mregion = -1; - } - - /* Unlike the ARM ARM pseudocode, we don't need to check whether this - * was an exception vector read from the vector table (which is always - * done using the default system address map), because those accesses - * are done in arm_v7m_load_vector(), which always does a direct - * read using address_space_ldl(), rather than going via this function. - */ - if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */ - hit = true; - } else if (m_is_ppb_region(env, address)) { - hit = true; - } else { - if (pmsav7_use_background_region(cpu, mmu_idx, is_user)) { - hit = true; - } - - for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { - /* region search */ - /* Note that the base address is bits [31:5] from the register - * with bits [4:0] all zeroes, but the limit address is bits - * [31:5] from the register with bits [4:0] all ones. - */ - uint32_t base = env->pmsav8.rbar[secure][n] & ~0x1f; - uint32_t limit = env->pmsav8.rlar[secure][n] | 0x1f; - - if (!(env->pmsav8.rlar[secure][n] & 0x1)) { - /* Region disabled */ - continue; - } - - if (address < base || address > limit) { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result in - * incorrect TLB hits for subsequent accesses to addresses that - * are in this MPU region. - */ - if (limit >= base && - ranges_overlap(base, limit - base + 1, - addr_page_base, - TARGET_PAGE_SIZE)) { - *is_subpage = true; - } - continue; - } - - if (base > addr_page_base || limit < addr_page_limit) { - *is_subpage = true; - } - - if (matchregion != -1) { - /* Multiple regions match -- always a failure (unlike - * PMSAv7 where highest-numbered-region wins) - */ - fi->type = ARMFault_Permission; - fi->level = 1; - return true; - } - - matchregion = n; - hit = true; - } - } - - if (!hit) { - /* background fault */ - fi->type = ARMFault_Background; - return true; - } - - if (matchregion == -1) { - /* hit using the background region */ - get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); - } else { - uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2); - uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1); - bool pxn = false; - - if (arm_feature(env, ARM_FEATURE_V8_1M)) { - pxn = extract32(env->pmsav8.rlar[secure][matchregion], 4, 1); - } - - if (m_is_system_region(env, address)) { - /* System space is always execute never */ - xn = 1; - } - - *prot = simple_ap_to_rw_prot(env, mmu_idx, ap); - if (*prot && !xn && !(pxn && !is_user)) { - *prot |= PAGE_EXEC; - } - /* We don't need to look the attribute up in the MAIR0/MAIR1 - * registers because that only tells us about cacheability. - */ - if (mregion) { - *mregion = matchregion; - } - } - - fi->type = ARMFault_Permission; - fi->level = 1; - return !(*prot & (1 << access_type)); -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 28caa7a7ae05..989e783cce9d 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -375,9 +375,8 @@ static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, return false; } -void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot) +static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, + int32_t address, int *prot) { if (!arm_feature(env, ARM_FEATURE_M)) { *prot = PAGE_READ | PAGE_WRITE; @@ -605,6 +604,147 @@ static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + hwaddr *phys_ptr, MemTxAttrs *txattrs, + int *prot, bool *is_subpage, + ARMMMUFaultInfo *fi, uint32_t *mregion) +{ + /* + * Perform a PMSAv8 MPU lookup (without also doing the SAU check + * that a full phys-to-virt translation does). + * mregion is (if not NULL) set to the region number which matched, + * or -1 if no region number is returned (MPU off, address did not + * hit a region, address hit in multiple regions). + * We set is_subpage to true if the region hit doesn't cover the + * entire TARGET_PAGE the address is within. + */ + ARMCPU *cpu = env_archcpu(env); + bool is_user = regime_is_user(env, mmu_idx); + uint32_t secure = regime_is_secure(env, mmu_idx); + int n; + int matchregion = -1; + bool hit = false; + uint32_t addr_page_base = address & TARGET_PAGE_MASK; + uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); + + *is_subpage = false; + *phys_ptr = address; + *prot = 0; + if (mregion) { + *mregion = -1; + } + + /* + * Unlike the ARM ARM pseudocode, we don't need to check whether this + * was an exception vector read from the vector table (which is always + * done using the default system address map), because those accesses + * are done in arm_v7m_load_vector(), which always does a direct + * read using address_space_ldl(), rather than going via this function. + */ + if (regime_translation_disabled(env, mmu_idx)) { /* MPU disabled */ + hit = true; + } else if (m_is_ppb_region(env, address)) { + hit = true; + } else { + if (pmsav7_use_background_region(cpu, mmu_idx, is_user)) { + hit = true; + } + + for (n = (int)cpu->pmsav7_dregion - 1; n >= 0; n--) { + /* region search */ + /* + * Note that the base address is bits [31:5] from the register + * with bits [4:0] all zeroes, but the limit address is bits + * [31:5] from the register with bits [4:0] all ones. + */ + uint32_t base = env->pmsav8.rbar[secure][n] & ~0x1f; + uint32_t limit = env->pmsav8.rlar[secure][n] | 0x1f; + + if (!(env->pmsav8.rlar[secure][n] & 0x1)) { + /* Region disabled */ + continue; + } + + if (address < base || address > limit) { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result in + * incorrect TLB hits for subsequent accesses to addresses that + * are in this MPU region. + */ + if (limit >= base && + ranges_overlap(base, limit - base + 1, + addr_page_base, + TARGET_PAGE_SIZE)) { + *is_subpage = true; + } + continue; + } + + if (base > addr_page_base || limit < addr_page_limit) { + *is_subpage = true; + } + + if (matchregion != -1) { + /* + * Multiple regions match -- always a failure (unlike + * PMSAv7 where highest-numbered-region wins) + */ + fi->type = ARMFault_Permission; + fi->level = 1; + return true; + } + + matchregion = n; + hit = true; + } + } + + if (!hit) { + /* background fault */ + fi->type = ARMFault_Background; + return true; + } + + if (matchregion == -1) { + /* hit using the background region */ + get_phys_addr_pmsav7_default(env, mmu_idx, address, prot); + } else { + uint32_t ap = extract32(env->pmsav8.rbar[secure][matchregion], 1, 2); + uint32_t xn = extract32(env->pmsav8.rbar[secure][matchregion], 0, 1); + bool pxn = false; + + if (arm_feature(env, ARM_FEATURE_V8_1M)) { + pxn = extract32(env->pmsav8.rlar[secure][matchregion], 4, 1); + } + + if (m_is_system_region(env, address)) { + /* System space is always execute never */ + xn = 1; + } + + *prot = simple_ap_to_rw_prot(env, mmu_idx, ap); + if (*prot && !xn && !(pxn && !is_user)) { + *prot |= PAGE_EXEC; + } + /* + * We don't need to look the attribute up in the MAIR0/MAIR1 + * registers because that only tells us about cacheability. + */ + if (mregion) { + *mregion = matchregion; + } + } + + fi->type = ARMFault_Permission; + fi->level = 1; + return !(*prot & (1 << access_type)); +} + static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d569507951f5..8d2e23971478 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -36,9 +36,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) bool m_is_ppb_region(CPUARMState *env, uint32_t address); bool m_is_system_region(CPUARMState *env, uint32_t address); -void get_phys_addr_pmsav7_default(CPUARMState *env, - ARMMMUIdx mmu_idx, - int32_t address, int *prot); bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, From c8e436c9f72bca95e37a155d6ead42d55f74eed2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 20/90] target/arm: Move pmsav7_use_background_region to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-11-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 19 ------------------- target/arm/ptw.c | 21 +++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index cb23413d8e5e..62e48f0925ce 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11678,25 +11678,6 @@ bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, return true; } -bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) -{ - /* Return true if we should use the default memory map as a - * "background" region if there are no hits against any MPU regions. - */ - CPUARMState *env = &cpu->env; - - if (is_user) { - return false; - } - - if (arm_feature(env, ARM_FEATURE_M)) { - return env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] - & R_V7M_MPU_CTRL_PRIVDEFENA_MASK; - } else { - return regime_sctlr(env, mmu_idx) & SCTLR_BR; - } -} - bool m_is_ppb_region(CPUARMState *env, uint32_t address) { /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 989e783cce9d..b82638b5a06f 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -415,6 +415,27 @@ static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, } } +static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, + bool is_user) +{ + /* + * Return true if we should use the default memory map as a + * "background" region if there are no hits against any MPU regions. + */ + CPUARMState *env = &cpu->env; + + if (is_user) { + return false; + } + + if (arm_feature(env, ARM_FEATURE_M)) { + return env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] + & R_V7M_MPU_CTRL_PRIVDEFENA_MASK; + } else { + return regime_sctlr(env, mmu_idx) & SCTLR_BR; + } +} + static bool get_phys_addr_pmsav7(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 8d2e23971478..d2d271190829 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -36,8 +36,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) bool m_is_ppb_region(CPUARMState *env, uint32_t address); bool m_is_system_region(CPUARMState *env, uint32_t address); -bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user); - bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 2c1f429df38822ce6955224d784f1567f8af219f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 21/90] target/arm: Move v8m_security_lookup to ptw.c This function has one private helper, v8m_is_sau_exempt, so move that at the same time. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-12-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 123 ------------------------------------------ target/arm/ptw.c | 126 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+), 123 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 62e48f0925ce..d6a749ad0ed7 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -9,7 +9,6 @@ #include "qemu/osdep.h" #include "qemu/units.h" #include "qemu/log.h" -#include "target/arm/idau.h" #include "trace.h" #include "cpu.h" #include "internals.h" @@ -11693,128 +11692,6 @@ bool m_is_system_region(CPUARMState *env, uint32_t address) return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; } -static bool v8m_is_sau_exempt(CPUARMState *env, - uint32_t address, MMUAccessType access_type) -{ - /* The architecture specifies that certain address ranges are - * exempt from v8M SAU/IDAU checks. - */ - return - (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) || - (address >= 0xe0000000 && address <= 0xe0002fff) || - (address >= 0xe000e000 && address <= 0xe000efff) || - (address >= 0xe002e000 && address <= 0xe002efff) || - (address >= 0xe0040000 && address <= 0xe0041fff) || - (address >= 0xe00ff000 && address <= 0xe00fffff); -} - -void v8m_security_lookup(CPUARMState *env, uint32_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - V8M_SAttributes *sattrs) -{ - /* Look up the security attributes for this address. Compare the - * pseudocode SecurityCheck() function. - * We assume the caller has zero-initialized *sattrs. - */ - ARMCPU *cpu = env_archcpu(env); - int r; - bool idau_exempt = false, idau_ns = true, idau_nsc = true; - int idau_region = IREGION_NOTVALID; - uint32_t addr_page_base = address & TARGET_PAGE_MASK; - uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); - - if (cpu->idau) { - IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau); - IDAUInterface *ii = IDAU_INTERFACE(cpu->idau); - - iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns, - &idau_nsc); - } - - if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) { - /* 0xf0000000..0xffffffff is always S for insn fetches */ - return; - } - - if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) { - sattrs->ns = !regime_is_secure(env, mmu_idx); - return; - } - - if (idau_region != IREGION_NOTVALID) { - sattrs->irvalid = true; - sattrs->iregion = idau_region; - } - - switch (env->sau.ctrl & 3) { - case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */ - break; - case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */ - sattrs->ns = true; - break; - default: /* SAU.ENABLE == 1 */ - for (r = 0; r < cpu->sau_sregion; r++) { - if (env->sau.rlar[r] & 1) { - uint32_t base = env->sau.rbar[r] & ~0x1f; - uint32_t limit = env->sau.rlar[r] | 0x1f; - - if (base <= address && limit >= address) { - if (base > addr_page_base || limit < addr_page_limit) { - sattrs->subpage = true; - } - if (sattrs->srvalid) { - /* If we hit in more than one region then we must report - * as Secure, not NS-Callable, with no valid region - * number info. - */ - sattrs->ns = false; - sattrs->nsc = false; - sattrs->sregion = 0; - sattrs->srvalid = false; - break; - } else { - if (env->sau.rlar[r] & 2) { - sattrs->nsc = true; - } else { - sattrs->ns = true; - } - sattrs->srvalid = true; - sattrs->sregion = r; - } - } else { - /* - * Address not in this region. We must check whether the - * region covers addresses in the same page as our address. - * In that case we must not report a size that covers the - * whole page for a subsequent hit against a different MPU - * region or the background region, because it would result - * in incorrect TLB hits for subsequent accesses to - * addresses that are in this MPU region. - */ - if (limit >= base && - ranges_overlap(base, limit - base + 1, - addr_page_base, - TARGET_PAGE_SIZE)) { - sattrs->subpage = true; - } - } - } - } - break; - } - - /* - * The IDAU will override the SAU lookup results if it specifies - * higher security than the SAU does. - */ - if (!idau_ns) { - if (sattrs->ns || (!idau_nsc && sattrs->nsc)) { - sattrs->ns = false; - sattrs->nsc = idau_nsc; - } - } -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index b82638b5a06f..c15fba43c311 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -11,6 +11,7 @@ #include "qemu/range.h" #include "cpu.h" #include "internals.h" +#include "idau.h" #include "ptw.h" @@ -766,6 +767,131 @@ bool pmsav8_mpu_lookup(CPUARMState *env, uint32_t address, return !(*prot & (1 << access_type)); } +static bool v8m_is_sau_exempt(CPUARMState *env, + uint32_t address, MMUAccessType access_type) +{ + /* + * The architecture specifies that certain address ranges are + * exempt from v8M SAU/IDAU checks. + */ + return + (access_type == MMU_INST_FETCH && m_is_system_region(env, address)) || + (address >= 0xe0000000 && address <= 0xe0002fff) || + (address >= 0xe000e000 && address <= 0xe000efff) || + (address >= 0xe002e000 && address <= 0xe002efff) || + (address >= 0xe0040000 && address <= 0xe0041fff) || + (address >= 0xe00ff000 && address <= 0xe00fffff); +} + +void v8m_security_lookup(CPUARMState *env, uint32_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + V8M_SAttributes *sattrs) +{ + /* + * Look up the security attributes for this address. Compare the + * pseudocode SecurityCheck() function. + * We assume the caller has zero-initialized *sattrs. + */ + ARMCPU *cpu = env_archcpu(env); + int r; + bool idau_exempt = false, idau_ns = true, idau_nsc = true; + int idau_region = IREGION_NOTVALID; + uint32_t addr_page_base = address & TARGET_PAGE_MASK; + uint32_t addr_page_limit = addr_page_base + (TARGET_PAGE_SIZE - 1); + + if (cpu->idau) { + IDAUInterfaceClass *iic = IDAU_INTERFACE_GET_CLASS(cpu->idau); + IDAUInterface *ii = IDAU_INTERFACE(cpu->idau); + + iic->check(ii, address, &idau_region, &idau_exempt, &idau_ns, + &idau_nsc); + } + + if (access_type == MMU_INST_FETCH && extract32(address, 28, 4) == 0xf) { + /* 0xf0000000..0xffffffff is always S for insn fetches */ + return; + } + + if (idau_exempt || v8m_is_sau_exempt(env, address, access_type)) { + sattrs->ns = !regime_is_secure(env, mmu_idx); + return; + } + + if (idau_region != IREGION_NOTVALID) { + sattrs->irvalid = true; + sattrs->iregion = idau_region; + } + + switch (env->sau.ctrl & 3) { + case 0: /* SAU.ENABLE == 0, SAU.ALLNS == 0 */ + break; + case 2: /* SAU.ENABLE == 0, SAU.ALLNS == 1 */ + sattrs->ns = true; + break; + default: /* SAU.ENABLE == 1 */ + for (r = 0; r < cpu->sau_sregion; r++) { + if (env->sau.rlar[r] & 1) { + uint32_t base = env->sau.rbar[r] & ~0x1f; + uint32_t limit = env->sau.rlar[r] | 0x1f; + + if (base <= address && limit >= address) { + if (base > addr_page_base || limit < addr_page_limit) { + sattrs->subpage = true; + } + if (sattrs->srvalid) { + /* + * If we hit in more than one region then we must report + * as Secure, not NS-Callable, with no valid region + * number info. + */ + sattrs->ns = false; + sattrs->nsc = false; + sattrs->sregion = 0; + sattrs->srvalid = false; + break; + } else { + if (env->sau.rlar[r] & 2) { + sattrs->nsc = true; + } else { + sattrs->ns = true; + } + sattrs->srvalid = true; + sattrs->sregion = r; + } + } else { + /* + * Address not in this region. We must check whether the + * region covers addresses in the same page as our address. + * In that case we must not report a size that covers the + * whole page for a subsequent hit against a different MPU + * region or the background region, because it would result + * in incorrect TLB hits for subsequent accesses to + * addresses that are in this MPU region. + */ + if (limit >= base && + ranges_overlap(base, limit - base + 1, + addr_page_base, + TARGET_PAGE_SIZE)) { + sattrs->subpage = true; + } + } + } + } + break; + } + + /* + * The IDAU will override the SAU lookup results if it specifies + * higher security than the SAU does. + */ + if (!idau_ns) { + if (sattrs->ns || (!idau_nsc && sattrs->nsc)) { + sattrs->ns = false; + sattrs->nsc = idau_nsc; + } + } +} + static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, MemTxAttrs *txattrs, From 47ff5ba9d08c2c769794a59b88f1c19ada004730 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:50 +0100 Subject: [PATCH 22/90] target/arm: Move m_is_{ppb,system}_region to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-13-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 15 --------------- target/arm/ptw.c | 16 ++++++++++++++++ target/arm/ptw.h | 3 --- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d6a749ad0ed7..d2ef12346b6e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11677,21 +11677,6 @@ bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, return true; } -bool m_is_ppb_region(CPUARMState *env, uint32_t address) -{ - /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ - return arm_feature(env, ARM_FEATURE_M) && - extract32(address, 20, 12) == 0xe00; -} - -bool m_is_system_region(CPUARMState *env, uint32_t address) -{ - /* True if address is in the M profile system region - * 0xe0000000 - 0xffffffff - */ - return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; -} - /* Combine either inner or outer cacheability attributes for normal * memory, according to table D4-42 and pseudocode procedure * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). diff --git a/target/arm/ptw.c b/target/arm/ptw.c index c15fba43c311..32ba2e5e8bf7 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -416,6 +416,22 @@ static void get_phys_addr_pmsav7_default(CPUARMState *env, ARMMMUIdx mmu_idx, } } +static bool m_is_ppb_region(CPUARMState *env, uint32_t address) +{ + /* True if address is in the M profile PPB region 0xe0000000 - 0xe00fffff */ + return arm_feature(env, ARM_FEATURE_M) && + extract32(address, 20, 12) == 0xe00; +} + +static bool m_is_system_region(CPUARMState *env, uint32_t address) +{ + /* + * True if address is in the M profile system region + * 0xe0000000 - 0xffffffff + */ + return arm_feature(env, ARM_FEATURE_M) && extract32(address, 29, 3) == 0x7; +} + static bool pmsav7_use_background_region(ARMCPU *cpu, ARMMMUIdx mmu_idx, bool is_user) { diff --git a/target/arm/ptw.h b/target/arm/ptw.h index d2d271190829..6c47a5759913 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -33,9 +33,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -bool m_is_ppb_region(CPUARMState *env, uint32_t address); -bool m_is_system_region(CPUARMState *env, uint32_t address); - bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 4c74ab157b056710b043a02c8101c449c179ae11 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 23/90] target/arm: Move get_level1_table_address to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-14-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 26 +------------------------- target/arm/ptw.c | 23 +++++++++++++++++++++++ target/arm/ptw.h | 4 ++-- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d2ef12346b6e..a144cb26413e 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10482,8 +10482,7 @@ static inline bool regime_translation_big_endian(CPUARMState *env, } /* Return the TTBR associated with this translation regime */ -static inline uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, - int ttbrn) +uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { if (mmu_idx == ARMMMUIdx_Stage2) { return env->cp15.vttbr_el2; @@ -10774,29 +10773,6 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address) -{ - /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ - TCR *tcr = regime_tcr(env, mmu_idx); - - if (address & tcr->mask) { - if (tcr->raw_tcr & TTBCR_PD1) { - /* Translation table walk disabled for TTBR1 */ - return false; - } - *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000; - } else { - if (tcr->raw_tcr & TTBCR_PD0) { - /* Translation table walk disabled for TTBR0 */ - return false; - } - *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask; - } - *table |= (address >> 18) & 0x3ffc; - return true; -} - static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 32ba2e5e8bf7..5737a3976b89 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -15,6 +15,29 @@ #include "ptw.h" +static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, + uint32_t *table, uint32_t address) +{ + /* Note that we can only get here for an AArch32 PL0/PL1 lookup */ + TCR *tcr = regime_tcr(env, mmu_idx); + + if (address & tcr->mask) { + if (tcr->raw_tcr & TTBCR_PD1) { + /* Translation table walk disabled for TTBR1 */ + return false; + } + *table = regime_ttbr(env, mmu_idx, 1) & 0xffffc000; + } else { + if (tcr->raw_tcr & TTBCR_PD0) { + /* Translation table walk disabled for TTBR0 */ + return false; + } + *table = regime_ttbr(env, mmu_idx, 0) & tcr->base_mask; + } + *table |= (address >> 18) & 0x3ffc; + return true; +} + static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 6c47a5759913..dd6fb93f336a 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -18,11 +18,11 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); +uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); + ARMCacheAttrs combine_cacheattrs(CPUARMState *env, ARMCacheAttrs s1, ARMCacheAttrs s2); -bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, - uint32_t *table, uint32_t address); int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); int simple_ap_to_rw_prot_is_user(int ap, bool is_user); From 966f4bb7d88554ed74e1006c516ddeef37e523d6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 24/90] target/arm: Move combine_cacheattrs and subroutines to ptw.c There are a handful of helpers for combine_cacheattrs that we can move at the same time as the main entry point. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-15-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 218 ------------------------------------------- target/arm/ptw.c | 221 ++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 - 3 files changed, 221 insertions(+), 221 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index a144cb26413e..dab485e64aee 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10977,36 +10977,6 @@ static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, } return true; } - -/* Translate from the 4-bit stage 2 representation of - * memory attributes (without cache-allocation hints) to - * the 8-bit representation of the stage 1 MAIR registers - * (which includes allocation hints). - * - * ref: shared/translation/attrs/S2AttrDecode() - * .../S2ConvertAttrsHints() - */ -static uint8_t convert_stage2_attrs(CPUARMState *env, uint8_t s2attrs) -{ - uint8_t hiattr = extract32(s2attrs, 2, 2); - uint8_t loattr = extract32(s2attrs, 0, 2); - uint8_t hihint = 0, lohint = 0; - - if (hiattr != 0) { /* normal memory */ - if (arm_hcr_el2_eff(env) & HCR_CD) { /* cache disabled */ - hiattr = loattr = 1; /* non-cacheable */ - } else { - if (hiattr != 1) { /* Write-through or write-back */ - hihint = 3; /* RW allocate */ - } - if (loattr != 1) { /* Write-through or write-back */ - lohint = 3; /* RW allocate */ - } - } - } - - return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint; -} #endif /* !CONFIG_USER_ONLY */ /* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ @@ -11653,194 +11623,6 @@ bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, return true; } -/* Combine either inner or outer cacheability attributes for normal - * memory, according to table D4-42 and pseudocode procedure - * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). - * - * NB: only stage 1 includes allocation hints (RW bits), leading to - * some asymmetry. - */ -static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2) -{ - if (s1 == 4 || s2 == 4) { - /* non-cacheable has precedence */ - return 4; - } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) { - /* stage 1 write-through takes precedence */ - return s1; - } else if (extract32(s2, 2, 2) == 2) { - /* stage 2 write-through takes precedence, but the allocation hint - * is still taken from stage 1 - */ - return (2 << 2) | extract32(s1, 0, 2); - } else { /* write-back */ - return s1; - } -} - -/* - * Combine the memory type and cacheability attributes of - * s1 and s2 for the HCR_EL2.FWB == 0 case, returning the - * combined attributes in MAIR_EL1 format. - */ -static uint8_t combined_attrs_nofwb(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs; - - s2_mair_attrs = convert_stage2_attrs(env, s2.attrs); - - s1lo = extract32(s1.attrs, 0, 4); - s2lo = extract32(s2_mair_attrs, 0, 4); - s1hi = extract32(s1.attrs, 4, 4); - s2hi = extract32(s2_mair_attrs, 4, 4); - - /* Combine memory type and cacheability attributes */ - if (s1hi == 0 || s2hi == 0) { - /* Device has precedence over normal */ - if (s1lo == 0 || s2lo == 0) { - /* nGnRnE has precedence over anything */ - ret_attrs = 0; - } else if (s1lo == 4 || s2lo == 4) { - /* non-Reordering has precedence over Reordering */ - ret_attrs = 4; /* nGnRE */ - } else if (s1lo == 8 || s2lo == 8) { - /* non-Gathering has precedence over Gathering */ - ret_attrs = 8; /* nGRE */ - } else { - ret_attrs = 0xc; /* GRE */ - } - } else { /* Normal memory */ - /* Outer/inner cacheability combine independently */ - ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4 - | combine_cacheattr_nibble(s1lo, s2lo); - } - return ret_attrs; -} - -static uint8_t force_cacheattr_nibble_wb(uint8_t attr) -{ - /* - * Given the 4 bits specifying the outer or inner cacheability - * in MAIR format, return a value specifying Normal Write-Back, - * with the allocation and transient hints taken from the input - * if the input specified some kind of cacheable attribute. - */ - if (attr == 0 || attr == 4) { - /* - * 0 == an UNPREDICTABLE encoding - * 4 == Non-cacheable - * Either way, force Write-Back RW allocate non-transient - */ - return 0xf; - } - /* Change WriteThrough to WriteBack, keep allocation and transient hints */ - return attr | 4; -} - -/* - * Combine the memory type and cacheability attributes of - * s1 and s2 for the HCR_EL2.FWB == 1 case, returning the - * combined attributes in MAIR_EL1 format. - */ -static uint8_t combined_attrs_fwb(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - switch (s2.attrs) { - case 7: - /* Use stage 1 attributes */ - return s1.attrs; - case 6: - /* - * Force Normal Write-Back. Note that if S1 is Normal cacheable - * then we take the allocation hints from it; otherwise it is - * RW allocate, non-transient. - */ - if ((s1.attrs & 0xf0) == 0) { - /* S1 is Device */ - return 0xff; - } - /* Need to check the Inner and Outer nibbles separately */ - return force_cacheattr_nibble_wb(s1.attrs & 0xf) | - force_cacheattr_nibble_wb(s1.attrs >> 4) << 4; - case 5: - /* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */ - if ((s1.attrs & 0xf0) == 0) { - return s1.attrs; - } - return 0x44; - case 0 ... 3: - /* Force Device, of subtype specified by S2 */ - return s2.attrs << 2; - default: - /* - * RESERVED values (including RES0 descriptor bit [5] being nonzero); - * arbitrarily force Device. - */ - return 0; - } -} - -/* Combine S1 and S2 cacheability/shareability attributes, per D4.5.4 - * and CombineS1S2Desc() - * - * @env: CPUARMState - * @s1: Attributes from stage 1 walk - * @s2: Attributes from stage 2 walk - */ -ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2) -{ - ARMCacheAttrs ret; - bool tagged = false; - - assert(s2.is_s2_format && !s1.is_s2_format); - ret.is_s2_format = false; - - if (s1.attrs == 0xf0) { - tagged = true; - s1.attrs = 0xff; - } - - /* Combine shareability attributes (table D4-43) */ - if (s1.shareability == 2 || s2.shareability == 2) { - /* if either are outer-shareable, the result is outer-shareable */ - ret.shareability = 2; - } else if (s1.shareability == 3 || s2.shareability == 3) { - /* if either are inner-shareable, the result is inner-shareable */ - ret.shareability = 3; - } else { - /* both non-shareable */ - ret.shareability = 0; - } - - /* Combine memory type and cacheability attributes */ - if (arm_hcr_el2_eff(env) & HCR_FWB) { - ret.attrs = combined_attrs_fwb(env, s1, s2); - } else { - ret.attrs = combined_attrs_nofwb(env, s1, s2); - } - - /* - * Any location for which the resultant memory type is any - * type of Device memory is always treated as Outer Shareable. - * Any location for which the resultant memory type is Normal - * Inner Non-cacheable, Outer Non-cacheable is always treated - * as Outer Shareable. - * TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC - */ - if ((ret.attrs & 0xf0) == 0 || ret.attrs == 0x44) { - ret.shareability = 2; - } - - /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */ - if (tagged && ret.attrs == 0xff) { - ret.attrs = 0xf0; - } - - return ret; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 5737a3976b89..f2ca2bb8fe12 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -1008,6 +1008,227 @@ static bool get_phys_addr_pmsav8(CPUARMState *env, uint32_t address, return ret; } +/* + * Translate from the 4-bit stage 2 representation of + * memory attributes (without cache-allocation hints) to + * the 8-bit representation of the stage 1 MAIR registers + * (which includes allocation hints). + * + * ref: shared/translation/attrs/S2AttrDecode() + * .../S2ConvertAttrsHints() + */ +static uint8_t convert_stage2_attrs(CPUARMState *env, uint8_t s2attrs) +{ + uint8_t hiattr = extract32(s2attrs, 2, 2); + uint8_t loattr = extract32(s2attrs, 0, 2); + uint8_t hihint = 0, lohint = 0; + + if (hiattr != 0) { /* normal memory */ + if (arm_hcr_el2_eff(env) & HCR_CD) { /* cache disabled */ + hiattr = loattr = 1; /* non-cacheable */ + } else { + if (hiattr != 1) { /* Write-through or write-back */ + hihint = 3; /* RW allocate */ + } + if (loattr != 1) { /* Write-through or write-back */ + lohint = 3; /* RW allocate */ + } + } + } + + return (hiattr << 6) | (hihint << 4) | (loattr << 2) | lohint; +} + +/* + * Combine either inner or outer cacheability attributes for normal + * memory, according to table D4-42 and pseudocode procedure + * CombineS1S2AttrHints() of ARM DDI 0487B.b (the ARMv8 ARM). + * + * NB: only stage 1 includes allocation hints (RW bits), leading to + * some asymmetry. + */ +static uint8_t combine_cacheattr_nibble(uint8_t s1, uint8_t s2) +{ + if (s1 == 4 || s2 == 4) { + /* non-cacheable has precedence */ + return 4; + } else if (extract32(s1, 2, 2) == 0 || extract32(s1, 2, 2) == 2) { + /* stage 1 write-through takes precedence */ + return s1; + } else if (extract32(s2, 2, 2) == 2) { + /* stage 2 write-through takes precedence, but the allocation hint + * is still taken from stage 1 + */ + return (2 << 2) | extract32(s1, 0, 2); + } else { /* write-back */ + return s1; + } +} + +/* + * Combine the memory type and cacheability attributes of + * s1 and s2 for the HCR_EL2.FWB == 0 case, returning the + * combined attributes in MAIR_EL1 format. + */ +static uint8_t combined_attrs_nofwb(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + uint8_t s1lo, s2lo, s1hi, s2hi, s2_mair_attrs, ret_attrs; + + s2_mair_attrs = convert_stage2_attrs(env, s2.attrs); + + s1lo = extract32(s1.attrs, 0, 4); + s2lo = extract32(s2_mair_attrs, 0, 4); + s1hi = extract32(s1.attrs, 4, 4); + s2hi = extract32(s2_mair_attrs, 4, 4); + + /* Combine memory type and cacheability attributes */ + if (s1hi == 0 || s2hi == 0) { + /* Device has precedence over normal */ + if (s1lo == 0 || s2lo == 0) { + /* nGnRnE has precedence over anything */ + ret_attrs = 0; + } else if (s1lo == 4 || s2lo == 4) { + /* non-Reordering has precedence over Reordering */ + ret_attrs = 4; /* nGnRE */ + } else if (s1lo == 8 || s2lo == 8) { + /* non-Gathering has precedence over Gathering */ + ret_attrs = 8; /* nGRE */ + } else { + ret_attrs = 0xc; /* GRE */ + } + } else { /* Normal memory */ + /* Outer/inner cacheability combine independently */ + ret_attrs = combine_cacheattr_nibble(s1hi, s2hi) << 4 + | combine_cacheattr_nibble(s1lo, s2lo); + } + return ret_attrs; +} + +static uint8_t force_cacheattr_nibble_wb(uint8_t attr) +{ + /* + * Given the 4 bits specifying the outer or inner cacheability + * in MAIR format, return a value specifying Normal Write-Back, + * with the allocation and transient hints taken from the input + * if the input specified some kind of cacheable attribute. + */ + if (attr == 0 || attr == 4) { + /* + * 0 == an UNPREDICTABLE encoding + * 4 == Non-cacheable + * Either way, force Write-Back RW allocate non-transient + */ + return 0xf; + } + /* Change WriteThrough to WriteBack, keep allocation and transient hints */ + return attr | 4; +} + +/* + * Combine the memory type and cacheability attributes of + * s1 and s2 for the HCR_EL2.FWB == 1 case, returning the + * combined attributes in MAIR_EL1 format. + */ +static uint8_t combined_attrs_fwb(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + switch (s2.attrs) { + case 7: + /* Use stage 1 attributes */ + return s1.attrs; + case 6: + /* + * Force Normal Write-Back. Note that if S1 is Normal cacheable + * then we take the allocation hints from it; otherwise it is + * RW allocate, non-transient. + */ + if ((s1.attrs & 0xf0) == 0) { + /* S1 is Device */ + return 0xff; + } + /* Need to check the Inner and Outer nibbles separately */ + return force_cacheattr_nibble_wb(s1.attrs & 0xf) | + force_cacheattr_nibble_wb(s1.attrs >> 4) << 4; + case 5: + /* If S1 attrs are Device, use them; otherwise Normal Non-cacheable */ + if ((s1.attrs & 0xf0) == 0) { + return s1.attrs; + } + return 0x44; + case 0 ... 3: + /* Force Device, of subtype specified by S2 */ + return s2.attrs << 2; + default: + /* + * RESERVED values (including RES0 descriptor bit [5] being nonzero); + * arbitrarily force Device. + */ + return 0; + } +} + +/* + * Combine S1 and S2 cacheability/shareability attributes, per D4.5.4 + * and CombineS1S2Desc() + * + * @env: CPUARMState + * @s1: Attributes from stage 1 walk + * @s2: Attributes from stage 2 walk + */ +static ARMCacheAttrs combine_cacheattrs(CPUARMState *env, + ARMCacheAttrs s1, ARMCacheAttrs s2) +{ + ARMCacheAttrs ret; + bool tagged = false; + + assert(s2.is_s2_format && !s1.is_s2_format); + ret.is_s2_format = false; + + if (s1.attrs == 0xf0) { + tagged = true; + s1.attrs = 0xff; + } + + /* Combine shareability attributes (table D4-43) */ + if (s1.shareability == 2 || s2.shareability == 2) { + /* if either are outer-shareable, the result is outer-shareable */ + ret.shareability = 2; + } else if (s1.shareability == 3 || s2.shareability == 3) { + /* if either are inner-shareable, the result is inner-shareable */ + ret.shareability = 3; + } else { + /* both non-shareable */ + ret.shareability = 0; + } + + /* Combine memory type and cacheability attributes */ + if (arm_hcr_el2_eff(env) & HCR_FWB) { + ret.attrs = combined_attrs_fwb(env, s1, s2); + } else { + ret.attrs = combined_attrs_nofwb(env, s1, s2); + } + + /* + * Any location for which the resultant memory type is any + * type of Device memory is always treated as Outer Shareable. + * Any location for which the resultant memory type is Normal + * Inner Non-cacheable, Outer Non-cacheable is always treated + * as Outer Shareable. + * TODO: FEAT_XS adds another value (0x40) also meaning iNCoNC + */ + if ((ret.attrs & 0xf0) == 0 || ret.attrs == 0x44) { + ret.shareability = 2; + } + + /* TODO: CombineS1S2Desc does not consider transient, only WB, RWA. */ + if (tagged && ret.attrs == 0xff) { + ret.attrs = 0xf0; + } + + return ret; +} + /** * get_phys_addr - get the physical address for this virtual address * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index dd6fb93f336a..b2dfe489bbed 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -20,9 +20,6 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); -ARMCacheAttrs combine_cacheattrs(CPUARMState *env, - ARMCacheAttrs s1, ARMCacheAttrs s2); - int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot); int simple_ap_to_rw_prot_is_user(int ap, bool is_user); From 3283222acd7579cdbc338360fa6e611027f3c8dc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 25/90] target/arm: Move get_phys_addr_lpae to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-16-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 416 +------------------------------------------- target/arm/ptw.c | 411 +++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 ++ 3 files changed, 429 insertions(+), 408 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index dab485e64aee..7de815fe9860 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10652,7 +10652,7 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) * @xn: XN (execute-never) bits * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 */ -static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) +int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) { int prot = 0; @@ -10703,8 +10703,8 @@ static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) * @xn: XN (execute-never) bit * @pxn: PXN (privileged execute-never) bit */ -static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn) +int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn) { bool is_user = regime_is_user(env, mmu_idx); int prot_rw, user_rw; @@ -10919,8 +10919,8 @@ uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, * Returns true if the suggested S2 translation parameters are OK and * false otherwise. */ -static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize) +bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize) { const int grainsize = stride + 3; int startsizecheck; @@ -10980,7 +10980,7 @@ static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, #endif /* !CONFIG_USER_ONLY */ /* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ -static const uint8_t pamax_map[] = { +const uint8_t pamax_map[] = { [0] = 32, [1] = 36, [2] = 40, @@ -11159,8 +11159,8 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, } #ifndef CONFIG_USER_ONLY -static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx) +ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx) { uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; uint32_t el = regime_el(env, mmu_idx); @@ -11223,406 +11223,6 @@ static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, }; } -/** - * get_phys_addr_lpae: perform one stage of page table walk, LPAE format - * - * Returns false if the translation was successful. Otherwise, phys_ptr, attrs, - * prot and page_size may not be filled in, and the populated fsr value provides - * information on why the translation aborted, in the format of a long-format - * DFSR/IFSR fault register, with the following caveats: - * * the WnR bit is never set (the caller must do this). - * - * @env: CPUARMState - * @address: virtual address to get physical address for - * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH - * @mmu_idx: MMU index indicating required translation regime - * @s1_is_el0: if @mmu_idx is ARMMMUIdx_Stage2 (so this is a stage 2 page table - * walk), must be true if this is stage 2 of a stage 1+2 walk for an - * EL0 access). If @mmu_idx is anything else, @s1_is_el0 is ignored. - * @phys_ptr: set to the physical address corresponding to the virtual address - * @attrs: set to the memory transaction attributes to use - * @prot: set to the permissions for the page containing phys_ptr - * @page_size_ptr: set to the size of the page containing phys_ptr - * @fi: set to fault info if the translation fails - * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes - */ -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) -{ - ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); - /* Read an LPAE long-descriptor translation table. */ - ARMFaultType fault_type = ARMFault_Translation; - uint32_t level; - ARMVAParameters param; - uint64_t ttbr; - hwaddr descaddr, indexmask, indexmask_grainsize; - uint32_t tableattrs; - target_ulong page_size; - uint32_t attrs; - int32_t stride; - int addrsize, inputsize, outputsize; - TCR *tcr = regime_tcr(env, mmu_idx); - int ap, ns, xn, pxn; - uint32_t el = regime_el(env, mmu_idx); - uint64_t descaddrmask; - bool aarch64 = arm_el_is_aa64(env, el); - bool guarded = false; - - /* TODO: This code does not support shareability levels. */ - if (aarch64) { - int ps; - - param = aa64_va_parameters(env, address, mmu_idx, - access_type != MMU_INST_FETCH); - level = 0; - - /* - * If TxSZ is programmed to a value larger than the maximum, - * or smaller than the effective minimum, it is IMPLEMENTATION - * DEFINED whether we behave as if the field were programmed - * within bounds, or if a level 0 Translation fault is generated. - * - * With FEAT_LVA, fault on less than minimum becomes required, - * so our choice is to always raise the fault. - */ - if (param.tsz_oob) { - fault_type = ARMFault_Translation; - goto do_fault; - } - - addrsize = 64 - 8 * param.tbi; - inputsize = 64 - param.tsz; - - /* - * Bound PS by PARANGE to find the effective output address size. - * ID_AA64MMFR0 is a read-only register so values outside of the - * supported mappings can be considered an implementation error. - */ - ps = FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); - ps = MIN(ps, param.ps); - assert(ps < ARRAY_SIZE(pamax_map)); - outputsize = pamax_map[ps]; - } else { - param = aa32_va_parameters(env, address, mmu_idx); - level = 1; - addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); - inputsize = addrsize - param.tsz; - outputsize = 40; - } - - /* - * We determined the region when collecting the parameters, but we - * have not yet validated that the address is valid for the region. - * Extract the top bits and verify that they all match select. - * - * For aa32, if inputsize == addrsize, then we have selected the - * region by exclusion in aa32_va_parameters and there is no more - * validation to do here. - */ - if (inputsize < addrsize) { - target_ulong top_bits = sextract64(address, inputsize, - addrsize - inputsize); - if (-top_bits != param.select) { - /* The gap between the two regions is a Translation fault */ - fault_type = ARMFault_Translation; - goto do_fault; - } - } - - if (param.using64k) { - stride = 13; - } else if (param.using16k) { - stride = 11; - } else { - stride = 9; - } - - /* Note that QEMU ignores shareability and cacheability attributes, - * so we don't need to do anything with the SH, ORGN, IRGN fields - * in the TTBCR. Similarly, TTBCR:A1 selects whether we get the - * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently - * implement any ASID-like capability so we can ignore it (instead - * we will always flush the TLB any time the ASID is changed). - */ - ttbr = regime_ttbr(env, mmu_idx, param.select); - - /* Here we should have set up all the parameters for the translation: - * inputsize, ttbr, epd, stride, tbi - */ - - if (param.epd) { - /* Translation table walk disabled => Translation fault on TLB miss - * Note: This is always 0 on 64-bit EL2 and EL3. - */ - goto do_fault; - } - - if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { - /* The starting level depends on the virtual address size (which can - * be up to 48 bits) and the translation granule size. It indicates - * the number of strides (stride bits at a time) needed to - * consume the bits of the input address. In the pseudocode this is: - * level = 4 - RoundUp((inputsize - grainsize) / stride) - * where their 'inputsize' is our 'inputsize', 'grainsize' is - * our 'stride + 3' and 'stride' is our 'stride'. - * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying: - * = 4 - (inputsize - stride - 3 + stride - 1) / stride - * = 4 - (inputsize - 4) / stride; - */ - level = 4 - (inputsize - 4) / stride; - } else { - /* For stage 2 translations the starting level is specified by the - * VTCR_EL2.SL0 field (whose interpretation depends on the page size) - */ - uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2); - uint32_t sl2 = extract64(tcr->raw_tcr, 33, 1); - uint32_t startlevel; - bool ok; - - /* SL2 is RES0 unless DS=1 & 4kb granule. */ - if (param.ds && stride == 9 && sl2) { - if (sl0 != 0) { - level = 0; - fault_type = ARMFault_Translation; - goto do_fault; - } - startlevel = -1; - } else if (!aarch64 || stride == 9) { - /* AArch32 or 4KB pages */ - startlevel = 2 - sl0; - - if (cpu_isar_feature(aa64_st, cpu)) { - startlevel &= 3; - } - } else { - /* 16KB or 64KB pages */ - startlevel = 3 - sl0; - } - - /* Check that the starting level is valid. */ - ok = check_s2_mmu_setup(cpu, aarch64, startlevel, - inputsize, stride, outputsize); - if (!ok) { - fault_type = ARMFault_Translation; - goto do_fault; - } - level = startlevel; - } - - indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3); - indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level))); - - /* Now we can extract the actual base address from the TTBR */ - descaddr = extract64(ttbr, 0, 48); - - /* - * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR. - * - * Otherwise, if the base address is out of range, raise AddressSizeFault. - * In the pseudocode, this is !IsZero(baseregister<47:outputsize>), - * but we've just cleared the bits above 47, so simplify the test. - */ - if (outputsize > 48) { - descaddr |= extract64(ttbr, 2, 4) << 48; - } else if (descaddr >> outputsize) { - level = 0; - fault_type = ARMFault_AddressSize; - goto do_fault; - } - - /* - * We rely on this masking to clear the RES0 bits at the bottom of the TTBR - * and also to mask out CnP (bit 0) which could validly be non-zero. - */ - descaddr &= ~indexmask; - - /* - * For AArch32, the address field in the descriptor goes up to bit 39 - * for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0 - * or an AddressSize fault is raised. So for v8 we extract those SBZ - * bits as part of the address, which will be checked via outputsize. - * For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2; - * the highest bits of a 52-bit output are placed elsewhere. - */ - if (param.ds) { - descaddrmask = MAKE_64BIT_MASK(0, 50); - } else if (arm_feature(env, ARM_FEATURE_V8)) { - descaddrmask = MAKE_64BIT_MASK(0, 48); - } else { - descaddrmask = MAKE_64BIT_MASK(0, 40); - } - descaddrmask &= ~indexmask_grainsize; - - /* Secure accesses start with the page table in secure memory and - * can be downgraded to non-secure at any step. Non-secure accesses - * remain non-secure. We implement this by just ORing in the NSTable/NS - * bits at each step. - */ - tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4); - for (;;) { - uint64_t descriptor; - bool nstable; - - descaddr |= (address >> (stride * (4 - level))) & indexmask; - descaddr &= ~7ULL; - nstable = extract32(tableattrs, 4, 1); - descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); - if (fi->type != ARMFault_None) { - goto do_fault; - } - - if (!(descriptor & 1) || - (!(descriptor & 2) && (level == 3))) { - /* Invalid, or the Reserved level 3 encoding */ - goto do_fault; - } - - descaddr = descriptor & descaddrmask; - - /* - * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12] - * of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of - * descaddr are in [9:8]. Otherwise, if descaddr is out of range, - * raise AddressSizeFault. - */ - if (outputsize > 48) { - if (param.ds) { - descaddr |= extract64(descriptor, 8, 2) << 50; - } else { - descaddr |= extract64(descriptor, 12, 4) << 48; - } - } else if (descaddr >> outputsize) { - fault_type = ARMFault_AddressSize; - goto do_fault; - } - - if ((descriptor & 2) && (level < 3)) { - /* Table entry. The top five bits are attributes which may - * propagate down through lower levels of the table (and - * which are all arranged so that 0 means "no effect", so - * we can gather them up by ORing in the bits at each level). - */ - tableattrs |= extract64(descriptor, 59, 5); - level++; - indexmask = indexmask_grainsize; - continue; - } - /* - * Block entry at level 1 or 2, or page entry at level 3. - * These are basically the same thing, although the number - * of bits we pull in from the vaddr varies. Note that although - * descaddrmask masks enough of the low bits of the descriptor - * to give a correct page or table address, the address field - * in a block descriptor is smaller; so we need to explicitly - * clear the lower bits here before ORing in the low vaddr bits. - */ - page_size = (1ULL << ((stride * (4 - level)) + 3)); - descaddr &= ~(page_size - 1); - descaddr |= (address & (page_size - 1)); - /* Extract attributes from the descriptor */ - attrs = extract64(descriptor, 2, 10) - | (extract64(descriptor, 52, 12) << 10); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - /* Stage 2 table descriptors do not include any attribute fields */ - break; - } - /* Merge in attributes from table descriptors */ - attrs |= nstable << 3; /* NS */ - guarded = extract64(descriptor, 50, 1); /* GP */ - if (param.hpd) { - /* HPD disables all the table attributes except NSTable. */ - break; - } - attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */ - /* The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1 - * means "force PL1 access only", which means forcing AP[1] to 0. - */ - attrs &= ~(extract32(tableattrs, 2, 1) << 4); /* !APT[0] => AP[1] */ - attrs |= extract32(tableattrs, 3, 1) << 5; /* APT[1] => AP[2] */ - break; - } - /* Here descaddr is the final physical address, and attributes - * are all in attrs. - */ - fault_type = ARMFault_AccessFlag; - if ((attrs & (1 << 8)) == 0) { - /* Access flag */ - goto do_fault; - } - - ap = extract32(attrs, 4, 2); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - ns = mmu_idx == ARMMMUIdx_Stage2; - xn = extract32(attrs, 11, 2); - *prot = get_S2prot(env, ap, xn, s1_is_el0); - } else { - ns = extract32(attrs, 3, 1); - xn = extract32(attrs, 12, 1); - pxn = extract32(attrs, 11, 1); - *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn); - } - - fault_type = ARMFault_Permission; - if (!(*prot & (1 << access_type))) { - goto do_fault; - } - - if (ns) { - /* The NS bit will (as required by the architecture) have no effect if - * the CPU doesn't support TZ or this is a non-secure translation - * regime, because the attribute will already be non-secure. - */ - txattrs->secure = false; - } - /* When in aarch64 mode, and BTI is enabled, remember GP in the IOTLB. */ - if (aarch64 && guarded && cpu_isar_feature(aa64_bti, cpu)) { - arm_tlb_bti_gp(txattrs) = true; - } - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - cacheattrs->is_s2_format = true; - cacheattrs->attrs = extract32(attrs, 0, 4); - } else { - /* Index into MAIR registers for cache attributes */ - uint8_t attrindx = extract32(attrs, 0, 3); - uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)]; - assert(attrindx <= 7); - cacheattrs->is_s2_format = false; - cacheattrs->attrs = extract64(mair, attrindx * 8, 8); - } - - /* - * For FEAT_LPA2 and effective DS, the SH field in the attributes - * was re-purposed for output address bits. The SH attribute in - * that case comes from TCR_ELx, which we extracted earlier. - */ - if (param.ds) { - cacheattrs->shareability = param.sh; - } else { - cacheattrs->shareability = extract32(attrs, 6, 2); - } - - *phys_ptr = descaddr; - *page_size_ptr = page_size; - return false; - -do_fault: - fi->type = fault_type; - fi->level = level; - /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ - fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2 || - mmu_idx == ARMMMUIdx_Stage2_S); - fi->s1ns = mmu_idx == ARMMMUIdx_Stage2; - return true; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index f2ca2bb8fe12..cbccf91b1320 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -314,6 +314,417 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, return true; } +/** + * get_phys_addr_lpae: perform one stage of page table walk, LPAE format + * + * Returns false if the translation was successful. Otherwise, phys_ptr, + * attrs, prot and page_size may not be filled in, and the populated fsr + * value provides information on why the translation aborted, in the format + * of a long-format DFSR/IFSR fault register, with the following caveat: + * the WnR bit is never set (the caller must do this). + * + * @env: CPUARMState + * @address: virtual address to get physical address for + * @access_type: MMU_DATA_LOAD, MMU_DATA_STORE or MMU_INST_FETCH + * @mmu_idx: MMU index indicating required translation regime + * @s1_is_el0: if @mmu_idx is ARMMMUIdx_Stage2 (so this is a stage 2 page + * table walk), must be true if this is stage 2 of a stage 1+2 + * walk for an EL0 access. If @mmu_idx is anything else, + * @s1_is_el0 is ignored. + * @phys_ptr: set to the physical address corresponding to the virtual address + * @attrs: set to the memory transaction attributes to use + * @prot: set to the permissions for the page containing phys_ptr + * @page_size_ptr: set to the size of the page containing phys_ptr + * @fi: set to fault info if the translation fails + * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes + */ +bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, + hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +{ + ARMCPU *cpu = env_archcpu(env); + CPUState *cs = CPU(cpu); + /* Read an LPAE long-descriptor translation table. */ + ARMFaultType fault_type = ARMFault_Translation; + uint32_t level; + ARMVAParameters param; + uint64_t ttbr; + hwaddr descaddr, indexmask, indexmask_grainsize; + uint32_t tableattrs; + target_ulong page_size; + uint32_t attrs; + int32_t stride; + int addrsize, inputsize, outputsize; + TCR *tcr = regime_tcr(env, mmu_idx); + int ap, ns, xn, pxn; + uint32_t el = regime_el(env, mmu_idx); + uint64_t descaddrmask; + bool aarch64 = arm_el_is_aa64(env, el); + bool guarded = false; + + /* TODO: This code does not support shareability levels. */ + if (aarch64) { + int ps; + + param = aa64_va_parameters(env, address, mmu_idx, + access_type != MMU_INST_FETCH); + level = 0; + + /* + * If TxSZ is programmed to a value larger than the maximum, + * or smaller than the effective minimum, it is IMPLEMENTATION + * DEFINED whether we behave as if the field were programmed + * within bounds, or if a level 0 Translation fault is generated. + * + * With FEAT_LVA, fault on less than minimum becomes required, + * so our choice is to always raise the fault. + */ + if (param.tsz_oob) { + fault_type = ARMFault_Translation; + goto do_fault; + } + + addrsize = 64 - 8 * param.tbi; + inputsize = 64 - param.tsz; + + /* + * Bound PS by PARANGE to find the effective output address size. + * ID_AA64MMFR0 is a read-only register so values outside of the + * supported mappings can be considered an implementation error. + */ + ps = FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); + ps = MIN(ps, param.ps); + assert(ps < ARRAY_SIZE(pamax_map)); + outputsize = pamax_map[ps]; + } else { + param = aa32_va_parameters(env, address, mmu_idx); + level = 1; + addrsize = (mmu_idx == ARMMMUIdx_Stage2 ? 40 : 32); + inputsize = addrsize - param.tsz; + outputsize = 40; + } + + /* + * We determined the region when collecting the parameters, but we + * have not yet validated that the address is valid for the region. + * Extract the top bits and verify that they all match select. + * + * For aa32, if inputsize == addrsize, then we have selected the + * region by exclusion in aa32_va_parameters and there is no more + * validation to do here. + */ + if (inputsize < addrsize) { + target_ulong top_bits = sextract64(address, inputsize, + addrsize - inputsize); + if (-top_bits != param.select) { + /* The gap between the two regions is a Translation fault */ + fault_type = ARMFault_Translation; + goto do_fault; + } + } + + if (param.using64k) { + stride = 13; + } else if (param.using16k) { + stride = 11; + } else { + stride = 9; + } + + /* + * Note that QEMU ignores shareability and cacheability attributes, + * so we don't need to do anything with the SH, ORGN, IRGN fields + * in the TTBCR. Similarly, TTBCR:A1 selects whether we get the + * ASID from TTBR0 or TTBR1, but QEMU's TLB doesn't currently + * implement any ASID-like capability so we can ignore it (instead + * we will always flush the TLB any time the ASID is changed). + */ + ttbr = regime_ttbr(env, mmu_idx, param.select); + + /* + * Here we should have set up all the parameters for the translation: + * inputsize, ttbr, epd, stride, tbi + */ + + if (param.epd) { + /* + * Translation table walk disabled => Translation fault on TLB miss + * Note: This is always 0 on 64-bit EL2 and EL3. + */ + goto do_fault; + } + + if (mmu_idx != ARMMMUIdx_Stage2 && mmu_idx != ARMMMUIdx_Stage2_S) { + /* + * The starting level depends on the virtual address size (which can + * be up to 48 bits) and the translation granule size. It indicates + * the number of strides (stride bits at a time) needed to + * consume the bits of the input address. In the pseudocode this is: + * level = 4 - RoundUp((inputsize - grainsize) / stride) + * where their 'inputsize' is our 'inputsize', 'grainsize' is + * our 'stride + 3' and 'stride' is our 'stride'. + * Applying the usual "rounded up m/n is (m+n-1)/n" and simplifying: + * = 4 - (inputsize - stride - 3 + stride - 1) / stride + * = 4 - (inputsize - 4) / stride; + */ + level = 4 - (inputsize - 4) / stride; + } else { + /* + * For stage 2 translations the starting level is specified by the + * VTCR_EL2.SL0 field (whose interpretation depends on the page size) + */ + uint32_t sl0 = extract32(tcr->raw_tcr, 6, 2); + uint32_t sl2 = extract64(tcr->raw_tcr, 33, 1); + uint32_t startlevel; + bool ok; + + /* SL2 is RES0 unless DS=1 & 4kb granule. */ + if (param.ds && stride == 9 && sl2) { + if (sl0 != 0) { + level = 0; + fault_type = ARMFault_Translation; + goto do_fault; + } + startlevel = -1; + } else if (!aarch64 || stride == 9) { + /* AArch32 or 4KB pages */ + startlevel = 2 - sl0; + + if (cpu_isar_feature(aa64_st, cpu)) { + startlevel &= 3; + } + } else { + /* 16KB or 64KB pages */ + startlevel = 3 - sl0; + } + + /* Check that the starting level is valid. */ + ok = check_s2_mmu_setup(cpu, aarch64, startlevel, + inputsize, stride, outputsize); + if (!ok) { + fault_type = ARMFault_Translation; + goto do_fault; + } + level = startlevel; + } + + indexmask_grainsize = MAKE_64BIT_MASK(0, stride + 3); + indexmask = MAKE_64BIT_MASK(0, inputsize - (stride * (4 - level))); + + /* Now we can extract the actual base address from the TTBR */ + descaddr = extract64(ttbr, 0, 48); + + /* + * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [5:2] of TTBR. + * + * Otherwise, if the base address is out of range, raise AddressSizeFault. + * In the pseudocode, this is !IsZero(baseregister<47:outputsize>), + * but we've just cleared the bits above 47, so simplify the test. + */ + if (outputsize > 48) { + descaddr |= extract64(ttbr, 2, 4) << 48; + } else if (descaddr >> outputsize) { + level = 0; + fault_type = ARMFault_AddressSize; + goto do_fault; + } + + /* + * We rely on this masking to clear the RES0 bits at the bottom of the TTBR + * and also to mask out CnP (bit 0) which could validly be non-zero. + */ + descaddr &= ~indexmask; + + /* + * For AArch32, the address field in the descriptor goes up to bit 39 + * for both v7 and v8. However, for v8 the SBZ bits [47:40] must be 0 + * or an AddressSize fault is raised. So for v8 we extract those SBZ + * bits as part of the address, which will be checked via outputsize. + * For AArch64, the address field goes up to bit 47, or 49 with FEAT_LPA2; + * the highest bits of a 52-bit output are placed elsewhere. + */ + if (param.ds) { + descaddrmask = MAKE_64BIT_MASK(0, 50); + } else if (arm_feature(env, ARM_FEATURE_V8)) { + descaddrmask = MAKE_64BIT_MASK(0, 48); + } else { + descaddrmask = MAKE_64BIT_MASK(0, 40); + } + descaddrmask &= ~indexmask_grainsize; + + /* + * Secure accesses start with the page table in secure memory and + * can be downgraded to non-secure at any step. Non-secure accesses + * remain non-secure. We implement this by just ORing in the NSTable/NS + * bits at each step. + */ + tableattrs = regime_is_secure(env, mmu_idx) ? 0 : (1 << 4); + for (;;) { + uint64_t descriptor; + bool nstable; + + descaddr |= (address >> (stride * (4 - level))) & indexmask; + descaddr &= ~7ULL; + nstable = extract32(tableattrs, 4, 1); + descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); + if (fi->type != ARMFault_None) { + goto do_fault; + } + + if (!(descriptor & 1) || + (!(descriptor & 2) && (level == 3))) { + /* Invalid, or the Reserved level 3 encoding */ + goto do_fault; + } + + descaddr = descriptor & descaddrmask; + + /* + * For FEAT_LPA and PS=6, bits [51:48] of descaddr are in [15:12] + * of descriptor. For FEAT_LPA2 and effective DS, bits [51:50] of + * descaddr are in [9:8]. Otherwise, if descaddr is out of range, + * raise AddressSizeFault. + */ + if (outputsize > 48) { + if (param.ds) { + descaddr |= extract64(descriptor, 8, 2) << 50; + } else { + descaddr |= extract64(descriptor, 12, 4) << 48; + } + } else if (descaddr >> outputsize) { + fault_type = ARMFault_AddressSize; + goto do_fault; + } + + if ((descriptor & 2) && (level < 3)) { + /* + * Table entry. The top five bits are attributes which may + * propagate down through lower levels of the table (and + * which are all arranged so that 0 means "no effect", so + * we can gather them up by ORing in the bits at each level). + */ + tableattrs |= extract64(descriptor, 59, 5); + level++; + indexmask = indexmask_grainsize; + continue; + } + /* + * Block entry at level 1 or 2, or page entry at level 3. + * These are basically the same thing, although the number + * of bits we pull in from the vaddr varies. Note that although + * descaddrmask masks enough of the low bits of the descriptor + * to give a correct page or table address, the address field + * in a block descriptor is smaller; so we need to explicitly + * clear the lower bits here before ORing in the low vaddr bits. + */ + page_size = (1ULL << ((stride * (4 - level)) + 3)); + descaddr &= ~(page_size - 1); + descaddr |= (address & (page_size - 1)); + /* Extract attributes from the descriptor */ + attrs = extract64(descriptor, 2, 10) + | (extract64(descriptor, 52, 12) << 10); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + /* Stage 2 table descriptors do not include any attribute fields */ + break; + } + /* Merge in attributes from table descriptors */ + attrs |= nstable << 3; /* NS */ + guarded = extract64(descriptor, 50, 1); /* GP */ + if (param.hpd) { + /* HPD disables all the table attributes except NSTable. */ + break; + } + attrs |= extract32(tableattrs, 0, 2) << 11; /* XN, PXN */ + /* + * The sense of AP[1] vs APTable[0] is reversed, as APTable[0] == 1 + * means "force PL1 access only", which means forcing AP[1] to 0. + */ + attrs &= ~(extract32(tableattrs, 2, 1) << 4); /* !APT[0] => AP[1] */ + attrs |= extract32(tableattrs, 3, 1) << 5; /* APT[1] => AP[2] */ + break; + } + /* + * Here descaddr is the final physical address, and attributes + * are all in attrs. + */ + fault_type = ARMFault_AccessFlag; + if ((attrs & (1 << 8)) == 0) { + /* Access flag */ + goto do_fault; + } + + ap = extract32(attrs, 4, 2); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + ns = mmu_idx == ARMMMUIdx_Stage2; + xn = extract32(attrs, 11, 2); + *prot = get_S2prot(env, ap, xn, s1_is_el0); + } else { + ns = extract32(attrs, 3, 1); + xn = extract32(attrs, 12, 1); + pxn = extract32(attrs, 11, 1); + *prot = get_S1prot(env, mmu_idx, aarch64, ap, ns, xn, pxn); + } + + fault_type = ARMFault_Permission; + if (!(*prot & (1 << access_type))) { + goto do_fault; + } + + if (ns) { + /* + * The NS bit will (as required by the architecture) have no effect if + * the CPU doesn't support TZ or this is a non-secure translation + * regime, because the attribute will already be non-secure. + */ + txattrs->secure = false; + } + /* When in aarch64 mode, and BTI is enabled, remember GP in the IOTLB. */ + if (aarch64 && guarded && cpu_isar_feature(aa64_bti, cpu)) { + arm_tlb_bti_gp(txattrs) = true; + } + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + cacheattrs->is_s2_format = true; + cacheattrs->attrs = extract32(attrs, 0, 4); + } else { + /* Index into MAIR registers for cache attributes */ + uint8_t attrindx = extract32(attrs, 0, 3); + uint64_t mair = env->cp15.mair_el[regime_el(env, mmu_idx)]; + assert(attrindx <= 7); + cacheattrs->is_s2_format = false; + cacheattrs->attrs = extract64(mair, attrindx * 8, 8); + } + + /* + * For FEAT_LPA2 and effective DS, the SH field in the attributes + * was re-purposed for output address bits. The SH attribute in + * that case comes from TCR_ELx, which we extracted earlier. + */ + if (param.ds) { + cacheattrs->shareability = param.sh; + } else { + cacheattrs->shareability = extract32(attrs, 6, 2); + } + + *phys_ptr = descaddr; + *page_size_ptr = page_size; + return false; + +do_fault: + fi->type = fault_type; + fi->level = level; + /* Tag the error as S2 for failed S1 PTW at S2 or ordinary S2. */ + fi->stage2 = fi->s1ptw || (mmu_idx == ARMMMUIdx_Stage2 || + mmu_idx == ARMMMUIdx_Stage2_S); + fi->s1ns = mmu_idx == ARMMMUIdx_Stage2; + return true; +} + static bool get_phys_addr_pmsav5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index b2dfe489bbed..31744df6646b 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,6 +11,8 @@ #ifndef CONFIG_USER_ONLY +extern const uint8_t pamax_map[7]; + uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, @@ -30,6 +32,14 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } +ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx); +bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize); +int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); +int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn); + bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, bool s1_is_el0, From 11552bb0d95484c99233ec2f09c25261885d08ed Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:51 +0100 Subject: [PATCH 26/90] target/arm: Move arm_{ldl,ldq}_ptw to ptw.c Move the ptw load functions, plus 3 common subroutines: S1_ptw_translate, ptw_attrs_are_device, and regime_translation_big_endian. This also allows get_phys_addr_lpae to become static again. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-17-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 141 -------------------------------------- target/arm/ptw.c | 160 ++++++++++++++++++++++++++++++++++++++++++-- target/arm/ptw.h | 13 ---- 3 files changed, 154 insertions(+), 160 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 7de815fe9860..398bcd62ab91 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10475,12 +10475,6 @@ bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; } -static inline bool regime_translation_big_endian(CPUARMState *env, - ARMMMUIdx mmu_idx) -{ - return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; -} - /* Return the TTBR associated with this translation regime */ uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) { @@ -10773,141 +10767,6 @@ int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } -static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) -{ - /* - * For an S1 page table walk, the stage 1 attributes are always - * some form of "this is Normal memory". The combined S1+S2 - * attributes are therefore only Device if stage 2 specifies Device. - * With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00, - * ie when cacheattrs.attrs bits [3:2] are 0b00. - * With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie - * when cacheattrs.attrs bit [2] is 0. - */ - assert(cacheattrs.is_s2_format); - if (arm_hcr_el2_eff(env) & HCR_FWB) { - return (cacheattrs.attrs & 0x4) == 0; - } else { - return (cacheattrs.attrs & 0xc) == 0; - } -} - -/* Translate a S1 pagetable walk through S2 if needed. */ -static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, - hwaddr addr, bool *is_secure, - ARMMMUFaultInfo *fi) -{ - if (arm_mmu_idx_is_stage1_of_2(mmu_idx) && - !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { - target_ulong s2size; - hwaddr s2pa; - int s2prot; - int ret; - ARMMMUIdx s2_mmu_idx = *is_secure ? ARMMMUIdx_Stage2_S - : ARMMMUIdx_Stage2; - ARMCacheAttrs cacheattrs = {}; - MemTxAttrs txattrs = {}; - - ret = get_phys_addr_lpae(env, addr, MMU_DATA_LOAD, s2_mmu_idx, false, - &s2pa, &txattrs, &s2prot, &s2size, fi, - &cacheattrs); - if (ret) { - assert(fi->type != ARMFault_None); - fi->s2addr = addr; - fi->stage2 = true; - fi->s1ptw = true; - fi->s1ns = !*is_secure; - return ~0; - } - if ((arm_hcr_el2_eff(env) & HCR_PTW) && - ptw_attrs_are_device(env, cacheattrs)) { - /* - * PTW set and S1 walk touched S2 Device memory: - * generate Permission fault. - */ - fi->type = ARMFault_Permission; - fi->s2addr = addr; - fi->stage2 = true; - fi->s1ptw = true; - fi->s1ns = !*is_secure; - return ~0; - } - - if (arm_is_secure_below_el3(env)) { - /* Check if page table walk is to secure or non-secure PA space. */ - if (*is_secure) { - *is_secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); - } else { - *is_secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); - } - } else { - assert(!*is_secure); - } - - addr = s2pa; - } - return addr; -} - -/* All loads done in the course of a page table walk go through here. */ -uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - MemTxAttrs attrs = {}; - MemTxResult result = MEMTX_OK; - AddressSpace *as; - uint32_t data; - - addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); - attrs.secure = is_secure; - as = arm_addressspace(cs, attrs); - if (fi->s1ptw) { - return 0; - } - if (regime_translation_big_endian(env, mmu_idx)) { - data = address_space_ldl_be(as, addr, attrs, &result); - } else { - data = address_space_ldl_le(as, addr, attrs, &result); - } - if (result == MEMTX_OK) { - return data; - } - fi->type = ARMFault_SyncExternalOnWalk; - fi->ea = arm_extabort_type(result); - return 0; -} - -uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - MemTxAttrs attrs = {}; - MemTxResult result = MEMTX_OK; - AddressSpace *as; - uint64_t data; - - addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); - attrs.secure = is_secure; - as = arm_addressspace(cs, attrs); - if (fi->s1ptw) { - return 0; - } - if (regime_translation_big_endian(env, mmu_idx)) { - data = address_space_ldq_be(as, addr, attrs, &result); - } else { - data = address_space_ldq_le(as, addr, attrs, &result); - } - if (result == MEMTX_OK) { - return data; - } - fi->type = ARMFault_SyncExternalOnWalk; - fi->ea = arm_extabort_type(result); - return 0; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index cbccf91b1320..e4b860d2aee8 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -15,6 +15,154 @@ #include "ptw.h" +static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, hwaddr *phys_ptr, + MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) + __attribute__((nonnull)); + +static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; +} + +static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) +{ + /* + * For an S1 page table walk, the stage 1 attributes are always + * some form of "this is Normal memory". The combined S1+S2 + * attributes are therefore only Device if stage 2 specifies Device. + * With HCR_EL2.FWB == 0 this is when descriptor bits [5:4] are 0b00, + * ie when cacheattrs.attrs bits [3:2] are 0b00. + * With HCR_EL2.FWB == 1 this is when descriptor bit [4] is 0, ie + * when cacheattrs.attrs bit [2] is 0. + */ + assert(cacheattrs.is_s2_format); + if (arm_hcr_el2_eff(env) & HCR_FWB) { + return (cacheattrs.attrs & 0x4) == 0; + } else { + return (cacheattrs.attrs & 0xc) == 0; + } +} + +/* Translate a S1 pagetable walk through S2 if needed. */ +static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, + hwaddr addr, bool *is_secure, + ARMMMUFaultInfo *fi) +{ + if (arm_mmu_idx_is_stage1_of_2(mmu_idx) && + !regime_translation_disabled(env, ARMMMUIdx_Stage2)) { + target_ulong s2size; + hwaddr s2pa; + int s2prot; + int ret; + ARMMMUIdx s2_mmu_idx = *is_secure ? ARMMMUIdx_Stage2_S + : ARMMMUIdx_Stage2; + ARMCacheAttrs cacheattrs = {}; + MemTxAttrs txattrs = {}; + + ret = get_phys_addr_lpae(env, addr, MMU_DATA_LOAD, s2_mmu_idx, false, + &s2pa, &txattrs, &s2prot, &s2size, fi, + &cacheattrs); + if (ret) { + assert(fi->type != ARMFault_None); + fi->s2addr = addr; + fi->stage2 = true; + fi->s1ptw = true; + fi->s1ns = !*is_secure; + return ~0; + } + if ((arm_hcr_el2_eff(env) & HCR_PTW) && + ptw_attrs_are_device(env, cacheattrs)) { + /* + * PTW set and S1 walk touched S2 Device memory: + * generate Permission fault. + */ + fi->type = ARMFault_Permission; + fi->s2addr = addr; + fi->stage2 = true; + fi->s1ptw = true; + fi->s1ns = !*is_secure; + return ~0; + } + + if (arm_is_secure_below_el3(env)) { + /* Check if page table walk is to secure or non-secure PA space. */ + if (*is_secure) { + *is_secure = !(env->cp15.vstcr_el2.raw_tcr & VSTCR_SW); + } else { + *is_secure = !(env->cp15.vtcr_el2.raw_tcr & VTCR_NSW); + } + } else { + assert(!*is_secure); + } + + addr = s2pa; + } + return addr; +} + +/* All loads done in the course of a page table walk go through here. */ +static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult result = MEMTX_OK; + AddressSpace *as; + uint32_t data; + + addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); + attrs.secure = is_secure; + as = arm_addressspace(cs, attrs); + if (fi->s1ptw) { + return 0; + } + if (regime_translation_big_endian(env, mmu_idx)) { + data = address_space_ldl_be(as, addr, attrs, &result); + } else { + data = address_space_ldl_le(as, addr, attrs, &result); + } + if (result == MEMTX_OK) { + return data; + } + fi->type = ARMFault_SyncExternalOnWalk; + fi->ea = arm_extabort_type(result); + return 0; +} + +static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, + ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + MemTxAttrs attrs = {}; + MemTxResult result = MEMTX_OK; + AddressSpace *as; + uint64_t data; + + addr = S1_ptw_translate(env, mmu_idx, addr, &is_secure, fi); + attrs.secure = is_secure; + as = arm_addressspace(cs, attrs); + if (fi->s1ptw) { + return 0; + } + if (regime_translation_big_endian(env, mmu_idx)) { + data = address_space_ldq_be(as, addr, attrs, &result); + } else { + data = address_space_ldq_le(as, addr, attrs, &result); + } + if (result == MEMTX_OK) { + return data; + } + fi->type = ARMFault_SyncExternalOnWalk; + fi->ea = arm_extabort_type(result); + return 0; +} + static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, uint32_t *table, uint32_t address) { @@ -338,12 +486,12 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, * @fi: set to fault info if the translation fails * @cacheattrs: (if non-NULL) set to the cacheability/shareability attributes */ -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) +static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, + MMUAccessType access_type, ARMMMUIdx mmu_idx, + bool s1_is_el0, hwaddr *phys_ptr, + MemTxAttrs *txattrs, int *prot, + target_ulong *page_size_ptr, + ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); CPUState *cs = CPU(cpu); diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 31744df6646b..28b8cb9fb896 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -13,11 +13,6 @@ extern const uint8_t pamax_map[7]; -uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); -uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, - ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi); - bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); @@ -40,13 +35,5 @@ int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, int ap, int ns, int xn, int pxn); -bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, - MMUAccessType access_type, ARMMMUIdx mmu_idx, - bool s1_is_el0, - hwaddr *phys_ptr, MemTxAttrs *txattrs, int *prot, - target_ulong *page_size_ptr, - ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) - __attribute__((nonnull)); - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From cd6bc4d51730e9cf47489029d078d18c3bcb3ae2 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 27/90] target/arm: Move {arm_s1_, }regime_using_lpae_format to tlb_helper.c These functions are used for both page table walking and for deciding what format in which to deliver exception results. Since ptw.c is only present for system mode, put the functions into tlb_helper.c. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-18-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 24 ------------------------ target/arm/tlb_helper.c | 26 ++++++++++++++++++++++++++ 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 398bcd62ab91..d2b196ff3e55 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10515,30 +10515,6 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) } #endif /* !CONFIG_USER_ONLY */ -/* Return true if the translation regime is using LPAE format page tables */ -bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - int el = regime_el(env, mmu_idx); - if (el == 2 || arm_el_is_aa64(env, el)) { - return true; - } - if (arm_feature(env, ARM_FEATURE_LPAE) - && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) { - return true; - } - return false; -} - -/* Returns true if the stage 1 translation regime is using LPAE format page - * tables. Used when raising alignment exceptions, whose FSR changes depending - * on whether the long or short descriptor format is in use. */ -bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - mmu_idx = stage_1_mmu_idx(mmu_idx); - - return regime_using_lpae_format(env, mmu_idx); -} - #ifndef CONFIG_USER_ONLY bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) { diff --git a/target/arm/tlb_helper.c b/target/arm/tlb_helper.c index 6421e16202e5..7d8a86b3c455 100644 --- a/target/arm/tlb_helper.c +++ b/target/arm/tlb_helper.c @@ -11,6 +11,32 @@ #include "exec/exec-all.h" #include "exec/helper-proto.h" + +/* Return true if the translation regime is using LPAE format page tables */ +bool regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + int el = regime_el(env, mmu_idx); + if (el == 2 || arm_el_is_aa64(env, el)) { + return true; + } + if (arm_feature(env, ARM_FEATURE_LPAE) + && (regime_tcr(env, mmu_idx)->raw_tcr & TTBCR_EAE)) { + return true; + } + return false; +} + +/* + * Returns true if the stage 1 translation regime is using LPAE format page + * tables. Used when raising alignment exceptions, whose FSR changes depending + * on whether the long or short descriptor format is in use. + */ +bool arm_s1_regime_using_lpae_format(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + mmu_idx = stage_1_mmu_idx(mmu_idx); + return regime_using_lpae_format(env, mmu_idx); +} + static inline uint32_t merge_syn_data_abort(uint32_t template_syn, unsigned int target_el, bool same_el, bool ea, From 1c73d84807712aad4c7b2e4e753d6870ab81145d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 28/90] target/arm: Move arm_pamax, pamax_map into ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-19-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 25 ------------------------- target/arm/ptw.c | 25 +++++++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 25 insertions(+), 27 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index d2b196ff3e55..563e34ecded1 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10814,31 +10814,6 @@ bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, } #endif /* !CONFIG_USER_ONLY */ -/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ -const uint8_t pamax_map[] = { - [0] = 32, - [1] = 36, - [2] = 40, - [3] = 42, - [4] = 44, - [5] = 48, - [6] = 52, -}; - -/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */ -unsigned int arm_pamax(ARMCPU *cpu) -{ - unsigned int parange = - FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); - - /* - * id_aa64mmfr0 is a read-only register so values outside of the - * supported mappings can be considered an implementation error. - */ - assert(parange < ARRAY_SIZE(pamax_map)); - return pamax_map[parange]; -} - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index e4b860d2aee8..d754273fa162 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -23,6 +23,31 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) __attribute__((nonnull)); +/* This mapping is common between ID_AA64MMFR0.PARANGE and TCR_ELx.{I}PS. */ +static const uint8_t pamax_map[] = { + [0] = 32, + [1] = 36, + [2] = 40, + [3] = 42, + [4] = 44, + [5] = 48, + [6] = 52, +}; + +/* The cpu-specific constant value of PAMax; also used by hw/arm/virt. */ +unsigned int arm_pamax(ARMCPU *cpu) +{ + unsigned int parange = + FIELD_EX64(cpu->isar.id_aa64mmfr0, ID_AA64MMFR0, PARANGE); + + /* + * id_aa64mmfr0 is a read-only register so values outside of the + * supported mappings can be considered an implementation error. + */ + assert(parange < ARRAY_SIZE(pamax_map)); + return pamax_map[parange]; +} + static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) { return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 28b8cb9fb896..fba650d01cae 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,8 +11,6 @@ #ifndef CONFIG_USER_ONLY -extern const uint8_t pamax_map[7]; - bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); From f8526edc2f32b66a585618afc1dabd960a4c9b95 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 29/90] target/arm: Move get_S1prot, get_S2prot to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-20-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 128 -------------------------------------------- target/arm/ptw.c | 128 ++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 -- 3 files changed, 128 insertions(+), 131 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 563e34ecded1..148eb28ba8cb 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10615,134 +10615,6 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) } } -/* Translate S2 section/page access permissions to protection flags - * - * @env: CPUARMState - * @s2ap: The 2-bit stage2 access permissions (S2AP) - * @xn: XN (execute-never) bits - * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 - */ -int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) -{ - int prot = 0; - - if (s2ap & 1) { - prot |= PAGE_READ; - } - if (s2ap & 2) { - prot |= PAGE_WRITE; - } - - if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) { - switch (xn) { - case 0: - prot |= PAGE_EXEC; - break; - case 1: - if (s1_is_el0) { - prot |= PAGE_EXEC; - } - break; - case 2: - break; - case 3: - if (!s1_is_el0) { - prot |= PAGE_EXEC; - } - break; - default: - g_assert_not_reached(); - } - } else { - if (!extract32(xn, 1, 1)) { - if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) { - prot |= PAGE_EXEC; - } - } - } - return prot; -} - -/* Translate section/page access permissions to protection flags - * - * @env: CPUARMState - * @mmu_idx: MMU index indicating required translation regime - * @is_aa64: TRUE if AArch64 - * @ap: The 2-bit simple AP (AP[2:1]) - * @ns: NS (non-secure) bit - * @xn: XN (execute-never) bit - * @pxn: PXN (privileged execute-never) bit - */ -int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn) -{ - bool is_user = regime_is_user(env, mmu_idx); - int prot_rw, user_rw; - bool have_wxn; - int wxn = 0; - - assert(mmu_idx != ARMMMUIdx_Stage2); - assert(mmu_idx != ARMMMUIdx_Stage2_S); - - user_rw = simple_ap_to_rw_prot_is_user(ap, true); - if (is_user) { - prot_rw = user_rw; - } else { - if (user_rw && regime_is_pan(env, mmu_idx)) { - /* PAN forbids data accesses but doesn't affect insn fetch */ - prot_rw = 0; - } else { - prot_rw = simple_ap_to_rw_prot_is_user(ap, false); - } - } - - if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) { - return prot_rw; - } - - /* TODO have_wxn should be replaced with - * ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2) - * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE - * compatible processors have EL2, which is required for [U]WXN. - */ - have_wxn = arm_feature(env, ARM_FEATURE_LPAE); - - if (have_wxn) { - wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN; - } - - if (is_aa64) { - if (regime_has_2_ranges(mmu_idx) && !is_user) { - xn = pxn || (user_rw & PAGE_WRITE); - } - } else if (arm_feature(env, ARM_FEATURE_V7)) { - switch (regime_el(env, mmu_idx)) { - case 1: - case 3: - if (is_user) { - xn = xn || !(user_rw & PAGE_READ); - } else { - int uwxn = 0; - if (have_wxn) { - uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN; - } - xn = xn || !(prot_rw & PAGE_READ) || pxn || - (uwxn && (user_rw & PAGE_WRITE)); - } - break; - case 2: - break; - } - } else { - xn = wxn = 0; - } - - if (xn || (wxn && (prot_rw & PAGE_WRITE))) { - return prot_rw; - } - return prot_rw | PAGE_EXEC; -} - /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.c b/target/arm/ptw.c index d754273fa162..af9ad420288e 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -487,6 +487,134 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, return true; } +/* + * Translate S2 section/page access permissions to protection flags + * @env: CPUARMState + * @s2ap: The 2-bit stage2 access permissions (S2AP) + * @xn: XN (execute-never) bits + * @s1_is_el0: true if this is S2 of an S1+2 walk for EL0 + */ +static int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0) +{ + int prot = 0; + + if (s2ap & 1) { + prot |= PAGE_READ; + } + if (s2ap & 2) { + prot |= PAGE_WRITE; + } + + if (cpu_isar_feature(any_tts2uxn, env_archcpu(env))) { + switch (xn) { + case 0: + prot |= PAGE_EXEC; + break; + case 1: + if (s1_is_el0) { + prot |= PAGE_EXEC; + } + break; + case 2: + break; + case 3: + if (!s1_is_el0) { + prot |= PAGE_EXEC; + } + break; + default: + g_assert_not_reached(); + } + } else { + if (!extract32(xn, 1, 1)) { + if (arm_el_is_aa64(env, 2) || prot & PAGE_READ) { + prot |= PAGE_EXEC; + } + } + } + return prot; +} + +/* + * Translate section/page access permissions to protection flags + * @env: CPUARMState + * @mmu_idx: MMU index indicating required translation regime + * @is_aa64: TRUE if AArch64 + * @ap: The 2-bit simple AP (AP[2:1]) + * @ns: NS (non-secure) bit + * @xn: XN (execute-never) bit + * @pxn: PXN (privileged execute-never) bit + */ +static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, + int ap, int ns, int xn, int pxn) +{ + bool is_user = regime_is_user(env, mmu_idx); + int prot_rw, user_rw; + bool have_wxn; + int wxn = 0; + + assert(mmu_idx != ARMMMUIdx_Stage2); + assert(mmu_idx != ARMMMUIdx_Stage2_S); + + user_rw = simple_ap_to_rw_prot_is_user(ap, true); + if (is_user) { + prot_rw = user_rw; + } else { + if (user_rw && regime_is_pan(env, mmu_idx)) { + /* PAN forbids data accesses but doesn't affect insn fetch */ + prot_rw = 0; + } else { + prot_rw = simple_ap_to_rw_prot_is_user(ap, false); + } + } + + if (ns && arm_is_secure(env) && (env->cp15.scr_el3 & SCR_SIF)) { + return prot_rw; + } + + /* TODO have_wxn should be replaced with + * ARM_FEATURE_V8 || (ARM_FEATURE_V7 && ARM_FEATURE_EL2) + * when ARM_FEATURE_EL2 starts getting set. For now we assume all LPAE + * compatible processors have EL2, which is required for [U]WXN. + */ + have_wxn = arm_feature(env, ARM_FEATURE_LPAE); + + if (have_wxn) { + wxn = regime_sctlr(env, mmu_idx) & SCTLR_WXN; + } + + if (is_aa64) { + if (regime_has_2_ranges(mmu_idx) && !is_user) { + xn = pxn || (user_rw & PAGE_WRITE); + } + } else if (arm_feature(env, ARM_FEATURE_V7)) { + switch (regime_el(env, mmu_idx)) { + case 1: + case 3: + if (is_user) { + xn = xn || !(user_rw & PAGE_READ); + } else { + int uwxn = 0; + if (have_wxn) { + uwxn = regime_sctlr(env, mmu_idx) & SCTLR_UWXN; + } + xn = xn || !(prot_rw & PAGE_READ) || pxn || + (uwxn && (user_rw & PAGE_WRITE)); + } + break; + case 2: + break; + } + } else { + xn = wxn = 0; + } + + if (xn || (wxn && (prot_rw & PAGE_WRITE))) { + return prot_rw; + } + return prot_rw | PAGE_EXEC; +} + /** * get_phys_addr_lpae: perform one stage of page table walk, LPAE format * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index fba650d01cae..93147e0b0656 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -29,9 +29,6 @@ ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, ARMMMUIdx mmu_idx); bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, int inputsize, int stride, int outputsize); -int get_S2prot(CPUARMState *env, int s2ap, int xn, bool s1_is_el0); -int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, - int ap, int ns, int xn, int pxn); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From c5168785d208527c45f455619abe2539182a9fff Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:52 +0100 Subject: [PATCH 30/90] target/arm: Move check_s2_mmu_setup to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-21-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 70 --------------------------------------------- target/arm/ptw.c | 70 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 2 -- 3 files changed, 70 insertions(+), 72 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 148eb28ba8cb..2526f4c6c4a8 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10614,76 +10614,6 @@ int simple_ap_to_rw_prot_is_user(int ap, bool is_user) g_assert_not_reached(); } } - -/* - * check_s2_mmu_setup - * @cpu: ARMCPU - * @is_aa64: True if the translation regime is in AArch64 state - * @startlevel: Suggested starting level - * @inputsize: Bitsize of IPAs - * @stride: Page-table stride (See the ARM ARM) - * - * Returns true if the suggested S2 translation parameters are OK and - * false otherwise. - */ -bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize) -{ - const int grainsize = stride + 3; - int startsizecheck; - - /* - * Negative levels are usually not allowed... - * Except for FEAT_LPA2, 4k page table, 52-bit address space, which - * begins with level -1. Note that previous feature tests will have - * eliminated this combination if it is not enabled. - */ - if (level < (inputsize == 52 && stride == 9 ? -1 : 0)) { - return false; - } - - startsizecheck = inputsize - ((3 - level) * stride + grainsize); - if (startsizecheck < 1 || startsizecheck > stride + 4) { - return false; - } - - if (is_aa64) { - switch (stride) { - case 13: /* 64KB Pages. */ - if (level == 0 || (level == 1 && outputsize <= 42)) { - return false; - } - break; - case 11: /* 16KB Pages. */ - if (level == 0 || (level == 1 && outputsize <= 40)) { - return false; - } - break; - case 9: /* 4KB Pages. */ - if (level == 0 && outputsize <= 42) { - return false; - } - break; - default: - g_assert_not_reached(); - } - - /* Inputsize checks. */ - if (inputsize > outputsize && - (arm_el_is_aa64(&cpu->env, 1) || inputsize > 40)) { - /* This is CONSTRAINED UNPREDICTABLE and we choose to fault. */ - return false; - } - } else { - /* AArch32 only supports 4KB pages. Assert on that. */ - assert(stride == 9); - - if (level == 0) { - return false; - } - } - return true; -} #endif /* !CONFIG_USER_ONLY */ int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index af9ad420288e..525272e99afe 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -615,6 +615,76 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } +/* + * check_s2_mmu_setup + * @cpu: ARMCPU + * @is_aa64: True if the translation regime is in AArch64 state + * @startlevel: Suggested starting level + * @inputsize: Bitsize of IPAs + * @stride: Page-table stride (See the ARM ARM) + * + * Returns true if the suggested S2 translation parameters are OK and + * false otherwise. + */ +static bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, + int inputsize, int stride, int outputsize) +{ + const int grainsize = stride + 3; + int startsizecheck; + + /* + * Negative levels are usually not allowed... + * Except for FEAT_LPA2, 4k page table, 52-bit address space, which + * begins with level -1. Note that previous feature tests will have + * eliminated this combination if it is not enabled. + */ + if (level < (inputsize == 52 && stride == 9 ? -1 : 0)) { + return false; + } + + startsizecheck = inputsize - ((3 - level) * stride + grainsize); + if (startsizecheck < 1 || startsizecheck > stride + 4) { + return false; + } + + if (is_aa64) { + switch (stride) { + case 13: /* 64KB Pages. */ + if (level == 0 || (level == 1 && outputsize <= 42)) { + return false; + } + break; + case 11: /* 16KB Pages. */ + if (level == 0 || (level == 1 && outputsize <= 40)) { + return false; + } + break; + case 9: /* 4KB Pages. */ + if (level == 0 && outputsize <= 42) { + return false; + } + break; + default: + g_assert_not_reached(); + } + + /* Inputsize checks. */ + if (inputsize > outputsize && + (arm_el_is_aa64(&cpu->env, 1) || inputsize > 40)) { + /* This is CONSTRAINED UNPREDICTABLE and we choose to fault. */ + return false; + } + } else { + /* AArch32 only supports 4KB pages. Assert on that. */ + assert(stride == 9); + + if (level == 0) { + return false; + } + } + return true; +} + /** * get_phys_addr_lpae: perform one stage of page table walk, LPAE format * diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 93147e0b0656..a71161b01bd4 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -27,8 +27,6 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, ARMMMUIdx mmu_idx); -bool check_s2_mmu_setup(ARMCPU *cpu, bool is_aa64, int level, - int inputsize, int stride, int outputsize); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 2f0ec92e9488e1ac6c7293fe37a5f1c0e1737b36 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 31/90] target/arm: Move aa32_va_parameters to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-22-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 64 --------------------------------------------- target/arm/ptw.c | 64 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 3 --- 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 2526f4c6c4a8..f61f1da61e4f 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10771,70 +10771,6 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, } #ifndef CONFIG_USER_ONLY -ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx) -{ - uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; - uint32_t el = regime_el(env, mmu_idx); - int select, tsz; - bool epd, hpd; - - assert(mmu_idx != ARMMMUIdx_Stage2_S); - - if (mmu_idx == ARMMMUIdx_Stage2) { - /* VTCR */ - bool sext = extract32(tcr, 4, 1); - bool sign = extract32(tcr, 3, 1); - - /* - * If the sign-extend bit is not the same as t0sz[3], the result - * is unpredictable. Flag this as a guest error. - */ - if (sign != sext) { - qemu_log_mask(LOG_GUEST_ERROR, - "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n"); - } - tsz = sextract32(tcr, 0, 4) + 8; - select = 0; - hpd = false; - epd = false; - } else if (el == 2) { - /* HTCR */ - tsz = extract32(tcr, 0, 3); - select = 0; - hpd = extract64(tcr, 24, 1); - epd = false; - } else { - int t0sz = extract32(tcr, 0, 3); - int t1sz = extract32(tcr, 16, 3); - - if (t1sz == 0) { - select = va > (0xffffffffu >> t0sz); - } else { - /* Note that we will detect errors later. */ - select = va >= ~(0xffffffffu >> t1sz); - } - if (!select) { - tsz = t0sz; - epd = extract32(tcr, 7, 1); - hpd = extract64(tcr, 41, 1); - } else { - tsz = t1sz; - epd = extract32(tcr, 23, 1); - hpd = extract64(tcr, 42, 1); - } - /* For aarch32, hpd0 is not enabled without t2e as well. */ - hpd &= extract32(tcr, 6, 1); - } - - return (ARMVAParameters) { - .tsz = tsz, - .select = select, - .epd = epd, - .hpd = hpd, - }; -} - hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, MemTxAttrs *attrs) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 525272e99afe..427813ea5632 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -615,6 +615,70 @@ static int get_S1prot(CPUARMState *env, ARMMMUIdx mmu_idx, bool is_aa64, return prot_rw | PAGE_EXEC; } +static ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, + ARMMMUIdx mmu_idx) +{ + uint64_t tcr = regime_tcr(env, mmu_idx)->raw_tcr; + uint32_t el = regime_el(env, mmu_idx); + int select, tsz; + bool epd, hpd; + + assert(mmu_idx != ARMMMUIdx_Stage2_S); + + if (mmu_idx == ARMMMUIdx_Stage2) { + /* VTCR */ + bool sext = extract32(tcr, 4, 1); + bool sign = extract32(tcr, 3, 1); + + /* + * If the sign-extend bit is not the same as t0sz[3], the result + * is unpredictable. Flag this as a guest error. + */ + if (sign != sext) { + qemu_log_mask(LOG_GUEST_ERROR, + "AArch32: VTCR.S / VTCR.T0SZ[3] mismatch\n"); + } + tsz = sextract32(tcr, 0, 4) + 8; + select = 0; + hpd = false; + epd = false; + } else if (el == 2) { + /* HTCR */ + tsz = extract32(tcr, 0, 3); + select = 0; + hpd = extract64(tcr, 24, 1); + epd = false; + } else { + int t0sz = extract32(tcr, 0, 3); + int t1sz = extract32(tcr, 16, 3); + + if (t1sz == 0) { + select = va > (0xffffffffu >> t0sz); + } else { + /* Note that we will detect errors later. */ + select = va >= ~(0xffffffffu >> t1sz); + } + if (!select) { + tsz = t0sz; + epd = extract32(tcr, 7, 1); + hpd = extract64(tcr, 41, 1); + } else { + tsz = t1sz; + epd = extract32(tcr, 23, 1); + hpd = extract64(tcr, 42, 1); + } + /* For aarch32, hpd0 is not enabled without t2e as well. */ + hpd &= extract32(tcr, 6, 1); + } + + return (ARMVAParameters) { + .tsz = tsz, + .select = select, + .epd = epd, + .hpd = hpd, + }; +} + /* * check_s2_mmu_setup * @cpu: ARMCPU diff --git a/target/arm/ptw.h b/target/arm/ptw.h index a71161b01bd4..9314fb4d23c5 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -25,8 +25,5 @@ simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); } -ARMVAParameters aa32_va_parameters(CPUARMState *env, uint32_t va, - ARMMMUIdx mmu_idx); - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 4845d3be1239a956a4cedd5c2ca7863837487927 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 32/90] target/arm: Move ap_to_tw_prot etc to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-23-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 77 ------------------------------------------ target/arm/ptw.c | 81 +++++++++++++++++++++++++++++++++++++++++++++ target/arm/ptw.h | 10 ------ 3 files changed, 81 insertions(+), 87 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index f61f1da61e4f..e894afcb491d 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10537,83 +10537,6 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) g_assert_not_reached(); } } - -/* Translate section/page access permissions to page - * R/W protection flags - * - * @env: CPUARMState - * @mmu_idx: MMU index indicating required translation regime - * @ap: The 3-bit access permissions (AP[2:0]) - * @domain_prot: The 2-bit domain access permissions - */ -int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap, int domain_prot) -{ - bool is_user = regime_is_user(env, mmu_idx); - - if (domain_prot == 3) { - return PAGE_READ | PAGE_WRITE; - } - - switch (ap) { - case 0: - if (arm_feature(env, ARM_FEATURE_V7)) { - return 0; - } - switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) { - case SCTLR_S: - return is_user ? 0 : PAGE_READ; - case SCTLR_R: - return PAGE_READ; - default: - return 0; - } - case 1: - return is_user ? 0 : PAGE_READ | PAGE_WRITE; - case 2: - if (is_user) { - return PAGE_READ; - } else { - return PAGE_READ | PAGE_WRITE; - } - case 3: - return PAGE_READ | PAGE_WRITE; - case 4: /* Reserved. */ - return 0; - case 5: - return is_user ? 0 : PAGE_READ; - case 6: - return PAGE_READ; - case 7: - if (!arm_feature(env, ARM_FEATURE_V6K)) { - return 0; - } - return PAGE_READ; - default: - g_assert_not_reached(); - } -} - -/* Translate section/page access permissions to page - * R/W protection flags. - * - * @ap: The 2-bit simple AP (AP[2:1]) - * @is_user: TRUE if accessing from PL0 - */ -int simple_ap_to_rw_prot_is_user(int ap, bool is_user) -{ - switch (ap) { - case 0: - return is_user ? 0 : PAGE_READ | PAGE_WRITE; - case 1: - return PAGE_READ | PAGE_WRITE; - case 2: - return is_user ? 0 : PAGE_READ; - case 3: - return PAGE_READ; - default: - g_assert_not_reached(); - } -} #endif /* !CONFIG_USER_ONLY */ int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 427813ea5632..9ab77c399807 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -211,6 +211,87 @@ static bool get_level1_table_address(CPUARMState *env, ARMMMUIdx mmu_idx, return true; } +/* + * Translate section/page access permissions to page R/W protection flags + * @env: CPUARMState + * @mmu_idx: MMU index indicating required translation regime + * @ap: The 3-bit access permissions (AP[2:0]) + * @domain_prot: The 2-bit domain access permissions + */ +static int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, + int ap, int domain_prot) +{ + bool is_user = regime_is_user(env, mmu_idx); + + if (domain_prot == 3) { + return PAGE_READ | PAGE_WRITE; + } + + switch (ap) { + case 0: + if (arm_feature(env, ARM_FEATURE_V7)) { + return 0; + } + switch (regime_sctlr(env, mmu_idx) & (SCTLR_S | SCTLR_R)) { + case SCTLR_S: + return is_user ? 0 : PAGE_READ; + case SCTLR_R: + return PAGE_READ; + default: + return 0; + } + case 1: + return is_user ? 0 : PAGE_READ | PAGE_WRITE; + case 2: + if (is_user) { + return PAGE_READ; + } else { + return PAGE_READ | PAGE_WRITE; + } + case 3: + return PAGE_READ | PAGE_WRITE; + case 4: /* Reserved. */ + return 0; + case 5: + return is_user ? 0 : PAGE_READ; + case 6: + return PAGE_READ; + case 7: + if (!arm_feature(env, ARM_FEATURE_V6K)) { + return 0; + } + return PAGE_READ; + default: + g_assert_not_reached(); + } +} + +/* + * Translate section/page access permissions to page R/W protection flags. + * @ap: The 2-bit simple AP (AP[2:1]) + * @is_user: TRUE if accessing from PL0 + */ +static int simple_ap_to_rw_prot_is_user(int ap, bool is_user) +{ + switch (ap) { + case 0: + return is_user ? 0 : PAGE_READ | PAGE_WRITE; + case 1: + return PAGE_READ | PAGE_WRITE; + case 2: + return is_user ? 0 : PAGE_READ; + case 3: + return PAGE_READ; + default: + g_assert_not_reached(); + } +} + +static int simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) +{ + return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); +} + static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, MMUAccessType access_type, ARMMMUIdx mmu_idx, hwaddr *phys_ptr, int *prot, diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 9314fb4d23c5..85ad5767944a 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -15,15 +15,5 @@ bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); -int ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, - int ap, int domain_prot); -int simple_ap_to_rw_prot_is_user(int ap, bool is_user); - -static inline int -simple_ap_to_rw_prot(CPUARMState *env, ARMMMUIdx mmu_idx, int ap) -{ - return simple_ap_to_rw_prot_is_user(ap, regime_is_user(env, mmu_idx)); -} - #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 0c23d56fc149ada0823648481c09b5d2047570f4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 33/90] target/arm: Move regime_is_user to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-24-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 24 ------------------------ target/arm/ptw.c | 22 ++++++++++++++++++++++ target/arm/ptw.h | 1 - 3 files changed, 22 insertions(+), 25 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index e894afcb491d..8deb0fa94c16 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10515,30 +10515,6 @@ ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) } #endif /* !CONFIG_USER_ONLY */ -#ifndef CONFIG_USER_ONLY -bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - switch (mmu_idx) { - case ARMMMUIdx_SE10_0: - case ARMMMUIdx_E20_0: - case ARMMMUIdx_SE20_0: - case ARMMMUIdx_Stage1_E0: - case ARMMMUIdx_Stage1_SE0: - case ARMMMUIdx_MUser: - case ARMMMUIdx_MSUser: - case ARMMMUIdx_MUserNegPri: - case ARMMMUIdx_MSUserNegPri: - return true; - default: - return false; - case ARMMMUIdx_E10_0: - case ARMMMUIdx_E10_1: - case ARMMMUIdx_E10_1_PAN: - g_assert_not_reached(); - } -} -#endif /* !CONFIG_USER_ONLY */ - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 9ab77c399807..8db4b5edf1ae 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -53,6 +53,28 @@ static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; } +static bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_SE10_0: + case ARMMMUIdx_E20_0: + case ARMMMUIdx_SE20_0: + case ARMMMUIdx_Stage1_E0: + case ARMMMUIdx_Stage1_SE0: + case ARMMMUIdx_MUser: + case ARMMMUIdx_MSUser: + case ARMMMUIdx_MUserNegPri: + case ARMMMUIdx_MSUserNegPri: + return true; + default: + return false; + case ARMMMUIdx_E10_0: + case ARMMMUIdx_E10_1: + case ARMMMUIdx_E10_1_PAN: + g_assert_not_reached(); + } +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 85ad5767944a..3d3061a43514 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -11,7 +11,6 @@ #ifndef CONFIG_USER_ONLY -bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx); bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); From 3b318aaeef120a71544771cd84884ccad5fdff0a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 34/90] target/arm: Move regime_ttbr to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-25-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 16 ---------------- target/arm/ptw.c | 16 ++++++++++++++++ target/arm/ptw.h | 1 - 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 8deb0fa94c16..fdda87e87e25 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10475,22 +10475,6 @@ bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; } -/* Return the TTBR associated with this translation regime */ -uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) -{ - if (mmu_idx == ARMMMUIdx_Stage2) { - return env->cp15.vttbr_el2; - } - if (mmu_idx == ARMMMUIdx_Stage2_S) { - return env->cp15.vsttbr_el2; - } - if (ttbrn == 0) { - return env->cp15.ttbr0_el[regime_el(env, mmu_idx)]; - } else { - return env->cp15.ttbr1_el[regime_el(env, mmu_idx)]; - } -} - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 8db4b5edf1ae..dc559e6bdfd3 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -75,6 +75,22 @@ static bool regime_is_user(CPUARMState *env, ARMMMUIdx mmu_idx) } } +/* Return the TTBR associated with this translation regime */ +static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) +{ + if (mmu_idx == ARMMMUIdx_Stage2) { + return env->cp15.vttbr_el2; + } + if (mmu_idx == ARMMMUIdx_Stage2_S) { + return env->cp15.vsttbr_el2; + } + if (ttbrn == 0) { + return env->cp15.ttbr0_el[regime_el(env, mmu_idx)]; + } else { + return env->cp15.ttbr1_el[regime_el(env, mmu_idx)]; + } +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h index 3d3061a43514..ed152ddaf4e4 100644 --- a/target/arm/ptw.h +++ b/target/arm/ptw.h @@ -12,7 +12,6 @@ #ifndef CONFIG_USER_ONLY bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); -uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn); #endif /* !CONFIG_USER_ONLY */ #endif /* TARGET_ARM_PTW_H */ From 8db1a3a0bba1dbe71a3d363b23416a3c30653ddd Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:53 +0100 Subject: [PATCH 35/90] target/arm: Move regime_translation_disabled to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-26-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 47 --------------------------------------------- target/arm/ptw.c | 47 ++++++++++++++++++++++++++++++++++++++++++++- target/arm/ptw.h | 17 ---------------- 3 files changed, 46 insertions(+), 65 deletions(-) delete mode 100644 target/arm/ptw.h diff --git a/target/arm/helper.c b/target/arm/helper.c index fdda87e87e25..69b1c060c1f2 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -36,7 +36,6 @@ #include "semihosting/common-semi.h" #endif #include "cpregs.h" -#include "ptw.h" #define ARM_CPU_FREQ 1000000000 /* FIXME: 1 GHz, should be configurable */ @@ -10429,52 +10428,6 @@ uint64_t arm_sctlr(CPUARMState *env, int el) } #ifndef CONFIG_USER_ONLY - -/* Return true if the specified stage of address translation is disabled */ -bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) -{ - uint64_t hcr_el2; - - if (arm_feature(env, ARM_FEATURE_M)) { - switch (env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] & - (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) { - case R_V7M_MPU_CTRL_ENABLE_MASK: - /* Enabled, but not for HardFault and NMI */ - return mmu_idx & ARM_MMU_IDX_M_NEGPRI; - case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK: - /* Enabled for all cases */ - return false; - case 0: - default: - /* HFNMIENA set and ENABLE clear is UNPREDICTABLE, but - * we warned about that in armv7m_nvic.c when the guest set it. - */ - return true; - } - } - - hcr_el2 = arm_hcr_el2_eff(env); - - if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { - /* HCR.DC means HCR.VM behaves as 1 */ - return (hcr_el2 & (HCR_DC | HCR_VM)) == 0; - } - - if (hcr_el2 & HCR_TGE) { - /* TGE means that NS EL0/1 act as if SCTLR_EL1.M is zero */ - if (!regime_is_secure(env, mmu_idx) && regime_el(env, mmu_idx) == 1) { - return true; - } - } - - if ((hcr_el2 & HCR_DC) && arm_mmu_idx_is_stage1_of_2(mmu_idx)) { - /* HCR.DC means SCTLR_EL1.M behaves as 0 */ - return true; - } - - return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; -} - /* Convert a possible stage1+2 MMU index into the appropriate * stage 1 MMU index */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index dc559e6bdfd3..ec60afd9bff5 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -12,7 +12,6 @@ #include "cpu.h" #include "internals.h" #include "idau.h" -#include "ptw.h" static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, @@ -91,6 +90,52 @@ static uint64_t regime_ttbr(CPUARMState *env, ARMMMUIdx mmu_idx, int ttbrn) } } +/* Return true if the specified stage of address translation is disabled */ +static bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx) +{ + uint64_t hcr_el2; + + if (arm_feature(env, ARM_FEATURE_M)) { + switch (env->v7m.mpu_ctrl[regime_is_secure(env, mmu_idx)] & + (R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK)) { + case R_V7M_MPU_CTRL_ENABLE_MASK: + /* Enabled, but not for HardFault and NMI */ + return mmu_idx & ARM_MMU_IDX_M_NEGPRI; + case R_V7M_MPU_CTRL_ENABLE_MASK | R_V7M_MPU_CTRL_HFNMIENA_MASK: + /* Enabled for all cases */ + return false; + case 0: + default: + /* + * HFNMIENA set and ENABLE clear is UNPREDICTABLE, but + * we warned about that in armv7m_nvic.c when the guest set it. + */ + return true; + } + } + + hcr_el2 = arm_hcr_el2_eff(env); + + if (mmu_idx == ARMMMUIdx_Stage2 || mmu_idx == ARMMMUIdx_Stage2_S) { + /* HCR.DC means HCR.VM behaves as 1 */ + return (hcr_el2 & (HCR_DC | HCR_VM)) == 0; + } + + if (hcr_el2 & HCR_TGE) { + /* TGE means that NS EL0/1 act as if SCTLR_EL1.M is zero */ + if (!regime_is_secure(env, mmu_idx) && regime_el(env, mmu_idx) == 1) { + return true; + } + } + + if ((hcr_el2 & HCR_DC) && arm_mmu_idx_is_stage1_of_2(mmu_idx)) { + /* HCR.DC means SCTLR_EL1.M behaves as 0 */ + return true; + } + + return (regime_sctlr(env, mmu_idx) & SCTLR_M) == 0; +} + static bool ptw_attrs_are_device(CPUARMState *env, ARMCacheAttrs cacheattrs) { /* diff --git a/target/arm/ptw.h b/target/arm/ptw.h deleted file mode 100644 index ed152ddaf4e4..000000000000 --- a/target/arm/ptw.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * ARM page table walking. - * - * This code is licensed under the GNU GPL v2 or later. - * - * SPDX-License-Identifier: GPL-2.0-or-later - */ - -#ifndef TARGET_ARM_PTW_H -#define TARGET_ARM_PTW_H - -#ifndef CONFIG_USER_ONLY - -bool regime_translation_disabled(CPUARMState *env, ARMMMUIdx mmu_idx); - -#endif /* !CONFIG_USER_ONLY */ -#endif /* TARGET_ARM_PTW_H */ From 23971205cf38e63964b7e110df7d5a293277a48e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 36/90] target/arm: Move arm_cpu_get_phys_page_attrs_debug to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-27-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 26 -------------------------- target/arm/ptw.c | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 69b1c060c1f2..fe1e426f883a 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10606,32 +10606,6 @@ ARMVAParameters aa64_va_parameters(CPUARMState *env, uint64_t va, }; } -#ifndef CONFIG_USER_ONLY -hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, - MemTxAttrs *attrs) -{ - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; - hwaddr phys_addr; - target_ulong page_size; - int prot; - bool ret; - ARMMMUFaultInfo fi = {}; - ARMMMUIdx mmu_idx = arm_mmu_idx(env); - ARMCacheAttrs cacheattrs = {}; - - *attrs = (MemTxAttrs) {}; - - ret = get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &phys_addr, - attrs, &prot, &page_size, &fi, &cacheattrs); - - if (ret) { - return -1; - } - return phys_addr; -} -#endif - /* Note that signed overflow is undefined in C. The following routines are careful to use unsigned types where modulo arithmetic is required. Failure to do so _will_ break on newer gcc. */ diff --git a/target/arm/ptw.c b/target/arm/ptw.c index ec60afd9bff5..e9f6870d0a6a 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -2491,3 +2491,27 @@ bool get_phys_addr(CPUARMState *env, target_ulong address, phys_ptr, prot, page_size, fi); } } + +hwaddr arm_cpu_get_phys_page_attrs_debug(CPUState *cs, vaddr addr, + MemTxAttrs *attrs) +{ + ARMCPU *cpu = ARM_CPU(cs); + CPUARMState *env = &cpu->env; + hwaddr phys_addr; + target_ulong page_size; + int prot; + bool ret; + ARMMMUFaultInfo fi = {}; + ARMMMUIdx mmu_idx = arm_mmu_idx(env); + ARMCacheAttrs cacheattrs = {}; + + *attrs = (MemTxAttrs) {}; + + ret = get_phys_addr(env, addr, MMU_DATA_LOAD, mmu_idx, &phys_addr, + attrs, &prot, &page_size, &fi, &cacheattrs); + + if (ret) { + return -1; + } + return phys_addr; +} From 1d2612553640a79e049708b9d87a9cd3ddb29af6 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 37/90] target/arm: Move stage_1_mmu_idx, arm_stage1_mmu_idx to ptw.c Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-28-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/helper.c | 32 -------------------------------- target/arm/ptw.c | 28 ++++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index fe1e426f883a..37cf9fa6aba1 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -10427,31 +10427,6 @@ uint64_t arm_sctlr(CPUARMState *env, int el) return env->cp15.sctlr_el[el]; } -#ifndef CONFIG_USER_ONLY -/* Convert a possible stage1+2 MMU index into the appropriate - * stage 1 MMU index - */ -ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) -{ - switch (mmu_idx) { - case ARMMMUIdx_SE10_0: - return ARMMMUIdx_Stage1_SE0; - case ARMMMUIdx_SE10_1: - return ARMMMUIdx_Stage1_SE1; - case ARMMMUIdx_SE10_1_PAN: - return ARMMMUIdx_Stage1_SE1_PAN; - case ARMMMUIdx_E10_0: - return ARMMMUIdx_Stage1_E0; - case ARMMMUIdx_E10_1: - return ARMMMUIdx_Stage1_E1; - case ARMMMUIdx_E10_1_PAN: - return ARMMMUIdx_Stage1_E1_PAN; - default: - return mmu_idx; - } -} -#endif /* !CONFIG_USER_ONLY */ - int aa64_va_parameter_tbi(uint64_t tcr, ARMMMUIdx mmu_idx) { if (regime_has_2_ranges(mmu_idx)) { @@ -11081,13 +11056,6 @@ ARMMMUIdx arm_mmu_idx(CPUARMState *env) return arm_mmu_idx_el(env, arm_current_el(env)); } -#ifndef CONFIG_USER_ONLY -ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) -{ - return stage_1_mmu_idx(arm_mmu_idx(env)); -} -#endif - static CPUARMTBFlags rebuild_hflags_common(CPUARMState *env, int fp_el, ARMMMUIdx mmu_idx, CPUARMTBFlags flags) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index e9f6870d0a6a..49e9a1d108e8 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -47,6 +47,34 @@ unsigned int arm_pamax(ARMCPU *cpu) return pamax_map[parange]; } +/* + * Convert a possible stage1+2 MMU index into the appropriate stage 1 MMU index + */ +ARMMMUIdx stage_1_mmu_idx(ARMMMUIdx mmu_idx) +{ + switch (mmu_idx) { + case ARMMMUIdx_SE10_0: + return ARMMMUIdx_Stage1_SE0; + case ARMMMUIdx_SE10_1: + return ARMMMUIdx_Stage1_SE1; + case ARMMMUIdx_SE10_1_PAN: + return ARMMMUIdx_Stage1_SE1_PAN; + case ARMMMUIdx_E10_0: + return ARMMMUIdx_Stage1_E0; + case ARMMMUIdx_E10_1: + return ARMMMUIdx_Stage1_E1; + case ARMMMUIdx_E10_1_PAN: + return ARMMMUIdx_Stage1_E1_PAN; + default: + return mmu_idx; + } +} + +ARMMMUIdx arm_stage1_mmu_idx(CPUARMState *env) +{ + return stage_1_mmu_idx(arm_mmu_idx(env)); +} + static bool regime_translation_big_endian(CPUARMState *env, ARMMMUIdx mmu_idx) { return (regime_sctlr(env, mmu_idx) & SCTLR_EE) != 0; From 5e79887ba67e22bfd890b72f94c482176a200fbc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 38/90] target/arm: Pass CPUARMState to arm_ld[lq]_ptw The use of ARM_CPU to recover env from cs calls object_class_dynamic_cast, which shows up on the profile. This is pointless, because all callers already have env, and the reverse operation, env_cpu, is only pointer arithmetic. Signed-off-by: Richard Henderson Message-id: 20220604040607.269301-29-richard.henderson@linaro.org Reviewed-by: Peter Maydell Signed-off-by: Peter Maydell --- target/arm/ptw.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/target/arm/ptw.c b/target/arm/ptw.c index 49e9a1d108e8..4d97a2480849 100644 --- a/target/arm/ptw.c +++ b/target/arm/ptw.c @@ -241,11 +241,10 @@ static hwaddr S1_ptw_translate(CPUARMState *env, ARMMMUIdx mmu_idx, } /* All loads done in the course of a page table walk go through here. */ -static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, +static uint32_t arm_ldl_ptw(CPUARMState *env, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUState *cs = env_cpu(env); MemTxAttrs attrs = {}; MemTxResult result = MEMTX_OK; AddressSpace *as; @@ -270,11 +269,10 @@ static uint32_t arm_ldl_ptw(CPUState *cs, hwaddr addr, bool is_secure, return 0; } -static uint64_t arm_ldq_ptw(CPUState *cs, hwaddr addr, bool is_secure, +static uint64_t arm_ldq_ptw(CPUARMState *env, hwaddr addr, bool is_secure, ARMMMUIdx mmu_idx, ARMMMUFaultInfo *fi) { - ARMCPU *cpu = ARM_CPU(cs); - CPUARMState *env = &cpu->env; + CPUState *cs = env_cpu(env); MemTxAttrs attrs = {}; MemTxResult result = MEMTX_OK; AddressSpace *as; @@ -409,7 +407,6 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, target_ulong *page_size, ARMMMUFaultInfo *fi) { - CPUState *cs = env_cpu(env); int level = 1; uint32_t table; uint32_t desc; @@ -427,7 +424,7 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, fi->type = ARMFault_Translation; goto do_fault; } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -466,7 +463,7 @@ static bool get_phys_addr_v5(CPUARMState *env, uint32_t address, /* Fine pagetable. */ table = (desc & 0xfffff000) | ((address >> 8) & 0xffc); } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -531,7 +528,6 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, hwaddr *phys_ptr, MemTxAttrs *attrs, int *prot, target_ulong *page_size, ARMMMUFaultInfo *fi) { - CPUState *cs = env_cpu(env); ARMCPU *cpu = env_archcpu(env); int level = 1; uint32_t table; @@ -553,7 +549,7 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, fi->type = ARMFault_Translation; goto do_fault; } - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -607,7 +603,7 @@ static bool get_phys_addr_v6(CPUARMState *env, uint32_t address, ns = extract32(desc, 3, 1); /* Lookup l2 entry. */ table = (desc & 0xfffffc00) | ((address >> 10) & 0x3fc); - desc = arm_ldl_ptw(cs, table, regime_is_secure(env, mmu_idx), + desc = arm_ldl_ptw(env, table, regime_is_secure(env, mmu_idx), mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; @@ -973,7 +969,6 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, ARMMMUFaultInfo *fi, ARMCacheAttrs *cacheattrs) { ARMCPU *cpu = env_archcpu(env); - CPUState *cs = CPU(cpu); /* Read an LPAE long-descriptor translation table. */ ARMFaultType fault_type = ARMFault_Translation; uint32_t level; @@ -1196,7 +1191,7 @@ static bool get_phys_addr_lpae(CPUARMState *env, uint64_t address, descaddr |= (address >> (stride * (4 - level))) & indexmask; descaddr &= ~7ULL; nstable = extract32(tableattrs, 4, 1); - descriptor = arm_ldq_ptw(cs, descaddr, !nstable, mmu_idx, fi); + descriptor = arm_ldq_ptw(env, descaddr, !nstable, mmu_idx, fi); if (fi->type != ARMFault_None) { goto do_fault; } From f45ce4c35f5e0873bbbc3119eff8539610233b7e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:54 +0100 Subject: [PATCH 39/90] target/arm: Rename TBFLAG_A64 ZCR_LEN to VL With SME, the vector length does not only come from ZCR_ELx. Comment that this is either NVL or SVL, like the pseudocode. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-2-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 3 ++- target/arm/helper.c | 2 +- target/arm/translate-a64.c | 2 +- target/arm/translate-a64.h | 2 +- target/arm/translate-sve.c | 2 +- target/arm/translate.h | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 0ee1705a4fae..e791ffdd6b62 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3241,7 +3241,8 @@ FIELD(TBFLAG_M32, MVE_NO_PRED, 5, 1) /* Not cached. */ */ FIELD(TBFLAG_A64, TBII, 0, 2) FIELD(TBFLAG_A64, SVEEXC_EL, 2, 2) -FIELD(TBFLAG_A64, ZCR_LEN, 4, 4) +/* The current vector length, either NVL or SVL. */ +FIELD(TBFLAG_A64, VL, 4, 4) FIELD(TBFLAG_A64, PAUTH_ACTIVE, 8, 1) FIELD(TBFLAG_A64, BT, 9, 1) FIELD(TBFLAG_A64, BTYPE, 10, 2) /* Not cached. */ diff --git a/target/arm/helper.c b/target/arm/helper.c index 37cf9fa6aba1..c228deca7550 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -11181,7 +11181,7 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, zcr_len = sve_zcr_len_for_el(env, el); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); - DP_TBFLAG_A64(flags, ZCR_LEN, zcr_len); + DP_TBFLAG_A64(flags, VL, zcr_len); } sctlr = regime_sctlr(env, stage1); diff --git a/target/arm/translate-a64.c b/target/arm/translate-a64.c index 935e1929bb93..d438fb89e73d 100644 --- a/target/arm/translate-a64.c +++ b/target/arm/translate-a64.c @@ -14608,7 +14608,7 @@ static void aarch64_tr_init_disas_context(DisasContextBase *dcbase, dc->align_mem = EX_TBFLAG_ANY(tb_flags, ALIGN_MEM); dc->pstate_il = EX_TBFLAG_ANY(tb_flags, PSTATE__IL); dc->sve_excp_el = EX_TBFLAG_A64(tb_flags, SVEEXC_EL); - dc->sve_len = (EX_TBFLAG_A64(tb_flags, ZCR_LEN) + 1) * 16; + dc->vl = (EX_TBFLAG_A64(tb_flags, VL) + 1) * 16; dc->pauth_active = EX_TBFLAG_A64(tb_flags, PAUTH_ACTIVE); dc->bt = EX_TBFLAG_A64(tb_flags, BT); dc->btype = EX_TBFLAG_A64(tb_flags, BTYPE); diff --git a/target/arm/translate-a64.h b/target/arm/translate-a64.h index f2e8ee0ee1f0..dbc917ee65bb 100644 --- a/target/arm/translate-a64.h +++ b/target/arm/translate-a64.h @@ -104,7 +104,7 @@ static inline TCGv_ptr vec_full_reg_ptr(DisasContext *s, int regno) /* Return the byte size of the "whole" vector register, VL / 8. */ static inline int vec_full_reg_size(DisasContext *s) { - return s->sve_len; + return s->vl; } bool disas_sve(DisasContext *, uint32_t); diff --git a/target/arm/translate-sve.c b/target/arm/translate-sve.c index 836511d71917..67761bf2cc55 100644 --- a/target/arm/translate-sve.c +++ b/target/arm/translate-sve.c @@ -111,7 +111,7 @@ static inline int pred_full_reg_offset(DisasContext *s, int regno) /* Return the byte size of the whole predicate register, VL / 64. */ static inline int pred_full_reg_size(DisasContext *s) { - return s->sve_len >> 3; + return s->vl >> 3; } /* Round up the size of a register to a size allowed by diff --git a/target/arm/translate.h b/target/arm/translate.h index 9f0bb270c5b5..f473a21ed484 100644 --- a/target/arm/translate.h +++ b/target/arm/translate.h @@ -42,7 +42,7 @@ typedef struct DisasContext { bool ns; /* Use non-secure CPREG bank on access */ int fp_excp_el; /* FP exception EL or 0 if enabled */ int sve_excp_el; /* SVE exception EL or 0 if enabled */ - int sve_len; /* SVE vector length in bytes */ + int vl; /* current vector length in bytes */ /* Flag indicating that exceptions from secure mode are routed to EL3. */ bool secure_routed_to_el3; bool vfp_enabled; /* FP enabled via FPSCR.EN */ From 8b599e5c0274cd5e4a33be6a10d6cd04631a74d1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 40/90] linux-user/aarch64: Introduce sve_vq Add an interface function to extract the digested vector length rather than the raw zcr_el[1] value. This fixes an incorrect return from do_prctl_set_vl where we didn't take into account the set of vector lengths supported by the cpu. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-3-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- linux-user/aarch64/signal.c | 4 ++-- linux-user/aarch64/target_prctl.h | 18 ++++++++++++------ target/arm/cpu.h | 11 +++++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/linux-user/aarch64/signal.c b/linux-user/aarch64/signal.c index 7de4c96eb9d0..7da0e36c6d47 100644 --- a/linux-user/aarch64/signal.c +++ b/linux-user/aarch64/signal.c @@ -315,7 +315,7 @@ static int target_restore_sigframe(CPUARMState *env, case TARGET_SVE_MAGIC: if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { - vq = (env->vfp.zcr_el[1] & 0xf) + 1; + vq = sve_vq(env); sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16); if (!sve && size == sve_size) { sve = (struct target_sve_context *)ctx; @@ -434,7 +434,7 @@ static void target_setup_frame(int usig, struct target_sigaction *ka, /* SVE state needs saving only if it exists. */ if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { - vq = (env->vfp.zcr_el[1] & 0xf) + 1; + vq = sve_vq(env); sve_size = QEMU_ALIGN_UP(TARGET_SVE_SIG_CONTEXT_SIZE(vq), 16); sve_ofs = alloc_sigframe_space(sve_size, &layout); } diff --git a/linux-user/aarch64/target_prctl.h b/linux-user/aarch64/target_prctl.h index 3f5a5d3933a0..1d440ffbea4b 100644 --- a/linux-user/aarch64/target_prctl.h +++ b/linux-user/aarch64/target_prctl.h @@ -10,7 +10,7 @@ static abi_long do_prctl_get_vl(CPUArchState *env) { ARMCPU *cpu = env_archcpu(env); if (cpu_isar_feature(aa64_sve, cpu)) { - return ((cpu->env.vfp.zcr_el[1] & 0xf) + 1) * 16; + return sve_vq(env) * 16; } return -TARGET_EINVAL; } @@ -25,18 +25,24 @@ static abi_long do_prctl_set_vl(CPUArchState *env, abi_long arg2) */ if (cpu_isar_feature(aa64_sve, env_archcpu(env)) && arg2 >= 0 && arg2 <= 512 * 16 && !(arg2 & 15)) { - ARMCPU *cpu = env_archcpu(env); uint32_t vq, old_vq; - old_vq = (env->vfp.zcr_el[1] & 0xf) + 1; + old_vq = sve_vq(env); + + /* + * Bound the value of arg2, so that we know that it fits into + * the 4-bit field in ZCR_EL1. Rely on the hflags rebuild to + * sort out the length supported by the cpu. + */ vq = MAX(arg2 / 16, 1); - vq = MIN(vq, cpu->sve_max_vq); + vq = MIN(vq, ARM_MAX_VQ); + env->vfp.zcr_el[1] = vq - 1; + arm_rebuild_hflags(env); + vq = sve_vq(env); if (vq < old_vq) { aarch64_sve_narrow_vq(env, vq); } - env->vfp.zcr_el[1] = vq - 1; - arm_rebuild_hflags(env); return vq * 16; } return -TARGET_EINVAL; diff --git a/target/arm/cpu.h b/target/arm/cpu.h index e791ffdd6b62..f5af88b686da 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -3286,6 +3286,17 @@ static inline int cpu_mmu_index(CPUARMState *env, bool ifetch) return EX_TBFLAG_ANY(env->hflags, MMUIDX); } +/** + * sve_vq + * @env: the cpu context + * + * Return the VL cached within env->hflags, in units of quadwords. + */ +static inline int sve_vq(CPUARMState *env) +{ + return EX_TBFLAG_A64(env->hflags, VL) + 1; +} + static inline bool bswap_code(bool sctlr_b) { #ifdef CONFIG_USER_ONLY From 61a8c23a3b56607ee6e745b4cf7975170df88801 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 41/90] target/arm: Remove route_to_el2 check from sve_exception_el We handle this routing in raise_exception. Promoting the value early means that we can't directly compare FPEXC_EL and SVEEXC_EL. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-4-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index c228deca7550..1bd77af7e500 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6149,8 +6149,7 @@ int sve_exception_el(CPUARMState *env, int el) /* fall through */ case 0: case 2: - /* route_to_el2 */ - return hcr_el2 & HCR_TGE ? 2 : 1; + return 1; } /* Check CPACR.FPEN. */ From 397d922c6248509c6d490f82088f00cbc716287c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 42/90] target/arm: Remove fp checks from sve_exception_el Instead of checking these bits in fp_exception_el and also in sve_exception_el, document that we must compare the results. The only place where we have not already checked that FP EL is zero is in rebuild_hflags_a64. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-5-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 58 +++++++++++++++------------------------------ 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 1bd77af7e500..4f4044c688db 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6129,11 +6129,15 @@ static const ARMCPRegInfo minimal_ras_reginfo[] = { .access = PL2_RW, .fieldoffset = offsetof(CPUARMState, cp15.vsesr_el2) }, }; -/* Return the exception level to which exceptions should be taken - * via SVEAccessTrap. If an exception should be routed through - * AArch64.AdvSIMDFPAccessTrap, return 0; fp_exception_el should - * take care of raising that exception. - * C.f. the ARM pseudocode function CheckSVEEnabled. +/* + * Return the exception level to which exceptions should be taken + * via SVEAccessTrap. This excludes the check for whether the exception + * should be routed through AArch64.AdvSIMDFPAccessTrap. That can easily + * be found by testing 0 < fp_exception_el < sve_exception_el. + * + * C.f. the ARM pseudocode function CheckSVEEnabled. Note that the + * pseudocode does *not* separate out the FP trap checks, but has them + * all in one function. */ int sve_exception_el(CPUARMState *env, int el) { @@ -6151,18 +6155,6 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 1; } - - /* Check CPACR.FPEN. */ - switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, FPEN)) { - case 1: - if (el != 0) { - break; - } - /* fall through */ - case 0: - case 2: - return 0; - } } /* @@ -6180,24 +6172,10 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 2; } - - switch (FIELD_EX32(env->cp15.cptr_el[2], CPTR_EL2, FPEN)) { - case 1: - if (el == 2 || !(hcr_el2 & HCR_TGE)) { - break; - } - /* fall through */ - case 0: - case 2: - return 0; - } } else if (arm_is_el2_enabled(env)) { if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TZ)) { return 2; } - if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TFP)) { - return 0; - } } } @@ -11168,19 +11146,21 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, if (cpu_isar_feature(aa64_sve, env_archcpu(env))) { int sve_el = sve_exception_el(env, el); - uint32_t zcr_len; /* - * If SVE is disabled, but FP is enabled, - * then the effective len is 0. + * If either FP or SVE are disabled, translator does not need len. + * If SVE EL > FP EL, FP exception has precedence, and translator + * does not need SVE EL. Save potential re-translations by forcing + * the unneeded data to zero. */ - if (sve_el != 0 && fp_el == 0) { - zcr_len = 0; - } else { - zcr_len = sve_zcr_len_for_el(env, el); + if (fp_el != 0) { + if (sve_el > fp_el) { + sve_el = 0; + } + } else if (sve_el == 0) { + DP_TBFLAG_A64(flags, VL, sve_zcr_len_for_el(env, el)); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); - DP_TBFLAG_A64(flags, VL, zcr_len); } sctlr = regime_sctlr(env, stage1); From 19668718ad188e1b6a162bb52357a67fd407c96c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:55 +0100 Subject: [PATCH 43/90] target/arm: Add el_is_in_host This (newish) ARM pseudocode function is easier to work with than open-coded tests for HCR_E2H etc. Use of the function will be staged into the code base in parts. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-6-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 28 ++++++++++++++++++++++++++++ target/arm/internals.h | 2 ++ 2 files changed, 30 insertions(+) diff --git a/target/arm/helper.c b/target/arm/helper.c index 4f4044c688db..322508170e3b 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -5282,6 +5282,34 @@ uint64_t arm_hcr_el2_eff(CPUARMState *env) return ret; } +/* + * Corresponds to ARM pseudocode function ELIsInHost(). + */ +bool el_is_in_host(CPUARMState *env, int el) +{ + uint64_t mask; + + /* + * Since we only care about E2H and TGE, we can skip arm_hcr_el2_eff(). + * Perform the simplest bit tests first, and validate EL2 afterward. + */ + if (el & 1) { + return false; /* EL1 or EL3 */ + } + + /* + * Note that hcr_write() checks isar_feature_aa64_vh(), + * aka HaveVirtHostExt(), in allowing HCR_E2H to be set. + */ + mask = el ? HCR_E2H : HCR_E2H | HCR_TGE; + if ((env->cp15.hcr_el2 & mask) != mask) { + return false; + } + + /* TGE and/or E2H set: double check those bits are currently legal. */ + return arm_is_el2_enabled(env) && arm_el_is_aa64(env, 2); +} + static void hcrx_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { diff --git a/target/arm/internals.h b/target/arm/internals.h index 1d83146d565e..ceaddcbfd6ea 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1347,6 +1347,8 @@ static inline void define_cortex_a72_a57_a53_cp_reginfo(ARMCPU *cpu) { } void define_cortex_a72_a57_a53_cp_reginfo(ARMCPU *cpu); #endif +bool el_is_in_host(CPUARMState *env, int el); + void aa32_max_features(ARMCPU *cpu); #endif From c6225bebc2498e9dd85bdd4de1840925650c7a9f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 44/90] target/arm: Use el_is_in_host for sve_zcr_len_for_el The ARM pseudocode function NVL uses this predicate now, and I think it's a bit clearer. Simplify the pseudocode condition by noting that IsInHost is always false for EL1. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-7-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 322508170e3b..6b17c0f507f5 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6238,8 +6238,7 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) ARMCPU *cpu = env_archcpu(env); uint32_t zcr_len = cpu->sve_max_vq - 1; - if (el <= 1 && - (arm_hcr_el2_eff(env) & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + if (el <= 1 && !el_is_in_host(env, el)) { zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[1]); } if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) { From aa4451b60e6a2b1bc4e9a8f6bbf90b266e2a4390 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 45/90] target/arm: Use el_is_in_host for sve_exception_el The ARM pseudocode function CheckNormalSVEEnabled uses this predicate now, and I think it's a bit clearer. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-8-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 6b17c0f507f5..40b60b1eea2d 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6170,9 +6170,7 @@ static const ARMCPRegInfo minimal_ras_reginfo[] = { int sve_exception_el(CPUARMState *env, int el) { #ifndef CONFIG_USER_ONLY - uint64_t hcr_el2 = arm_hcr_el2_eff(env); - - if (el <= 1 && (hcr_el2 & (HCR_E2H | HCR_TGE)) != (HCR_E2H | HCR_TGE)) { + if (el <= 1 && !el_is_in_host(env, el)) { switch (FIELD_EX64(env->cp15.cpacr_el1, CPACR_EL1, ZEN)) { case 1: if (el != 0) { @@ -6189,6 +6187,7 @@ int sve_exception_el(CPUARMState *env, int el) * CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */ if (el <= 2) { + uint64_t hcr_el2 = arm_hcr_el2_eff(env); if (hcr_el2 & HCR_E2H) { switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, ZEN)) { case 1: From 7d38cb92aad848b9c935588815d0b3a5e157aabb Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 46/90] target/arm: Hoist arm_is_el2_enabled check in sve_exception_el This check is buried within arm_hcr_el2_eff(), but since we have to have the explicit check for CPTR_EL2.TZ, we might as well just check it once at the beginning of the block. Once this is done, we can test HCR_EL2.{E2H,TGE} directly, rather than going through arm_hcr_el2_eff(). Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-9-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 40b60b1eea2d..61e8026d0e36 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6183,15 +6183,12 @@ int sve_exception_el(CPUARMState *env, int el) } } - /* - * CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). - */ - if (el <= 2) { - uint64_t hcr_el2 = arm_hcr_el2_eff(env); - if (hcr_el2 & HCR_E2H) { + if (el <= 2 && arm_is_el2_enabled(env)) { + /* CPTR_EL2 changes format with HCR_EL2.E2H (regardless of TGE). */ + if (env->cp15.hcr_el2 & HCR_E2H) { switch (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, ZEN)) { case 1: - if (el != 0 || !(hcr_el2 & HCR_TGE)) { + if (el != 0 || !(env->cp15.hcr_el2 & HCR_TGE)) { break; } /* fall through */ @@ -6199,7 +6196,7 @@ int sve_exception_el(CPUARMState *env, int el) case 2: return 2; } - } else if (arm_is_el2_enabled(env)) { + } else { if (FIELD_EX64(env->cp15.cptr_el[2], CPTR_EL2, TZ)) { return 2; } From 87252bdecdafd49f8f4da6244a40415c27053bc3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:56 +0100 Subject: [PATCH 47/90] target/arm: Do not use aarch64_sve_zcr_get_valid_len in reset We don't need to constrain the value set in zcr_el[1], because it will be done by sve_zcr_len_for_el. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-10-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index d2bd74c2ed42..062194416748 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -208,8 +208,7 @@ static void arm_cpu_reset(DeviceState *dev) CPACR_EL1, ZEN, 3); /* with reasonable vector length */ if (cpu_isar_feature(aa64_sve, cpu)) { - env->vfp.zcr_el[1] = - aarch64_sve_zcr_get_valid_len(cpu, cpu->sve_default_vq - 1); + env->vfp.zcr_el[1] = cpu->sve_default_vq - 1; } /* * Enable 48-bit address space (TODO: take reserved_va into account). From 9b5f422559a528f4117aa9c3098fddf3d7c535ba Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 48/90] target/arm: Merge aarch64_sve_zcr_get_valid_len into caller This function is used only once, and will need modification for Streaming SVE mode. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-11-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/helper.c | 30 +++++++++++------------------- target/arm/internals.h | 11 ----------- 2 files changed, 11 insertions(+), 30 deletions(-) diff --git a/target/arm/helper.c b/target/arm/helper.c index 61e8026d0e36..de159c644cd8 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6212,39 +6212,31 @@ int sve_exception_el(CPUARMState *env, int el) return 0; } -uint32_t aarch64_sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len) -{ - uint32_t end_len; - - start_len = MIN(start_len, ARM_MAX_VQ - 1); - end_len = start_len; - - if (!test_bit(start_len, cpu->sve_vq_map)) { - end_len = find_last_bit(cpu->sve_vq_map, start_len); - assert(end_len < start_len); - } - return end_len; -} - /* * Given that SVE is enabled, return the vector length for EL. */ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); - uint32_t zcr_len = cpu->sve_max_vq - 1; + uint32_t len = cpu->sve_max_vq - 1; + uint32_t end_len; if (el <= 1 && !el_is_in_host(env, el)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[1]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[1]); } if (el <= 2 && arm_feature(env, ARM_FEATURE_EL2)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[2]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[2]); } if (arm_feature(env, ARM_FEATURE_EL3)) { - zcr_len = MIN(zcr_len, 0xf & (uint32_t)env->vfp.zcr_el[3]); + len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[3]); } - return aarch64_sve_zcr_get_valid_len(cpu, zcr_len); + end_len = len; + if (!test_bit(len, cpu->sve_vq_map)) { + end_len = find_last_bit(cpu->sve_vq_map, len); + assert(end_len < len); + } + return end_len; } static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, diff --git a/target/arm/internals.h b/target/arm/internals.h index ceaddcbfd6ea..79eb46375382 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -189,17 +189,6 @@ void arm_translate_init(void); void arm_cpu_synchronize_from_tb(CPUState *cs, const TranslationBlock *tb); #endif /* CONFIG_TCG */ -/** - * aarch64_sve_zcr_get_valid_len: - * @cpu: cpu context - * @start_len: maximum len to consider - * - * Return the maximum supported sve vector length <= @start_len. - * Note that both @start_len and the return value are in units - * of ZCR_ELx.LEN, so the vector bit length is (x + 1) * 128. - */ -uint32_t aarch64_sve_zcr_get_valid_len(ARMCPU *cpu, uint32_t start_len); - enum arm_fprounding { FPROUNDING_TIEEVEN, FPROUNDING_POSINF, From 886902ece71b5e795fea3e052a32f047d2f8fe33 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 49/90] target/arm: Use uint32_t instead of bitmap for sve vq's The bitmap need only hold 15 bits; bitmap is over-complicated. We can simplify operations quite a bit with plain logical ops. The introduction of SVE_VQ_POW2_MAP eliminates the need for looping in order to search for powers of two. Simply perform the logical ops and use count leading or trailing zeros as required to find the result. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-12-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 6 +-- target/arm/cpu64.c | 117 ++++++++++++++++++++--------------------- target/arm/helper.c | 9 +--- target/arm/internals.h | 5 ++ target/arm/kvm64.c | 36 +++---------- target/arm/kvm_arm.h | 7 ++- 6 files changed, 75 insertions(+), 105 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index f5af88b686da..73f24a57603f 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1041,9 +1041,9 @@ struct ArchCPU { * Bits set in sve_vq_supported represent valid vector lengths for * the CPU type. */ - DECLARE_BITMAP(sve_vq_map, ARM_MAX_VQ); - DECLARE_BITMAP(sve_vq_init, ARM_MAX_VQ); - DECLARE_BITMAP(sve_vq_supported, ARM_MAX_VQ); + uint32_t sve_vq_map; + uint32_t sve_vq_init; + uint32_t sve_vq_supported; /* Generic timer counter frequency, in Hz */ uint64_t gt_cntfrq_hz; diff --git a/target/arm/cpu64.c b/target/arm/cpu64.c index cce68dd82a23..15665c962b2d 100644 --- a/target/arm/cpu64.c +++ b/target/arm/cpu64.c @@ -355,8 +355,11 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * any of the above. Finally, if SVE is not disabled, then at least one * vector length must be enabled. */ - DECLARE_BITMAP(tmp, ARM_MAX_VQ); - uint32_t vq, max_vq = 0; + uint32_t vq_map = cpu->sve_vq_map; + uint32_t vq_init = cpu->sve_vq_init; + uint32_t vq_supported; + uint32_t vq_mask = 0; + uint32_t tmp, vq, max_vq = 0; /* * CPU models specify a set of supported vector lengths which are @@ -364,10 +367,16 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * in the supported bitmap results in an error. When KVM is enabled we * fetch the supported bitmap from the host. */ - if (kvm_enabled() && kvm_arm_sve_supported()) { - kvm_arm_sve_get_vls(CPU(cpu), cpu->sve_vq_supported); - } else if (kvm_enabled()) { - assert(!cpu_isar_feature(aa64_sve, cpu)); + if (kvm_enabled()) { + if (kvm_arm_sve_supported()) { + cpu->sve_vq_supported = kvm_arm_sve_get_vls(CPU(cpu)); + vq_supported = cpu->sve_vq_supported; + } else { + assert(!cpu_isar_feature(aa64_sve, cpu)); + vq_supported = 0; + } + } else { + vq_supported = cpu->sve_vq_supported; } /* @@ -375,8 +384,9 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * From the properties, sve_vq_map implies sve_vq_init. * Check first for any sve enabled. */ - if (!bitmap_empty(cpu->sve_vq_map, ARM_MAX_VQ)) { - max_vq = find_last_bit(cpu->sve_vq_map, ARM_MAX_VQ) + 1; + if (vq_map != 0) { + max_vq = 32 - clz32(vq_map); + vq_mask = MAKE_64BIT_MASK(0, max_vq); if (cpu->sve_max_vq && max_vq > cpu->sve_max_vq) { error_setg(errp, "cannot enable sve%d", max_vq * 128); @@ -392,15 +402,10 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * For KVM we have to automatically enable all supported unitialized * lengths, even when the smaller lengths are not all powers-of-two. */ - bitmap_andnot(tmp, cpu->sve_vq_supported, cpu->sve_vq_init, max_vq); - bitmap_or(cpu->sve_vq_map, cpu->sve_vq_map, tmp, max_vq); + vq_map |= vq_supported & ~vq_init & vq_mask; } else { /* Propagate enabled bits down through required powers-of-two. */ - for (vq = pow2floor(max_vq); vq >= 1; vq >>= 1) { - if (!test_bit(vq - 1, cpu->sve_vq_init)) { - set_bit(vq - 1, cpu->sve_vq_map); - } - } + vq_map |= SVE_VQ_POW2_MAP & ~vq_init & vq_mask; } } else if (cpu->sve_max_vq == 0) { /* @@ -413,25 +418,18 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) if (kvm_enabled()) { /* Disabling a supported length disables all larger lengths. */ - for (vq = 1; vq <= ARM_MAX_VQ; ++vq) { - if (test_bit(vq - 1, cpu->sve_vq_init) && - test_bit(vq - 1, cpu->sve_vq_supported)) { - break; - } - } + tmp = vq_init & vq_supported; } else { /* Disabling a power-of-two disables all larger lengths. */ - for (vq = 1; vq <= ARM_MAX_VQ; vq <<= 1) { - if (test_bit(vq - 1, cpu->sve_vq_init)) { - break; - } - } + tmp = vq_init & SVE_VQ_POW2_MAP; } + vq = ctz32(tmp) + 1; max_vq = vq <= ARM_MAX_VQ ? vq - 1 : ARM_MAX_VQ; - bitmap_andnot(cpu->sve_vq_map, cpu->sve_vq_supported, - cpu->sve_vq_init, max_vq); - if (max_vq == 0 || bitmap_empty(cpu->sve_vq_map, max_vq)) { + vq_mask = MAKE_64BIT_MASK(0, max_vq); + vq_map = vq_supported & ~vq_init & vq_mask; + + if (max_vq == 0 || vq_map == 0) { error_setg(errp, "cannot disable sve%d", vq * 128); error_append_hint(errp, "Disabling sve%d results in all " "vector lengths being disabled.\n", @@ -441,7 +439,8 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) return; } - max_vq = find_last_bit(cpu->sve_vq_map, max_vq) + 1; + max_vq = 32 - clz32(vq_map); + vq_mask = MAKE_64BIT_MASK(0, max_vq); } /* @@ -451,9 +450,9 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) */ if (cpu->sve_max_vq != 0) { max_vq = cpu->sve_max_vq; + vq_mask = MAKE_64BIT_MASK(0, max_vq); - if (!test_bit(max_vq - 1, cpu->sve_vq_map) && - test_bit(max_vq - 1, cpu->sve_vq_init)) { + if (vq_init & ~vq_map & (1 << (max_vq - 1))) { error_setg(errp, "cannot disable sve%d", max_vq * 128); error_append_hint(errp, "The maximum vector length must be " "enabled, sve-max-vq=%d (%d bits)\n", @@ -462,8 +461,7 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) } /* Set all bits not explicitly set within sve-max-vq. */ - bitmap_complement(tmp, cpu->sve_vq_init, max_vq); - bitmap_or(cpu->sve_vq_map, cpu->sve_vq_map, tmp, max_vq); + vq_map |= ~vq_init & vq_mask; } /* @@ -472,13 +470,14 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) * are clear, just in case anybody looks. */ assert(max_vq != 0); - bitmap_clear(cpu->sve_vq_map, max_vq, ARM_MAX_VQ - max_vq); + assert(vq_mask != 0); + vq_map &= vq_mask; /* Ensure the set of lengths matches what is supported. */ - bitmap_xor(tmp, cpu->sve_vq_map, cpu->sve_vq_supported, max_vq); - if (!bitmap_empty(tmp, max_vq)) { - vq = find_last_bit(tmp, max_vq) + 1; - if (test_bit(vq - 1, cpu->sve_vq_map)) { + tmp = vq_map ^ (vq_supported & vq_mask); + if (tmp) { + vq = 32 - clz32(tmp); + if (vq_map & (1 << (vq - 1))) { if (cpu->sve_max_vq) { error_setg(errp, "cannot set sve-max-vq=%d", cpu->sve_max_vq); error_append_hint(errp, "This CPU does not support " @@ -502,15 +501,15 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) return; } else { /* Ensure all required powers-of-two are enabled. */ - for (vq = pow2floor(max_vq); vq >= 1; vq >>= 1) { - if (!test_bit(vq - 1, cpu->sve_vq_map)) { - error_setg(errp, "cannot disable sve%d", vq * 128); - error_append_hint(errp, "sve%d is required as it " - "is a power-of-two length smaller " - "than the maximum, sve%d\n", - vq * 128, max_vq * 128); - return; - } + tmp = SVE_VQ_POW2_MAP & vq_mask & ~vq_map; + if (tmp) { + vq = 32 - clz32(tmp); + error_setg(errp, "cannot disable sve%d", vq * 128); + error_append_hint(errp, "sve%d is required as it " + "is a power-of-two length smaller " + "than the maximum, sve%d\n", + vq * 128, max_vq * 128); + return; } } } @@ -530,6 +529,7 @@ void arm_cpu_sve_finalize(ARMCPU *cpu, Error **errp) /* From now on sve_max_vq is the actual maximum supported length. */ cpu->sve_max_vq = max_vq; + cpu->sve_vq_map = vq_map; } static void cpu_max_get_sve_max_vq(Object *obj, Visitor *v, const char *name, @@ -590,7 +590,7 @@ static void cpu_arm_get_sve_vq(Object *obj, Visitor *v, const char *name, if (!cpu_isar_feature(aa64_sve, cpu)) { value = false; } else { - value = test_bit(vq - 1, cpu->sve_vq_map); + value = extract32(cpu->sve_vq_map, vq - 1, 1); } visit_type_bool(v, name, &value, errp); } @@ -612,12 +612,8 @@ static void cpu_arm_set_sve_vq(Object *obj, Visitor *v, const char *name, return; } - if (value) { - set_bit(vq - 1, cpu->sve_vq_map); - } else { - clear_bit(vq - 1, cpu->sve_vq_map); - } - set_bit(vq - 1, cpu->sve_vq_init); + cpu->sve_vq_map = deposit32(cpu->sve_vq_map, vq - 1, 1, value); + cpu->sve_vq_init |= 1 << (vq - 1); } static bool cpu_arm_get_sve(Object *obj, Error **errp) @@ -979,7 +975,7 @@ static void aarch64_max_initfn(Object *obj) cpu->dcz_blocksize = 7; /* 512 bytes */ #endif - bitmap_fill(cpu->sve_vq_supported, ARM_MAX_VQ); + cpu->sve_vq_supported = MAKE_64BIT_MASK(0, ARM_MAX_VQ); aarch64_add_pauth_properties(obj); aarch64_add_sve_properties(obj); @@ -1026,12 +1022,11 @@ static void aarch64_a64fx_initfn(Object *obj) cpu->gic_vprebits = 5; cpu->gic_pribits = 5; - /* Suppport of A64FX's vector length are 128,256 and 512bit only */ + /* The A64FX supports only 128, 256 and 512 bit vector lengths */ aarch64_add_sve_properties(obj); - bitmap_zero(cpu->sve_vq_supported, ARM_MAX_VQ); - set_bit(0, cpu->sve_vq_supported); /* 128bit */ - set_bit(1, cpu->sve_vq_supported); /* 256bit */ - set_bit(3, cpu->sve_vq_supported); /* 512bit */ + cpu->sve_vq_supported = (1 << 0) /* 128bit */ + | (1 << 1) /* 256bit */ + | (1 << 3); /* 512bit */ cpu->isar.reset_pmcr_el0 = 0x46014040; diff --git a/target/arm/helper.c b/target/arm/helper.c index de159c644cd8..90aac6bc12d1 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6219,7 +6219,6 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); uint32_t len = cpu->sve_max_vq - 1; - uint32_t end_len; if (el <= 1 && !el_is_in_host(env, el)) { len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[1]); @@ -6231,12 +6230,8 @@ uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) len = MIN(len, 0xf & (uint32_t)env->vfp.zcr_el[3]); } - end_len = len; - if (!test_bit(len, cpu->sve_vq_map)) { - end_len = find_last_bit(cpu->sve_vq_map, len); - assert(end_len < len); - } - return end_len; + len = 31 - clz32(cpu->sve_vq_map & MAKE_64BIT_MASK(0, len + 1)); + return len; } static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, diff --git a/target/arm/internals.h b/target/arm/internals.h index 79eb46375382..a1bae4588aed 100644 --- a/target/arm/internals.h +++ b/target/arm/internals.h @@ -1340,4 +1340,9 @@ bool el_is_in_host(CPUARMState *env, int el); void aa32_max_features(ARMCPU *cpu); +/* Powers of 2 for sve_vq_map et al. */ +#define SVE_VQ_POW2_MAP \ + ((1 << (1 - 1)) | (1 << (2 - 1)) | \ + (1 << (4 - 1)) | (1 << (8 - 1)) | (1 << (16 - 1))) + #endif diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index 363032da9034..b3f635fc952b 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -760,15 +760,13 @@ bool kvm_arm_steal_time_supported(void) QEMU_BUILD_BUG_ON(KVM_ARM64_SVE_VQ_MIN != 1); -void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) +uint32_t kvm_arm_sve_get_vls(CPUState *cs) { /* Only call this function if kvm_arm_sve_supported() returns true. */ static uint64_t vls[KVM_ARM64_SVE_VLS_WORDS]; static bool probed; uint32_t vq = 0; - int i, j; - - bitmap_zero(map, ARM_MAX_VQ); + int i; /* * KVM ensures all host CPUs support the same set of vector lengths. @@ -809,46 +807,24 @@ void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) if (vq > ARM_MAX_VQ) { warn_report("KVM supports vector lengths larger than " "QEMU can enable"); + vls[0] &= MAKE_64BIT_MASK(0, ARM_MAX_VQ); } } - for (i = 0; i < KVM_ARM64_SVE_VLS_WORDS; ++i) { - if (!vls[i]) { - continue; - } - for (j = 1; j <= 64; ++j) { - vq = j + i * 64; - if (vq > ARM_MAX_VQ) { - return; - } - if (vls[i] & (1UL << (j - 1))) { - set_bit(vq - 1, map); - } - } - } + return vls[0]; } static int kvm_arm_sve_set_vls(CPUState *cs) { - uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = {0}; + ARMCPU *cpu = ARM_CPU(cs); + uint64_t vls[KVM_ARM64_SVE_VLS_WORDS] = { cpu->sve_vq_map }; struct kvm_one_reg reg = { .id = KVM_REG_ARM64_SVE_VLS, .addr = (uint64_t)&vls[0], }; - ARMCPU *cpu = ARM_CPU(cs); - uint32_t vq; - int i, j; assert(cpu->sve_max_vq <= KVM_ARM64_SVE_VQ_MAX); - for (vq = 1; vq <= cpu->sve_max_vq; ++vq) { - if (test_bit(vq - 1, cpu->sve_vq_map)) { - i = (vq - 1) / 64; - j = (vq - 1) % 64; - vls[i] |= 1UL << j; - } - } - return kvm_vcpu_ioctl(cs, KVM_SET_ONE_REG, ®); } diff --git a/target/arm/kvm_arm.h b/target/arm/kvm_arm.h index b7f78b521545..99017b635ce4 100644 --- a/target/arm/kvm_arm.h +++ b/target/arm/kvm_arm.h @@ -239,13 +239,12 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf); /** * kvm_arm_sve_get_vls: * @cs: CPUState - * @map: bitmap to fill in * * Get all the SVE vector lengths supported by the KVM host, setting * the bits corresponding to their length in quadwords minus one - * (vq - 1) in @map up to ARM_MAX_VQ. + * (vq - 1) up to ARM_MAX_VQ. Return the resulting map. */ -void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map); +uint32_t kvm_arm_sve_get_vls(CPUState *cs); /** * kvm_arm_set_cpu_features_from_host: @@ -439,7 +438,7 @@ static inline void kvm_arm_steal_time_finalize(ARMCPU *cpu, Error **errp) g_assert_not_reached(); } -static inline void kvm_arm_sve_get_vls(CPUState *cs, unsigned long *map) +static inline uint32_t kvm_arm_sve_get_vls(CPUState *cs) { g_assert_not_reached(); } From 5ef3cc563699d9b96e6b1acb15c7c02cf75d8266 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 50/90] target/arm: Rename sve_zcr_len_for_el to sve_vqm1_for_el This will be used for both Normal and Streaming SVE, and the value does not necessarily come from ZCR_ELx. While we're at it, emphasize the units in which the value is returned. Patch produced by git grep -l sve_zcr_len_for_el | \ xargs -n1 sed -i 's/sve_zcr_len_for_el/sve_vqm1_for_el/g' and then adding a function comment. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-13-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/arch_dump.c | 2 +- target/arm/cpu.c | 2 +- target/arm/cpu.h | 11 ++++++++++- target/arm/gdbstub64.c | 2 +- target/arm/helper.c | 12 ++++++------ 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/target/arm/arch_dump.c b/target/arm/arch_dump.c index 018484531091..b1f040e69f22 100644 --- a/target/arm/arch_dump.c +++ b/target/arm/arch_dump.c @@ -166,7 +166,7 @@ static off_t sve_fpcr_offset(uint32_t vq) static uint32_t sve_current_vq(CPUARMState *env) { - return sve_zcr_len_for_el(env, arm_current_el(env)) + 1; + return sve_vqm1_for_el(env, arm_current_el(env)) + 1; } static size_t sve_size_vq(uint32_t vq) diff --git a/target/arm/cpu.c b/target/arm/cpu.c index 062194416748..1b5d53578800 100644 --- a/target/arm/cpu.c +++ b/target/arm/cpu.c @@ -925,7 +925,7 @@ static void aarch64_cpu_dump_state(CPUState *cs, FILE *f, int flags) vfp_get_fpcr(env), vfp_get_fpsr(env)); if (cpu_isar_feature(aa64_sve, cpu) && sve_exception_el(env, el) == 0) { - int j, zcr_len = sve_zcr_len_for_el(env, el); + int j, zcr_len = sve_vqm1_for_el(env, el); for (i = 0; i <= FFR_PRED_NUM; i++) { bool eol; diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 73f24a57603f..e45b5cb7fe1b 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -1132,7 +1132,16 @@ void aarch64_sync_64_to_32(CPUARMState *env); int fp_exception_el(CPUARMState *env, int cur_el); int sve_exception_el(CPUARMState *env, int cur_el); -uint32_t sve_zcr_len_for_el(CPUARMState *env, int el); + +/** + * sve_vqm1_for_el: + * @env: CPUARMState + * @el: exception level + * + * Compute the current SVE vector length for @el, in units of + * Quadwords Minus 1 -- the same scale used for ZCR_ELx.LEN. + */ +uint32_t sve_vqm1_for_el(CPUARMState *env, int el); static inline bool is_a64(CPUARMState *env) { diff --git a/target/arm/gdbstub64.c b/target/arm/gdbstub64.c index 596878666d73..07a6746944d2 100644 --- a/target/arm/gdbstub64.c +++ b/target/arm/gdbstub64.c @@ -152,7 +152,7 @@ int arm_gdb_get_svereg(CPUARMState *env, GByteArray *buf, int reg) * We report in Vector Granules (VG) which is 64bit in a Z reg * while the ZCR works in Vector Quads (VQ) which is 128bit chunks. */ - int vq = sve_zcr_len_for_el(env, arm_current_el(env)) + 1; + int vq = sve_vqm1_for_el(env, arm_current_el(env)) + 1; return gdb_get_reg64(buf, vq * 2); } default: diff --git a/target/arm/helper.c b/target/arm/helper.c index 90aac6bc12d1..400f7cd1dba7 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -6215,7 +6215,7 @@ int sve_exception_el(CPUARMState *env, int el) /* * Given that SVE is enabled, return the vector length for EL. */ -uint32_t sve_zcr_len_for_el(CPUARMState *env, int el) +uint32_t sve_vqm1_for_el(CPUARMState *env, int el) { ARMCPU *cpu = env_archcpu(env); uint32_t len = cpu->sve_max_vq - 1; @@ -6238,7 +6238,7 @@ static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, uint64_t value) { int cur_el = arm_current_el(env); - int old_len = sve_zcr_len_for_el(env, cur_el); + int old_len = sve_vqm1_for_el(env, cur_el); int new_len; /* Bits other than [3:0] are RAZ/WI. */ @@ -6249,7 +6249,7 @@ static void zcr_write(CPUARMState *env, const ARMCPRegInfo *ri, * Because we arrived here, we know both FP and SVE are enabled; * otherwise we would have trapped access to the ZCR_ELn register. */ - new_len = sve_zcr_len_for_el(env, cur_el); + new_len = sve_vqm1_for_el(env, cur_el); if (new_len < old_len) { aarch64_sve_narrow_vq(env, new_len + 1); } @@ -11168,7 +11168,7 @@ static CPUARMTBFlags rebuild_hflags_a64(CPUARMState *env, int el, int fp_el, sve_el = 0; } } else if (sve_el == 0) { - DP_TBFLAG_A64(flags, VL, sve_zcr_len_for_el(env, el)); + DP_TBFLAG_A64(flags, VL, sve_vqm1_for_el(env, el)); } DP_TBFLAG_A64(flags, SVEEXC_EL, sve_el); } @@ -11534,10 +11534,10 @@ void aarch64_sve_change_el(CPUARMState *env, int old_el, */ old_a64 = old_el ? arm_el_is_aa64(env, old_el) : el0_a64; old_len = (old_a64 && !sve_exception_el(env, old_el) - ? sve_zcr_len_for_el(env, old_el) : 0); + ? sve_vqm1_for_el(env, old_el) : 0); new_a64 = new_el ? arm_el_is_aa64(env, new_el) : el0_a64; new_len = (new_a64 && !sve_exception_el(env, new_el) - ? sve_zcr_len_for_el(env, new_el) : 0); + ? sve_vqm1_for_el(env, new_el) : 0); /* When changing vector length, clear inaccessible state. */ if (new_len < old_len) { From d1ce862602ac23de7b3e8541507fc78a8c4a5766 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:57 +0100 Subject: [PATCH 51/90] target/arm: Split out load/store primitives to sve_ldst_internal.h Begin creation of sve_ldst_internal.h by moving the primitives that access host and tlb memory. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-14-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 107 +-------------------------- target/arm/sve_ldst_internal.h | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 106 deletions(-) create mode 100644 target/arm/sve_ldst_internal.h diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 3bdcd4ce9d04..0c6dde00aa66 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -21,12 +21,12 @@ #include "cpu.h" #include "internals.h" #include "exec/exec-all.h" -#include "exec/cpu_ldst.h" #include "exec/helper-proto.h" #include "tcg/tcg-gvec-desc.h" #include "fpu/softfloat.h" #include "tcg/tcg.h" #include "vec_internal.h" +#include "sve_ldst_internal.h" /* Return a value for NZCV as per the ARM PredTest pseudofunction. @@ -5301,111 +5301,6 @@ void HELPER(sve_fcmla_zpzzz_d)(void *vd, void *vn, void *vm, void *va, * Load contiguous data, protected by a governing predicate. */ -/* - * Load one element into @vd + @reg_off from @host. - * The controlling predicate is known to be true. - */ -typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); - -/* - * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). - * The controlling predicate is known to be true. - */ -typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, - target_ulong vaddr, uintptr_t retaddr); - -/* - * Generate the above primitives. - */ - -#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ -static void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ -{ \ - TYPEM val = HOST(host); \ - *(TYPEE *)(vd + H(reg_off)) = val; \ -} - -#define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ -static void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ -{ HOST(host, (TYPEM)*(TYPEE *)(vd + H(reg_off))); } - -#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ -static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off, \ - target_ulong addr, uintptr_t ra) \ -{ \ - *(TYPEE *)(vd + H(reg_off)) = \ - (TYPEM)TLB(env, useronly_clean_ptr(addr), ra); \ -} - -#define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ -static void sve_##NAME##_tlb(CPUARMState *env, void *vd, intptr_t reg_off, \ - target_ulong addr, uintptr_t ra) \ -{ \ - TLB(env, useronly_clean_ptr(addr), \ - (TYPEM)*(TYPEE *)(vd + H(reg_off)), ra); \ -} - -#define DO_LD_PRIM_1(NAME, H, TE, TM) \ - DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ - DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) - -DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) -DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) -DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) -DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) -DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) -DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) -DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) - -#define DO_ST_PRIM_1(NAME, H, TE, TM) \ - DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ - DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) - -DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) -DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) -DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) -DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) - -#define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ - DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ - DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ - DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ - DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) - -#define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ - DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ - DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ - DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ - DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) - -DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) -DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) -DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) -DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) -DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) - -DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) -DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) -DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) - -DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) -DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) -DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) - -DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) -DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) - -DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) -DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) - -#undef DO_LD_TLB -#undef DO_ST_TLB -#undef DO_LD_HOST -#undef DO_LD_PRIM_1 -#undef DO_ST_PRIM_1 -#undef DO_LD_PRIM_2 -#undef DO_ST_PRIM_2 - /* * Skip through a sequence of inactive elements in the guarding predicate @vg, * beginning at @reg_off bounded by @reg_max. Return the offset of the active diff --git a/target/arm/sve_ldst_internal.h b/target/arm/sve_ldst_internal.h new file mode 100644 index 000000000000..ef9117e84c1f --- /dev/null +++ b/target/arm/sve_ldst_internal.h @@ -0,0 +1,127 @@ +/* + * ARM SVE Load/Store Helpers + * + * Copyright (c) 2018-2022 Linaro + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef TARGET_ARM_SVE_LDST_INTERNAL_H +#define TARGET_ARM_SVE_LDST_INTERNAL_H + +#include "exec/cpu_ldst.h" + +/* + * Load one element into @vd + @reg_off from @host. + * The controlling predicate is known to be true. + */ +typedef void sve_ldst1_host_fn(void *vd, intptr_t reg_off, void *host); + +/* + * Load one element into @vd + @reg_off from (@env, @vaddr, @ra). + * The controlling predicate is known to be true. + */ +typedef void sve_ldst1_tlb_fn(CPUARMState *env, void *vd, intptr_t reg_off, + target_ulong vaddr, uintptr_t retaddr); + +/* + * Generate the above primitives. + */ + +#define DO_LD_HOST(NAME, H, TYPEE, TYPEM, HOST) \ +static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ +{ TYPEM val = HOST(host); *(TYPEE *)(vd + H(reg_off)) = val; } + +#define DO_ST_HOST(NAME, H, TYPEE, TYPEM, HOST) \ +static inline void sve_##NAME##_host(void *vd, intptr_t reg_off, void *host) \ +{ TYPEM val = *(TYPEE *)(vd + H(reg_off)); HOST(host, val); } + +#define DO_LD_TLB(NAME, H, TYPEE, TYPEM, TLB) \ +static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ + intptr_t reg_off, target_ulong addr, uintptr_t ra) \ +{ \ + TYPEM val = TLB(env, useronly_clean_ptr(addr), ra); \ + *(TYPEE *)(vd + H(reg_off)) = val; \ +} + +#define DO_ST_TLB(NAME, H, TYPEE, TYPEM, TLB) \ +static inline void sve_##NAME##_tlb(CPUARMState *env, void *vd, \ + intptr_t reg_off, target_ulong addr, uintptr_t ra) \ +{ \ + TYPEM val = *(TYPEE *)(vd + H(reg_off)); \ + TLB(env, useronly_clean_ptr(addr), val, ra); \ +} + +#define DO_LD_PRIM_1(NAME, H, TE, TM) \ + DO_LD_HOST(NAME, H, TE, TM, ldub_p) \ + DO_LD_TLB(NAME, H, TE, TM, cpu_ldub_data_ra) + +DO_LD_PRIM_1(ld1bb, H1, uint8_t, uint8_t) +DO_LD_PRIM_1(ld1bhu, H1_2, uint16_t, uint8_t) +DO_LD_PRIM_1(ld1bhs, H1_2, uint16_t, int8_t) +DO_LD_PRIM_1(ld1bsu, H1_4, uint32_t, uint8_t) +DO_LD_PRIM_1(ld1bss, H1_4, uint32_t, int8_t) +DO_LD_PRIM_1(ld1bdu, H1_8, uint64_t, uint8_t) +DO_LD_PRIM_1(ld1bds, H1_8, uint64_t, int8_t) + +#define DO_ST_PRIM_1(NAME, H, TE, TM) \ + DO_ST_HOST(st1##NAME, H, TE, TM, stb_p) \ + DO_ST_TLB(st1##NAME, H, TE, TM, cpu_stb_data_ra) + +DO_ST_PRIM_1(bb, H1, uint8_t, uint8_t) +DO_ST_PRIM_1(bh, H1_2, uint16_t, uint8_t) +DO_ST_PRIM_1(bs, H1_4, uint32_t, uint8_t) +DO_ST_PRIM_1(bd, H1_8, uint64_t, uint8_t) + +#define DO_LD_PRIM_2(NAME, H, TE, TM, LD) \ + DO_LD_HOST(ld1##NAME##_be, H, TE, TM, LD##_be_p) \ + DO_LD_HOST(ld1##NAME##_le, H, TE, TM, LD##_le_p) \ + DO_LD_TLB(ld1##NAME##_be, H, TE, TM, cpu_##LD##_be_data_ra) \ + DO_LD_TLB(ld1##NAME##_le, H, TE, TM, cpu_##LD##_le_data_ra) + +#define DO_ST_PRIM_2(NAME, H, TE, TM, ST) \ + DO_ST_HOST(st1##NAME##_be, H, TE, TM, ST##_be_p) \ + DO_ST_HOST(st1##NAME##_le, H, TE, TM, ST##_le_p) \ + DO_ST_TLB(st1##NAME##_be, H, TE, TM, cpu_##ST##_be_data_ra) \ + DO_ST_TLB(st1##NAME##_le, H, TE, TM, cpu_##ST##_le_data_ra) + +DO_LD_PRIM_2(hh, H1_2, uint16_t, uint16_t, lduw) +DO_LD_PRIM_2(hsu, H1_4, uint32_t, uint16_t, lduw) +DO_LD_PRIM_2(hss, H1_4, uint32_t, int16_t, lduw) +DO_LD_PRIM_2(hdu, H1_8, uint64_t, uint16_t, lduw) +DO_LD_PRIM_2(hds, H1_8, uint64_t, int16_t, lduw) + +DO_ST_PRIM_2(hh, H1_2, uint16_t, uint16_t, stw) +DO_ST_PRIM_2(hs, H1_4, uint32_t, uint16_t, stw) +DO_ST_PRIM_2(hd, H1_8, uint64_t, uint16_t, stw) + +DO_LD_PRIM_2(ss, H1_4, uint32_t, uint32_t, ldl) +DO_LD_PRIM_2(sdu, H1_8, uint64_t, uint32_t, ldl) +DO_LD_PRIM_2(sds, H1_8, uint64_t, int32_t, ldl) + +DO_ST_PRIM_2(ss, H1_4, uint32_t, uint32_t, stl) +DO_ST_PRIM_2(sd, H1_8, uint64_t, uint32_t, stl) + +DO_LD_PRIM_2(dd, H1_8, uint64_t, uint64_t, ldq) +DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) + +#undef DO_LD_TLB +#undef DO_ST_TLB +#undef DO_LD_HOST +#undef DO_LD_PRIM_1 +#undef DO_ST_PRIM_1 +#undef DO_LD_PRIM_2 +#undef DO_ST_PRIM_2 + +#endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */ From 0b68112b399d77c765a0a5d9bb4f4b4e57541ee7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 52/90] target/arm: Export sve contiguous ldst support functions Export all of the support functions for performing bulk fault analysis on a set of elements at contiguous addresses controlled by a predicate. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-15-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 87 ++++++------------------------- target/arm/sve_ldst_internal.h | 94 ++++++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 70 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 0c6dde00aa66..8cd371e3e37b 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -5341,16 +5341,9 @@ static intptr_t find_next_active(uint64_t *vg, intptr_t reg_off, * exit via page fault exception. */ -typedef struct { - void *host; - int flags; - MemTxAttrs attrs; -} SVEHostPage; - -static bool sve_probe_page(SVEHostPage *info, bool nofault, - CPUARMState *env, target_ulong addr, - int mem_off, MMUAccessType access_type, - int mmu_idx, uintptr_t retaddr) +bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, + target_ulong addr, int mem_off, MMUAccessType access_type, + int mmu_idx, uintptr_t retaddr) { int flags; @@ -5406,59 +5399,13 @@ static bool sve_probe_page(SVEHostPage *info, bool nofault, return true; } - -/* - * Analyse contiguous data, protected by a governing predicate. - */ - -typedef enum { - FAULT_NO, - FAULT_FIRST, - FAULT_ALL, -} SVEContFault; - -typedef struct { - /* - * First and last element wholly contained within the two pages. - * mem_off_first[0] and reg_off_first[0] are always set >= 0. - * reg_off_last[0] may be < 0 if the first element crosses pages. - * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] - * are set >= 0 only if there are complete elements on a second page. - * - * The reg_off_* offsets are relative to the internal vector register. - * The mem_off_first offset is relative to the memory address; the - * two offsets are different when a load operation extends, a store - * operation truncates, or for multi-register operations. - */ - int16_t mem_off_first[2]; - int16_t reg_off_first[2]; - int16_t reg_off_last[2]; - - /* - * One element that is misaligned and spans both pages, - * or -1 if there is no such active element. - */ - int16_t mem_off_split; - int16_t reg_off_split; - - /* - * The byte offset at which the entire operation crosses a page boundary. - * Set >= 0 if and only if the entire operation spans two pages. - */ - int16_t page_split; - - /* TLB data for the two pages. */ - SVEHostPage page[2]; -} SVEContLdSt; - /* * Find first active element on each page, and a loose bound for the * final element on each page. Identify any single element that spans * the page boundary. Return true if there are any active elements. */ -static bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, - uint64_t *vg, intptr_t reg_max, - int esz, int msize) +bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, + intptr_t reg_max, int esz, int msize) { const int esize = 1 << esz; const uint64_t pg_mask = pred_esz_masks[esz]; @@ -5548,9 +5495,9 @@ static bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, * Control the generation of page faults with @fault. Return false if * there is no work to do, which can only happen with @fault == FAULT_NO. */ -static bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, - CPUARMState *env, target_ulong addr, - MMUAccessType access_type, uintptr_t retaddr) +bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, + CPUARMState *env, target_ulong addr, + MMUAccessType access_type, uintptr_t retaddr) { int mmu_idx = cpu_mmu_index(env, false); int mem_off = info->mem_off_first[0]; @@ -5606,12 +5553,12 @@ static bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, return have_work; } -static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, - int esize, int msize, int wp_access, - uintptr_t retaddr) -{ #ifndef CONFIG_USER_ONLY +void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, + int esize, int msize, int wp_access, + uintptr_t retaddr) +{ intptr_t mem_off, reg_off, reg_last; int flags0 = info->page[0].flags; int flags1 = info->page[1].flags; @@ -5667,12 +5614,12 @@ static void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, } while (reg_off & 63); } while (reg_off <= reg_last); } -#endif } +#endif -static void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, - uint64_t *vg, target_ulong addr, int esize, - int msize, uint32_t mtedesc, uintptr_t ra) +void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, int esize, + int msize, uint32_t mtedesc, uintptr_t ra) { intptr_t mem_off, reg_off, reg_last; diff --git a/target/arm/sve_ldst_internal.h b/target/arm/sve_ldst_internal.h index ef9117e84c1f..b5c473fc48b1 100644 --- a/target/arm/sve_ldst_internal.h +++ b/target/arm/sve_ldst_internal.h @@ -124,4 +124,98 @@ DO_ST_PRIM_2(dd, H1_8, uint64_t, uint64_t, stq) #undef DO_LD_PRIM_2 #undef DO_ST_PRIM_2 +/* + * Resolve the guest virtual address to info->host and info->flags. + * If @nofault, return false if the page is invalid, otherwise + * exit via page fault exception. + */ + +typedef struct { + void *host; + int flags; + MemTxAttrs attrs; +} SVEHostPage; + +bool sve_probe_page(SVEHostPage *info, bool nofault, CPUARMState *env, + target_ulong addr, int mem_off, MMUAccessType access_type, + int mmu_idx, uintptr_t retaddr); + +/* + * Analyse contiguous data, protected by a governing predicate. + */ + +typedef enum { + FAULT_NO, + FAULT_FIRST, + FAULT_ALL, +} SVEContFault; + +typedef struct { + /* + * First and last element wholly contained within the two pages. + * mem_off_first[0] and reg_off_first[0] are always set >= 0. + * reg_off_last[0] may be < 0 if the first element crosses pages. + * All of mem_off_first[1], reg_off_first[1] and reg_off_last[1] + * are set >= 0 only if there are complete elements on a second page. + * + * The reg_off_* offsets are relative to the internal vector register. + * The mem_off_first offset is relative to the memory address; the + * two offsets are different when a load operation extends, a store + * operation truncates, or for multi-register operations. + */ + int16_t mem_off_first[2]; + int16_t reg_off_first[2]; + int16_t reg_off_last[2]; + + /* + * One element that is misaligned and spans both pages, + * or -1 if there is no such active element. + */ + int16_t mem_off_split; + int16_t reg_off_split; + + /* + * The byte offset at which the entire operation crosses a page boundary. + * Set >= 0 if and only if the entire operation spans two pages. + */ + int16_t page_split; + + /* TLB data for the two pages. */ + SVEHostPage page[2]; +} SVEContLdSt; + +/* + * Find first active element on each page, and a loose bound for the + * final element on each page. Identify any single element that spans + * the page boundary. Return true if there are any active elements. + */ +bool sve_cont_ldst_elements(SVEContLdSt *info, target_ulong addr, uint64_t *vg, + intptr_t reg_max, int esz, int msize); + +/* + * Resolve the guest virtual addresses to info->page[]. + * Control the generation of page faults with @fault. Return false if + * there is no work to do, which can only happen with @fault == FAULT_NO. + */ +bool sve_cont_ldst_pages(SVEContLdSt *info, SVEContFault fault, + CPUARMState *env, target_ulong addr, + MMUAccessType access_type, uintptr_t retaddr); + +#ifdef CONFIG_USER_ONLY +static inline void +sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, + target_ulong addr, int esize, int msize, + int wp_access, uintptr_t retaddr) +{ } +#else +void sve_cont_ldst_watchpoints(SVEContLdSt *info, CPUARMState *env, + uint64_t *vg, target_ulong addr, + int esize, int msize, int wp_access, + uintptr_t retaddr); +#endif + +void sve_cont_ldst_mte_check(SVEContLdSt *info, CPUARMState *env, uint64_t *vg, + target_ulong addr, int esize, int msize, + uint32_t mtedesc, uintptr_t ra); + #endif /* TARGET_ARM_SVE_LDST_INTERNAL_H */ From 820e0bb9ce1941af82d14ec4a981792cb3697956 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 53/90] target/arm: Move expand_pred_b to vec_internal.h Put the inline function near the array declaration. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-16-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 9 --------- target/arm/vec_internal.h | 8 +++++++- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index 8cd371e3e37b..e865c1252732 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -103,15 +103,6 @@ uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) return flags; } -/* - * Expand active predicate bits to bytes, for byte elements. - * (The data table itself is in vec_helper.c as MVE also needs it.) - */ -static inline uint64_t expand_pred_b(uint8_t byte) -{ - return expand_pred_b_data[byte]; -} - /* Similarly for half-word elements. * for (i = 0; i < 256; ++i) { * unsigned long m = 0; diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index 1d63402042f5..d1a1ea4a668f 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -50,8 +50,14 @@ #define H8(x) (x) #define H1_8(x) (x) -/* Data for expanding active predicate bits to bytes, for byte elements. */ +/* + * Expand active predicate bits to bytes, for byte elements. + */ extern const uint64_t expand_pred_b_data[256]; +static inline uint64_t expand_pred_b(uint8_t byte) +{ + return expand_pred_b_data[byte]; +} static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) { From 05dd14bdfa89ffeb1a82b08776687d44b538185b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 54/90] target/arm: Use expand_pred_b in mve_helper.c Use the function instead of the array directly. Because the function performs its own masking, via the uint8_t parameter, we need to do nothing extra within the users: the bits above the first 2 (_uh) or 4 (_uw) will be discarded by assignment to the local bmask variables, and of course _uq uses the entire uint64_t result. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-17-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/mve_helper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target/arm/mve_helper.c b/target/arm/mve_helper.c index 846962bf4c5a..403b345ea3b9 100644 --- a/target/arm/mve_helper.c +++ b/target/arm/mve_helper.c @@ -726,7 +726,7 @@ static void mergemask_sb(int8_t *d, int8_t r, uint16_t mask) static void mergemask_uh(uint16_t *d, uint16_t r, uint16_t mask) { - uint16_t bmask = expand_pred_b_data[mask & 3]; + uint16_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } @@ -737,7 +737,7 @@ static void mergemask_sh(int16_t *d, int16_t r, uint16_t mask) static void mergemask_uw(uint32_t *d, uint32_t r, uint16_t mask) { - uint32_t bmask = expand_pred_b_data[mask & 0xf]; + uint32_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } @@ -748,7 +748,7 @@ static void mergemask_sw(int32_t *d, int32_t r, uint16_t mask) static void mergemask_uq(uint64_t *d, uint64_t r, uint16_t mask) { - uint64_t bmask = expand_pred_b_data[mask & 0xff]; + uint64_t bmask = expand_pred_b(mask); *d = (*d & ~bmask) | (r & bmask); } From a613cf2d4ae61fbcdf084103ea10de193bfbe169 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 55/90] target/arm: Move expand_pred_h to vec_internal.h Move the data to vec_helper.c and the inline to vec_internal.h. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-18-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/sve_helper.c | 29 ----------------------------- target/arm/vec_helper.c | 26 ++++++++++++++++++++++++++ target/arm/vec_internal.h | 7 +++++++ 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/target/arm/sve_helper.c b/target/arm/sve_helper.c index e865c1252732..1654c0bbf9ea 100644 --- a/target/arm/sve_helper.c +++ b/target/arm/sve_helper.c @@ -103,35 +103,6 @@ uint32_t HELPER(sve_predtest)(void *vd, void *vg, uint32_t words) return flags; } -/* Similarly for half-word elements. - * for (i = 0; i < 256; ++i) { - * unsigned long m = 0; - * if (i & 0xaa) { - * continue; - * } - * for (j = 0; j < 8; j += 2) { - * if ((i >> j) & 1) { - * m |= 0xfffful << (j << 3); - * } - * } - * printf("[0x%x] = 0x%016lx,\n", i, m); - * } - */ -static inline uint64_t expand_pred_h(uint8_t byte) -{ - static const uint64_t word[] = { - [0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000, - [0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000, - [0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000, - [0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000, - [0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000, - [0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000, - [0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000, - [0x55] = 0xffffffffffffffff, - }; - return word[byte & 0x55]; -} - /* Similarly for single word elements. */ static inline uint64_t expand_pred_s(uint8_t byte) { diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index 17fb1583622e..26c373e522f2 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -127,6 +127,32 @@ const uint64_t expand_pred_b_data[256] = { 0xffffffffffffffff, }; +/* + * Similarly for half-word elements. + * for (i = 0; i < 256; ++i) { + * unsigned long m = 0; + * if (i & 0xaa) { + * continue; + * } + * for (j = 0; j < 8; j += 2) { + * if ((i >> j) & 1) { + * m |= 0xfffful << (j << 3); + * } + * } + * printf("[0x%x] = 0x%016lx,\n", i, m); + * } + */ +const uint64_t expand_pred_h_data[0x55 + 1] = { + [0x01] = 0x000000000000ffff, [0x04] = 0x00000000ffff0000, + [0x05] = 0x00000000ffffffff, [0x10] = 0x0000ffff00000000, + [0x11] = 0x0000ffff0000ffff, [0x14] = 0x0000ffffffff0000, + [0x15] = 0x0000ffffffffffff, [0x40] = 0xffff000000000000, + [0x41] = 0xffff00000000ffff, [0x44] = 0xffff0000ffff0000, + [0x45] = 0xffff0000ffffffff, [0x50] = 0xffffffff00000000, + [0x51] = 0xffffffff0000ffff, [0x54] = 0xffffffffffff0000, + [0x55] = 0xffffffffffffffff, +}; + /* Signed saturating rounding doubling multiply-accumulate high half, 8-bit */ int8_t do_sqrdmlah_b(int8_t src1, int8_t src2, int8_t src3, bool neg, bool round) diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index d1a1ea4a668f..1d527fadac12 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -59,6 +59,13 @@ static inline uint64_t expand_pred_b(uint8_t byte) return expand_pred_b_data[byte]; } +/* Similarly for half-word elements. */ +extern const uint64_t expand_pred_h_data[0x55 + 1]; +static inline uint64_t expand_pred_h(uint8_t byte) +{ + return expand_pred_h_data[byte & 0x55]; +} + static inline void clear_tail(void *vd, uintptr_t opr_sz, uintptr_t max_sz) { uint64_t *d = vd + opr_sz; From 72db2aa353a98bd3486c9e5ef7e31ae7cf934849 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:58 +0100 Subject: [PATCH 56/90] target/arm: Export bfdotadd from vec_helper.c We will need this over in sme_helper.c. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-19-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/vec_helper.c | 2 +- target/arm/vec_internal.h | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/target/arm/vec_helper.c b/target/arm/vec_helper.c index 26c373e522f2..9a9c034e36f5 100644 --- a/target/arm/vec_helper.c +++ b/target/arm/vec_helper.c @@ -2557,7 +2557,7 @@ DO_MMLA_B(gvec_usmmla_b, do_usmmla_b) * BFloat16 Dot Product */ -static float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2) +float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2) { /* FPCR is ignored for BFDOT and BFMMLA. */ float_status bf_status = { diff --git a/target/arm/vec_internal.h b/target/arm/vec_internal.h index 1d527fadac12..1f4ed80ff76e 100644 --- a/target/arm/vec_internal.h +++ b/target/arm/vec_internal.h @@ -230,4 +230,17 @@ uint64_t pmull_h(uint64_t op1, uint64_t op2); */ uint64_t pmull_w(uint64_t op1, uint64_t op2); +/** + * bfdotadd: + * @sum: addend + * @e1, @e2: multiplicand vectors + * + * BFloat16 2-way dot product of @e1 & @e2, accumulating with @sum. + * The @e1 and @e2 operands correspond to the 32-bit source vector + * slots and contain two Bfloat16 values each. + * + * Corresponds to the ARM pseudocode function BFDotAdd. + */ +float32 bfdotadd(float32 sum, uint32_t e1, uint32_t e2); + #endif /* TARGET_ARM_VEC_INTERNAL_H */ From f305bf9436896b4cd9ef622034e166b024780874 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:59 +0100 Subject: [PATCH 57/90] target/arm: Add isar_feature_aa64_sme This will be used for implementing FEAT_SME. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-20-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index e45b5cb7fe1b..2e6153c5409a 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -4048,6 +4048,11 @@ static inline bool isar_feature_aa64_mte(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, MTE) >= 2; } +static inline bool isar_feature_aa64_sme(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64pfr1, ID_AA64PFR1, SME) != 0; +} + static inline bool isar_feature_aa64_pmu_8_1(const ARMISARegisters *id) { return FIELD_EX64(id->id_aa64dfr0, ID_AA64DFR0, PMUVER) >= 4 && From 414c54d515dba16bfaef643a8acec200c05f229a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 8 Jun 2022 19:38:59 +0100 Subject: [PATCH 58/90] target/arm: Add ID_AA64SMFR0_EL1 This register is allocated from the existing block of id registers, so it is already RES0 for cpus that do not implement SME. Reviewed-by: Peter Maydell Signed-off-by: Richard Henderson Message-id: 20220607203306.657998-21-richard.henderson@linaro.org Signed-off-by: Peter Maydell --- target/arm/cpu.h | 25 +++++++++++++++++++++++++ target/arm/helper.c | 4 ++-- target/arm/kvm64.c | 11 +++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/target/arm/cpu.h b/target/arm/cpu.h index 2e6153c5409a..78dbcb5592c7 100644 --- a/target/arm/cpu.h +++ b/target/arm/cpu.h @@ -966,6 +966,7 @@ struct ArchCPU { uint64_t id_aa64dfr0; uint64_t id_aa64dfr1; uint64_t id_aa64zfr0; + uint64_t id_aa64smfr0; uint64_t reset_pmcr_el0; } isar; uint64_t midr; @@ -2190,6 +2191,15 @@ FIELD(ID_AA64ZFR0, I8MM, 44, 4) FIELD(ID_AA64ZFR0, F32MM, 52, 4) FIELD(ID_AA64ZFR0, F64MM, 56, 4) +FIELD(ID_AA64SMFR0, F32F32, 32, 1) +FIELD(ID_AA64SMFR0, B16F32, 34, 1) +FIELD(ID_AA64SMFR0, F16F32, 35, 1) +FIELD(ID_AA64SMFR0, I8I32, 36, 4) +FIELD(ID_AA64SMFR0, F64F64, 48, 1) +FIELD(ID_AA64SMFR0, I16I64, 52, 4) +FIELD(ID_AA64SMFR0, SMEVER, 56, 4) +FIELD(ID_AA64SMFR0, FA64, 63, 1) + FIELD(ID_DFR0, COPDBG, 0, 4) FIELD(ID_DFR0, COPSDBG, 4, 4) FIELD(ID_DFR0, MMAPDBG, 8, 4) @@ -4195,6 +4205,21 @@ static inline bool isar_feature_aa64_sve_f64mm(const ARMISARegisters *id) return FIELD_EX64(id->id_aa64zfr0, ID_AA64ZFR0, F64MM) != 0; } +static inline bool isar_feature_aa64_sme_f64f64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, F64F64); +} + +static inline bool isar_feature_aa64_sme_i16i64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, I16I64) == 0xf; +} + +static inline bool isar_feature_aa64_sme_fa64(const ARMISARegisters *id) +{ + return FIELD_EX64(id->id_aa64smfr0, ID_AA64SMFR0, FA64); +} + /* * Feature tests for "does this exist in either 32-bit or 64-bit?" */ diff --git a/target/arm/helper.c b/target/arm/helper.c index 400f7cd1dba7..ac9942d750d4 100644 --- a/target/arm/helper.c +++ b/target/arm/helper.c @@ -7722,11 +7722,11 @@ void register_cp_regs_for_features(ARMCPU *cpu) .access = PL1_R, .type = ARM_CP_CONST, .accessfn = access_aa64_tid3, .resetvalue = cpu->isar.id_aa64zfr0 }, - { .name = "ID_AA64PFR5_EL1_RESERVED", .state = ARM_CP_STATE_AA64, + { .name = "ID_AA64SMFR0_EL1", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 5, .access = PL1_R, .type = ARM_CP_CONST, .accessfn = access_aa64_tid3, - .resetvalue = 0 }, + .resetvalue = cpu->isar.id_aa64smfr0 }, { .name = "ID_AA64PFR6_EL1_RESERVED", .state = ARM_CP_STATE_AA64, .opc0 = 3, .opc1 = 0, .crn = 0, .crm = 4, .opc2 = 6, .access = PL1_R, .type = ARM_CP_CONST, diff --git a/target/arm/kvm64.c b/target/arm/kvm64.c index b3f635fc952b..ff8f65da22f5 100644 --- a/target/arm/kvm64.c +++ b/target/arm/kvm64.c @@ -574,6 +574,8 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) } else { err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64pfr1, ARM64_SYS_REG(3, 0, 0, 4, 1)); + err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64smfr0, + ARM64_SYS_REG(3, 0, 0, 4, 5)); err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr0, ARM64_SYS_REG(3, 0, 0, 5, 0)); err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64dfr1, @@ -682,10 +684,11 @@ bool kvm_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf) ahcf->isar.id_aa64pfr0 = t; /* - * Before v5.1, KVM did not support SVE and did not expose - * ID_AA64ZFR0_EL1 even as RAZ. After v5.1, KVM still does - * not expose the register to "user" requests like this - * unless the host supports SVE. + * There is a range of kernels between kernel commit 73433762fcae + * and f81cb2c3ad41 which have a bug where the kernel doesn't expose + * SYS_ID_AA64ZFR0_EL1 via the ONE_REG API unless the VM has enabled + * SVE support, so we only read it here, rather than together with all + * the other ID registers earlier. */ err |= read_sys_reg64(fdarray[2], &ahcf->isar.id_aa64zfr0, ARM64_SYS_REG(3, 0, 0, 4, 4)); From 7851b21a8192750adecbcf6e8780a20de5891ad6 Mon Sep 17 00:00:00 2001 From: Bernhard Beschow Date: Fri, 13 May 2022 20:09:55 +0200 Subject: [PATCH 59/90] hw/ide/piix: Remove redundant "piix3-ide-xen" device class Commit 0f8445820f11a69154309863960328dda3dc1ad4 'xen: piix reuse pci generic class init function' already resolved redundant code which in turn rendered piix3-ide-xen redundant. Signed-off-by: Bernhard Beschow Reviewed-by: Anthony PERARD Message-Id: <20220513180957.90514-2-shentey@gmail.com> Signed-off-by: Anthony PERARD --- hw/i386/pc_piix.c | 3 +-- hw/ide/piix.c | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/hw/i386/pc_piix.c b/hw/i386/pc_piix.c index 578e537b3525..0e45521e74d1 100644 --- a/hw/i386/pc_piix.c +++ b/hw/i386/pc_piix.c @@ -246,8 +246,7 @@ static void pc_init1(MachineState *machine, if (pcmc->pci_enabled) { PCIDevice *dev; - dev = pci_create_simple(pci_bus, piix3_devfn + 1, - xen_enabled() ? "piix3-ide-xen" : "piix3-ide"); + dev = pci_create_simple(pci_bus, piix3_devfn + 1, "piix3-ide"); pci_ide_create_devs(dev); idebus[0] = qdev_get_child_bus(&dev->qdev, "ide.0"); idebus[1] = qdev_get_child_bus(&dev->qdev, "ide.1"); diff --git a/hw/ide/piix.c b/hw/ide/piix.c index ce89fd0aa368..2345fe9e1dd0 100644 --- a/hw/ide/piix.c +++ b/hw/ide/piix.c @@ -241,12 +241,6 @@ static const TypeInfo piix3_ide_info = { .class_init = piix3_ide_class_init, }; -static const TypeInfo piix3_ide_xen_info = { - .name = "piix3-ide-xen", - .parent = TYPE_PCI_IDE, - .class_init = piix3_ide_class_init, -}; - /* NOTE: for the PIIX4, the IRQs and IOports are hardcoded */ static void piix4_ide_class_init(ObjectClass *klass, void *data) { @@ -272,7 +266,6 @@ static const TypeInfo piix4_ide_info = { static void piix_ide_register_types(void) { type_register_static(&piix3_ide_info); - type_register_static(&piix3_ide_xen_info); type_register_static(&piix4_ide_info); } From 36902417460e3f9f1c63a79d802622fd774f1c30 Mon Sep 17 00:00:00 2001 From: Bernhard Beschow Date: Fri, 13 May 2022 20:09:56 +0200 Subject: [PATCH 60/90] hw/ide/piix: Add some documentation to pci_piix3_xen_ide_unplug() The comment is based on commit message ae4d2eb273b167dad748ea4249720319240b1ac2 'xen-platform: add missing disk unplug option'. Since it seems to describe design decisions and limitations that still apply it seems worth having. Signed-off-by: Bernhard Beschow Reviewed-by: Anthony PERARD Message-Id: <20220513180957.90514-3-shentey@gmail.com> Signed-off-by: Anthony PERARD --- hw/ide/piix.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/hw/ide/piix.c b/hw/ide/piix.c index 2345fe9e1dd0..bc1b37512ab3 100644 --- a/hw/ide/piix.c +++ b/hw/ide/piix.c @@ -173,6 +173,17 @@ static void pci_piix_ide_realize(PCIDevice *dev, Error **errp) } } +/* + * The Xen HVM unplug protocol [1] specifies a mechanism to allow guests to + * request unplug of 'aux' disks (which is stated to mean all IDE disks, + * except the primary master). + * + * NOTE: The semantics of what happens if unplug of all disks and 'aux' disks + * is simultaneously requested is not clear. The implementation assumes + * that an 'all' request overrides an 'aux' request. + * + * [1] https://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=docs/misc/hvm-emulated-unplug.pandoc + */ int pci_piix3_xen_ide_unplug(DeviceState *dev, bool aux) { PCIIDEState *pci_ide; From 6a8a8b62bdc8e3d7c5fc0f82ef4583707183b12f Mon Sep 17 00:00:00 2001 From: Bernhard Beschow Date: Fri, 13 May 2022 20:09:57 +0200 Subject: [PATCH 61/90] include/hw/ide: Unexport pci_piix3_xen_ide_unplug() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This function was declared in a generic and public header, implemented in a device-specific source file but only used in xen_platform. Given its 'aux' parameter, this function is more xen-specific than piix-specific. Also, the hardcoded magic constants seem to be generic and related to PCIIDEState and IDEBus rather than piix. Therefore, move this function to xen_platform, unexport it, and drop the "piix3" in the function name as well. Signed-off-by: Bernhard Beschow Reviewed-by: Paul Durrant Acked-by: Anthony PERARD Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20220513180957.90514-4-shentey@gmail.com> Signed-off-by: Anthony PERARD --- hw/i386/xen/xen_platform.c | 48 +++++++++++++++++++++++++++++++++++++- hw/ide/piix.c | 46 ------------------------------------ include/hw/ide.h | 3 --- 3 files changed, 47 insertions(+), 50 deletions(-) diff --git a/hw/i386/xen/xen_platform.c b/hw/i386/xen/xen_platform.c index 72028449bae0..a64265cca073 100644 --- a/hw/i386/xen/xen_platform.c +++ b/hw/i386/xen/xen_platform.c @@ -26,6 +26,7 @@ #include "qemu/osdep.h" #include "qapi/error.h" #include "hw/ide.h" +#include "hw/ide/pci.h" #include "hw/pci/pci.h" #include "hw/xen/xen_common.h" #include "migration/vmstate.h" @@ -134,6 +135,51 @@ static void pci_unplug_nics(PCIBus *bus) pci_for_each_device(bus, 0, unplug_nic, NULL); } +/* + * The Xen HVM unplug protocol [1] specifies a mechanism to allow guests to + * request unplug of 'aux' disks (which is stated to mean all IDE disks, + * except the primary master). + * + * NOTE: The semantics of what happens if unplug of all disks and 'aux' disks + * is simultaneously requested is not clear. The implementation assumes + * that an 'all' request overrides an 'aux' request. + * + * [1] https://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=docs/misc/hvm-emulated-unplug.pandoc + */ +static void pci_xen_ide_unplug(DeviceState *dev, bool aux) +{ + PCIIDEState *pci_ide; + int i; + IDEDevice *idedev; + IDEBus *idebus; + BlockBackend *blk; + + pci_ide = PCI_IDE(dev); + + for (i = aux ? 1 : 0; i < 4; i++) { + idebus = &pci_ide->bus[i / 2]; + blk = idebus->ifs[i % 2].blk; + + if (blk && idebus->ifs[i % 2].drive_kind != IDE_CD) { + if (!(i % 2)) { + idedev = idebus->master; + } else { + idedev = idebus->slave; + } + + blk_drain(blk); + blk_flush(blk); + + blk_detach_dev(blk, DEVICE(idedev)); + idebus->ifs[i % 2].blk = NULL; + idedev->conf.blk = NULL; + monitor_remove_blk(blk); + blk_unref(blk); + } + } + qdev_reset_all(dev); +} + static void unplug_disks(PCIBus *b, PCIDevice *d, void *opaque) { uint32_t flags = *(uint32_t *)opaque; @@ -147,7 +193,7 @@ static void unplug_disks(PCIBus *b, PCIDevice *d, void *opaque) switch (pci_get_word(d->config + PCI_CLASS_DEVICE)) { case PCI_CLASS_STORAGE_IDE: - pci_piix3_xen_ide_unplug(DEVICE(d), aux); + pci_xen_ide_unplug(DEVICE(d), aux); break; case PCI_CLASS_STORAGE_SCSI: diff --git a/hw/ide/piix.c b/hw/ide/piix.c index bc1b37512ab3..9a9b28078e6d 100644 --- a/hw/ide/piix.c +++ b/hw/ide/piix.c @@ -173,52 +173,6 @@ static void pci_piix_ide_realize(PCIDevice *dev, Error **errp) } } -/* - * The Xen HVM unplug protocol [1] specifies a mechanism to allow guests to - * request unplug of 'aux' disks (which is stated to mean all IDE disks, - * except the primary master). - * - * NOTE: The semantics of what happens if unplug of all disks and 'aux' disks - * is simultaneously requested is not clear. The implementation assumes - * that an 'all' request overrides an 'aux' request. - * - * [1] https://xenbits.xen.org/gitweb/?p=xen.git;a=blob;f=docs/misc/hvm-emulated-unplug.pandoc - */ -int pci_piix3_xen_ide_unplug(DeviceState *dev, bool aux) -{ - PCIIDEState *pci_ide; - int i; - IDEDevice *idedev; - IDEBus *idebus; - BlockBackend *blk; - - pci_ide = PCI_IDE(dev); - - for (i = aux ? 1 : 0; i < 4; i++) { - idebus = &pci_ide->bus[i / 2]; - blk = idebus->ifs[i % 2].blk; - - if (blk && idebus->ifs[i % 2].drive_kind != IDE_CD) { - if (!(i % 2)) { - idedev = idebus->master; - } else { - idedev = idebus->slave; - } - - blk_drain(blk); - blk_flush(blk); - - blk_detach_dev(blk, DEVICE(idedev)); - idebus->ifs[i % 2].blk = NULL; - idedev->conf.blk = NULL; - monitor_remove_blk(blk); - blk_unref(blk); - } - } - qdev_reset_all(dev); - return 0; -} - static void pci_piix_ide_exitfn(PCIDevice *dev) { PCIIDEState *d = PCI_IDE(dev); diff --git a/include/hw/ide.h b/include/hw/ide.h index c5ce5da4f478..60f1f4f71494 100644 --- a/include/hw/ide.h +++ b/include/hw/ide.h @@ -8,9 +8,6 @@ ISADevice *isa_ide_init(ISABus *bus, int iobase, int iobase2, int isairq, DriveInfo *hd0, DriveInfo *hd1); -/* ide-pci.c */ -int pci_piix3_xen_ide_unplug(DeviceState *dev, bool aux); - /* ide-mmio.c */ void mmio_ide_init_drives(DeviceState *dev, DriveInfo *hd0, DriveInfo *hd1); From efe1592c43fe9b4053bf2987581a05736062a3cd Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Mon, 9 May 2022 11:13:39 +0200 Subject: [PATCH 62/90] MAINTAINERS: Cover hw/core/uboot_image.h within Generic Loader section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Alistair Francis Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20220509091339.26016-1-alistair.francis@wdc.com> --- MAINTAINERS | 1 + 1 file changed, 1 insertion(+) diff --git a/MAINTAINERS b/MAINTAINERS index 5580a36b68e1..b3af081c51e3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2198,6 +2198,7 @@ Generic Loader M: Alistair Francis S: Maintained F: hw/core/generic-loader.c +F: hw/core/uboot_image.h F: include/hw/core/generic-loader.h F: docs/system/generic-loader.rst From de799beba7f927b2a1ed38128309316511311605 Mon Sep 17 00:00:00 2001 From: Weiwei Li Date: Tue, 31 May 2022 11:07:32 +0800 Subject: [PATCH 63/90] target/riscv: add support for zmmul extension v0.1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add support for the zmmul extension v0.1. This extension includes all multiplication operations from the M extension but not the divide ops. Signed-off-by: Weiwei Li Signed-off-by: Junqiang Wang Reviewed-by: Víctor Colombo Reviewed-by: Alistair Francis Message-Id: <20220531030732.3850-1-liweiwei@iscas.ac.cn> Signed-off-by: Alistair Francis --- target/riscv/cpu.c | 7 +++++++ target/riscv/cpu.h | 1 + target/riscv/insn_trans/trans_rvm.c.inc | 18 ++++++++++++------ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index a91253d4bd53..bcbba3fbd58e 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -600,6 +600,11 @@ static void riscv_cpu_realize(DeviceState *dev, Error **errp) cpu->cfg.ext_ifencei = true; } + if (cpu->cfg.ext_m && cpu->cfg.ext_zmmul) { + warn_report("Zmmul will override M"); + cpu->cfg.ext_m = false; + } + if (cpu->cfg.ext_i && cpu->cfg.ext_e) { error_setg(errp, "I and E extensions are incompatible"); @@ -905,6 +910,7 @@ static Property riscv_cpu_properties[] = { /* These are experimental so mark with 'x-' */ DEFINE_PROP_BOOL("x-j", RISCVCPU, cfg.ext_j, false), + DEFINE_PROP_BOOL("x-zmmul", RISCVCPU, cfg.ext_zmmul, false), /* ePMP 0.9.3 */ DEFINE_PROP_BOOL("x-epmp", RISCVCPU, cfg.epmp, false), DEFINE_PROP_BOOL("x-aia", RISCVCPU, cfg.aia, false), @@ -1031,6 +1037,7 @@ static void riscv_isa_string_ext(RISCVCPU *cpu, char **isa_str, int max_str_len) struct isa_ext_data isa_edata_arr[] = { ISA_EDATA_ENTRY(zicsr, ext_icsr), ISA_EDATA_ENTRY(zifencei, ext_ifencei), + ISA_EDATA_ENTRY(zmmul, ext_zmmul), ISA_EDATA_ENTRY(zfh, ext_zfh), ISA_EDATA_ENTRY(zfhmin, ext_zfhmin), ISA_EDATA_ENTRY(zfinx, ext_zfinx), diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index f08c3e8813bf..890d33cebb2e 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -411,6 +411,7 @@ struct RISCVCPUConfig { bool ext_zhinxmin; bool ext_zve32f; bool ext_zve64f; + bool ext_zmmul; uint32_t mvendorid; uint64_t marchid; diff --git a/target/riscv/insn_trans/trans_rvm.c.inc b/target/riscv/insn_trans/trans_rvm.c.inc index 16b029edf004..ec7f705aabb3 100644 --- a/target/riscv/insn_trans/trans_rvm.c.inc +++ b/target/riscv/insn_trans/trans_rvm.c.inc @@ -18,6 +18,12 @@ * this program. If not, see . */ +#define REQUIRE_M_OR_ZMMUL(ctx) do { \ + if (!ctx->cfg_ptr->ext_zmmul && !has_ext(ctx, RVM)) { \ + return false; \ + } \ +} while (0) + static void gen_mulhu_i128(TCGv r2, TCGv r3, TCGv al, TCGv ah, TCGv bl, TCGv bh) { TCGv tmpl = tcg_temp_new(); @@ -65,7 +71,7 @@ static void gen_mul_i128(TCGv rl, TCGv rh, static bool trans_mul(DisasContext *ctx, arg_mul *a) { - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); return gen_arith(ctx, a, EXT_NONE, tcg_gen_mul_tl, gen_mul_i128); } @@ -109,7 +115,7 @@ static void gen_mulh_w(TCGv ret, TCGv s1, TCGv s2) static bool trans_mulh(DisasContext *ctx, arg_mulh *a) { - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); return gen_arith_per_ol(ctx, a, EXT_SIGN, gen_mulh, gen_mulh_w, gen_mulh_i128); } @@ -161,7 +167,7 @@ static void gen_mulhsu_w(TCGv ret, TCGv arg1, TCGv arg2) static bool trans_mulhsu(DisasContext *ctx, arg_mulhsu *a) { - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); return gen_arith_per_ol(ctx, a, EXT_NONE, gen_mulhsu, gen_mulhsu_w, gen_mulhsu_i128); } @@ -176,7 +182,7 @@ static void gen_mulhu(TCGv ret, TCGv s1, TCGv s2) static bool trans_mulhu(DisasContext *ctx, arg_mulhu *a) { - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); /* gen_mulh_w works for either sign as input. */ return gen_arith_per_ol(ctx, a, EXT_ZERO, gen_mulhu, gen_mulh_w, gen_mulhu_i128); @@ -349,7 +355,7 @@ static bool trans_remu(DisasContext *ctx, arg_remu *a) static bool trans_mulw(DisasContext *ctx, arg_mulw *a) { REQUIRE_64_OR_128BIT(ctx); - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); ctx->ol = MXL_RV32; return gen_arith(ctx, a, EXT_NONE, tcg_gen_mul_tl, NULL); } @@ -389,7 +395,7 @@ static bool trans_remuw(DisasContext *ctx, arg_remuw *a) static bool trans_muld(DisasContext *ctx, arg_muld *a) { REQUIRE_128BIT(ctx); - REQUIRE_EXT(ctx, RVM); + REQUIRE_M_OR_ZMMUL(ctx); ctx->ol = MXL_RV64; return gen_arith(ctx, a, EXT_SIGN, tcg_gen_mul_tl, NULL); } From f9a461b2d3b8ef4f36b7891eb4040693ee071719 Mon Sep 17 00:00:00 2001 From: Atish Patra Date: Thu, 26 May 2022 13:35:00 -0700 Subject: [PATCH 64/90] hw/riscv: virt: Generate fw_cfg DT node correctly fw_cfg DT node is generated after the create_fdt without any check if the DT is being loaded from the commandline. This results in FDT_ERR_EXISTS error if dtb is loaded from the commandline. Generate fw_cfg node only if the DT is not loaded from the commandline. Signed-off-by: Atish Patra Reviewed-by: Alistair Francis Message-Id: <20220526203500.847165-1-atishp@rivosinc.com> Signed-off-by: Alistair Francis --- hw/riscv/virt.c | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c index 293e9c95b7cc..bc424dd2f523 100644 --- a/hw/riscv/virt.c +++ b/hw/riscv/virt.c @@ -975,6 +975,23 @@ static void create_fdt_flash(RISCVVirtState *s, const MemMapEntry *memmap) g_free(name); } +static void create_fdt_fw_cfg(RISCVVirtState *s, const MemMapEntry *memmap) +{ + char *nodename; + MachineState *mc = MACHINE(s); + hwaddr base = memmap[VIRT_FW_CFG].base; + hwaddr size = memmap[VIRT_FW_CFG].size; + + nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base); + qemu_fdt_add_subnode(mc->fdt, nodename); + qemu_fdt_setprop_string(mc->fdt, nodename, + "compatible", "qemu,fw-cfg-mmio"); + qemu_fdt_setprop_sized_cells(mc->fdt, nodename, "reg", + 2, base, 2, size); + qemu_fdt_setprop(mc->fdt, nodename, "dma-coherent", NULL, 0); + g_free(nodename); +} + static void create_fdt(RISCVVirtState *s, const MemMapEntry *memmap, uint64_t mem_size, const char *cmdline, bool is_32_bit) { @@ -1023,6 +1040,7 @@ static void create_fdt(RISCVVirtState *s, const MemMapEntry *memmap, create_fdt_rtc(s, memmap, irq_mmio_phandle); create_fdt_flash(s, memmap); + create_fdt_fw_cfg(s, memmap); update_bootargs: if (cmdline && *cmdline) { @@ -1082,22 +1100,12 @@ static inline DeviceState *gpex_pcie_init(MemoryRegion *sys_mem, static FWCfgState *create_fw_cfg(const MachineState *mc) { hwaddr base = virt_memmap[VIRT_FW_CFG].base; - hwaddr size = virt_memmap[VIRT_FW_CFG].size; FWCfgState *fw_cfg; - char *nodename; fw_cfg = fw_cfg_init_mem_wide(base + 8, base, 8, base + 16, &address_space_memory); fw_cfg_add_i16(fw_cfg, FW_CFG_NB_CPUS, (uint16_t)mc->smp.cpus); - nodename = g_strdup_printf("/fw-cfg@%" PRIx64, base); - qemu_fdt_add_subnode(mc->fdt, nodename); - qemu_fdt_setprop_string(mc->fdt, nodename, - "compatible", "qemu,fw-cfg-mmio"); - qemu_fdt_setprop_sized_cells(mc->fdt, nodename, "reg", - 2, base, 2, size); - qemu_fdt_setprop(mc->fdt, nodename, "dma-coherent", NULL, 0); - g_free(nodename); return fw_cfg; } From 40244040a7ac00d40db4dea02234d13502c30112 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 1 Jun 2022 11:36:31 +1000 Subject: [PATCH 65/90] hw/intc: sifive_plic: Avoid overflowing the addr_config buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit ad40be27 "target/riscv: Support start kernel directly by KVM" we have been overflowing the addr_config on "M,MS..." configurations, as reported https://gitlab.com/qemu-project/qemu/-/issues/1050. This commit changes the loop in sifive_plic_create() from iterating over the number of harts to just iterating over the addr_config. The addr_config is based on the hart_config, and will contain interrup details for all harts. This way we can't iterate past the end of addr_config. Fixes: ad40be27084536 ("target/riscv: Support start kernel directly by KVM") Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1050 Signed-off-by: Alistair Francis Reviewed-by: Mingwang Li Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20220601013631.196854-1-alistair.francis@opensource.wdc.com> --- hw/intc/sifive_plic.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/hw/intc/sifive_plic.c b/hw/intc/sifive_plic.c index eebbcf33d4f6..56d60e9ac935 100644 --- a/hw/intc/sifive_plic.c +++ b/hw/intc/sifive_plic.c @@ -431,7 +431,7 @@ DeviceState *sifive_plic_create(hwaddr addr, char *hart_config, uint32_t context_stride, uint32_t aperture_size) { DeviceState *dev = qdev_new(TYPE_SIFIVE_PLIC); - int i, j = 0; + int i; SiFivePLICState *plic; assert(enable_stride == (enable_stride & -enable_stride)); @@ -451,18 +451,17 @@ DeviceState *sifive_plic_create(hwaddr addr, char *hart_config, sysbus_mmio_map(SYS_BUS_DEVICE(dev), 0, addr); plic = SIFIVE_PLIC(dev); - for (i = 0; i < num_harts; i++) { - CPUState *cpu = qemu_get_cpu(hartid_base + i); - if (plic->addr_config[j].mode == PLICMode_M) { - j++; - qdev_connect_gpio_out(dev, num_harts + i, + for (i = 0; i < plic->num_addrs; i++) { + int cpu_num = plic->addr_config[i].hartid; + CPUState *cpu = qemu_get_cpu(hartid_base + cpu_num); + + if (plic->addr_config[i].mode == PLICMode_M) { + qdev_connect_gpio_out(dev, num_harts + cpu_num, qdev_get_gpio_in(DEVICE(cpu), IRQ_M_EXT)); } - - if (plic->addr_config[j].mode == PLICMode_S) { - j++; - qdev_connect_gpio_out(dev, i, + if (plic->addr_config[i].mode == PLICMode_S) { + qdev_connect_gpio_out(dev, cpu_num, qdev_get_gpio_in(DEVICE(cpu), IRQ_S_EXT)); } } From af9751316e53cdf7e98131afe6928a5f4445fe16 Mon Sep 17 00:00:00 2001 From: Jamie Iles Date: Thu, 11 Nov 2021 14:11:40 +0000 Subject: [PATCH 66/90] hw/core/loader: return image sizes as ssize_t MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Various loader functions return an int which limits images to 2GB which is fine for things like a BIOS/kernel image, but if we want to be able to load memory images or large ramdisks then any file over 2GB would silently fail to load. Cc: Luc Michel Signed-off-by: Jamie Iles Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Luc Michel Reviewed-by: Alistair Francis Message-Id: <20211111141141.3295094-2-jamie@nuviainc.com> Signed-off-by: Alistair Francis --- hw/arm/armv7m.c | 2 +- hw/arm/boot.c | 8 ++-- hw/core/generic-loader.c | 2 +- hw/core/loader.c | 81 +++++++++++++++++++++------------------- hw/i386/x86.c | 2 +- hw/riscv/boot.c | 5 ++- include/hw/loader.h | 55 +++++++++++++-------------- 7 files changed, 80 insertions(+), 75 deletions(-) diff --git a/hw/arm/armv7m.c b/hw/arm/armv7m.c index 32349ec94b17..990861ee5efb 100644 --- a/hw/arm/armv7m.c +++ b/hw/arm/armv7m.c @@ -570,7 +570,7 @@ static void armv7m_reset(void *opaque) void armv7m_load_kernel(ARMCPU *cpu, const char *kernel_filename, int mem_size) { - int image_size; + ssize_t image_size; uint64_t entry; int big_endian; AddressSpace *as; diff --git a/hw/arm/boot.c b/hw/arm/boot.c index a8de33fd6470..ada2717f7605 100644 --- a/hw/arm/boot.c +++ b/hw/arm/boot.c @@ -881,7 +881,7 @@ static int do_arm_linux_init(Object *obj, void *opaque) return 0; } -static int64_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry, +static ssize_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry, uint64_t *lowaddr, uint64_t *highaddr, int elf_machine, AddressSpace *as) { @@ -892,7 +892,7 @@ static int64_t arm_load_elf(struct arm_boot_info *info, uint64_t *pentry, } elf_header; int data_swab = 0; bool big_endian; - int64_t ret = -1; + ssize_t ret = -1; Error *err = NULL; @@ -1014,7 +1014,7 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, /* Set up for a direct boot of a kernel image file. */ CPUState *cs; AddressSpace *as = arm_boot_address_space(cpu, info); - int kernel_size; + ssize_t kernel_size; int initrd_size; int is_linux = 0; uint64_t elf_entry; @@ -1093,7 +1093,7 @@ static void arm_setup_direct_kernel_boot(ARMCPU *cpu, if (kernel_size > info->ram_size) { error_report("kernel '%s' is too large to fit in RAM " - "(kernel size %d, RAM size %" PRId64 ")", + "(kernel size %zd, RAM size %" PRId64 ")", info->kernel_filename, kernel_size, info->ram_size); exit(1); } diff --git a/hw/core/generic-loader.c b/hw/core/generic-loader.c index c666545aa00a..4f4d77908dad 100644 --- a/hw/core/generic-loader.c +++ b/hw/core/generic-loader.c @@ -67,7 +67,7 @@ static void generic_loader_realize(DeviceState *dev, Error **errp) GenericLoaderState *s = GENERIC_LOADER(dev); hwaddr entry; int big_endian; - int size = 0; + ssize_t size = 0; s->set_pc = false; diff --git a/hw/core/loader.c b/hw/core/loader.c index edde657ac33b..0548830733e6 100644 --- a/hw/core/loader.c +++ b/hw/core/loader.c @@ -114,17 +114,17 @@ ssize_t read_targphys(const char *name, return did; } -int load_image_targphys(const char *filename, - hwaddr addr, uint64_t max_sz) +ssize_t load_image_targphys(const char *filename, + hwaddr addr, uint64_t max_sz) { return load_image_targphys_as(filename, addr, max_sz, NULL); } /* return the size or -1 if error */ -int load_image_targphys_as(const char *filename, - hwaddr addr, uint64_t max_sz, AddressSpace *as) +ssize_t load_image_targphys_as(const char *filename, + hwaddr addr, uint64_t max_sz, AddressSpace *as) { - int size; + ssize_t size; size = get_image_size(filename); if (size < 0 || size > max_sz) { @@ -138,9 +138,9 @@ int load_image_targphys_as(const char *filename, return size; } -int load_image_mr(const char *filename, MemoryRegion *mr) +ssize_t load_image_mr(const char *filename, MemoryRegion *mr) { - int size; + ssize_t size; if (!memory_access_is_direct(mr, false)) { /* Can only load an image into RAM or ROM */ @@ -222,8 +222,8 @@ static void bswap_ahdr(struct exec *e) : (_N_SEGMENT_ROUND (_N_TXTENDADDR(x, target_page_size), target_page_size))) -int load_aout(const char *filename, hwaddr addr, int max_sz, - int bswap_needed, hwaddr target_page_size) +ssize_t load_aout(const char *filename, hwaddr addr, int max_sz, + int bswap_needed, hwaddr target_page_size) { int fd; ssize_t size, ret; @@ -617,13 +617,14 @@ ssize_t gunzip(void *dst, size_t dstlen, uint8_t *src, size_t srclen) } /* Load a U-Boot image. */ -static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr, - int *is_linux, uint8_t image_type, - uint64_t (*translate_fn)(void *, uint64_t), - void *translate_opaque, AddressSpace *as) +static ssize_t load_uboot_image(const char *filename, hwaddr *ep, + hwaddr *loadaddr, int *is_linux, + uint8_t image_type, + uint64_t (*translate_fn)(void *, uint64_t), + void *translate_opaque, AddressSpace *as) { int fd; - int size; + ssize_t size; hwaddr address; uboot_image_header_t h; uboot_image_header_t *hdr = &h; @@ -760,40 +761,40 @@ static int load_uboot_image(const char *filename, hwaddr *ep, hwaddr *loadaddr, return ret; } -int load_uimage(const char *filename, hwaddr *ep, hwaddr *loadaddr, - int *is_linux, - uint64_t (*translate_fn)(void *, uint64_t), - void *translate_opaque) +ssize_t load_uimage(const char *filename, hwaddr *ep, hwaddr *loadaddr, + int *is_linux, + uint64_t (*translate_fn)(void *, uint64_t), + void *translate_opaque) { return load_uboot_image(filename, ep, loadaddr, is_linux, IH_TYPE_KERNEL, translate_fn, translate_opaque, NULL); } -int load_uimage_as(const char *filename, hwaddr *ep, hwaddr *loadaddr, - int *is_linux, - uint64_t (*translate_fn)(void *, uint64_t), - void *translate_opaque, AddressSpace *as) +ssize_t load_uimage_as(const char *filename, hwaddr *ep, hwaddr *loadaddr, + int *is_linux, + uint64_t (*translate_fn)(void *, uint64_t), + void *translate_opaque, AddressSpace *as) { return load_uboot_image(filename, ep, loadaddr, is_linux, IH_TYPE_KERNEL, translate_fn, translate_opaque, as); } /* Load a ramdisk. */ -int load_ramdisk(const char *filename, hwaddr addr, uint64_t max_sz) +ssize_t load_ramdisk(const char *filename, hwaddr addr, uint64_t max_sz) { return load_ramdisk_as(filename, addr, max_sz, NULL); } -int load_ramdisk_as(const char *filename, hwaddr addr, uint64_t max_sz, - AddressSpace *as) +ssize_t load_ramdisk_as(const char *filename, hwaddr addr, uint64_t max_sz, + AddressSpace *as) { return load_uboot_image(filename, NULL, &addr, NULL, IH_TYPE_RAMDISK, NULL, NULL, as); } /* Load a gzip-compressed kernel to a dynamically allocated buffer. */ -int load_image_gzipped_buffer(const char *filename, uint64_t max_sz, - uint8_t **buffer) +ssize_t load_image_gzipped_buffer(const char *filename, uint64_t max_sz, + uint8_t **buffer) { uint8_t *compressed_data = NULL; uint8_t *data = NULL; @@ -838,9 +839,9 @@ int load_image_gzipped_buffer(const char *filename, uint64_t max_sz, } /* Load a gzip-compressed kernel. */ -int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz) +ssize_t load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz) { - int bytes; + ssize_t bytes; uint8_t *data; bytes = load_image_gzipped_buffer(filename, max_sz, &data); @@ -970,14 +971,15 @@ static void *rom_set_mr(Rom *rom, Object *owner, const char *name, bool ro) return data; } -int rom_add_file(const char *file, const char *fw_dir, - hwaddr addr, int32_t bootindex, - bool option_rom, MemoryRegion *mr, - AddressSpace *as) +ssize_t rom_add_file(const char *file, const char *fw_dir, + hwaddr addr, int32_t bootindex, + bool option_rom, MemoryRegion *mr, + AddressSpace *as) { MachineClass *mc = MACHINE_GET_CLASS(qdev_get_machine()); Rom *rom; - int rc, fd = -1; + ssize_t rc; + int fd = -1; char devpath[100]; if (as && mr) { @@ -1019,7 +1021,7 @@ int rom_add_file(const char *file, const char *fw_dir, lseek(fd, 0, SEEK_SET); rc = read(fd, rom->data, rom->datasize); if (rc != rom->datasize) { - fprintf(stderr, "rom: file %-20s: read error: rc=%d (expected %zd)\n", + fprintf(stderr, "rom: file %-20s: read error: rc=%zd (expected %zd)\n", rom->name, rc, rom->datasize); goto err; } @@ -1138,12 +1140,12 @@ int rom_add_elf_program(const char *name, GMappedFile *mapped_file, void *data, return 0; } -int rom_add_vga(const char *file) +ssize_t rom_add_vga(const char *file) { return rom_add_file(file, "vgaroms", 0, -1, true, NULL, NULL); } -int rom_add_option(const char *file, int32_t bootindex) +ssize_t rom_add_option(const char *file, int32_t bootindex) { return rom_add_file(file, "genroms", 0, bootindex, true, NULL, NULL); } @@ -1846,11 +1848,12 @@ static int parse_hex_blob(const char *filename, hwaddr *addr, uint8_t *hex_blob, } /* return size or -1 if error */ -int load_targphys_hex_as(const char *filename, hwaddr *entry, AddressSpace *as) +ssize_t load_targphys_hex_as(const char *filename, hwaddr *entry, + AddressSpace *as) { gsize hex_blob_size; gchar *hex_blob; - int total_size = 0; + ssize_t total_size = 0; if (!g_file_get_contents(filename, &hex_blob, &hex_blob_size, NULL)) { return -1; diff --git a/hw/i386/x86.c b/hw/i386/x86.c index 78b05ab7a2d1..6003b4b2dfea 100644 --- a/hw/i386/x86.c +++ b/hw/i386/x86.c @@ -1115,7 +1115,7 @@ void x86_bios_rom_init(MachineState *ms, const char *default_firmware, char *filename; MemoryRegion *bios, *isa_bios; int bios_size, isa_bios_size; - int ret; + ssize_t ret; /* BIOS load */ bios_name = ms->firmware ?: default_firmware; diff --git a/hw/riscv/boot.c b/hw/riscv/boot.c index 57a41df8e990..2d80f40b3169 100644 --- a/hw/riscv/boot.c +++ b/hw/riscv/boot.c @@ -129,7 +129,8 @@ target_ulong riscv_load_firmware(const char *firmware_filename, hwaddr firmware_load_addr, symbol_fn_t sym_cb) { - uint64_t firmware_entry, firmware_size, firmware_end; + uint64_t firmware_entry, firmware_end; + ssize_t firmware_size; if (load_elf_ram_sym(firmware_filename, NULL, NULL, NULL, &firmware_entry, NULL, &firmware_end, NULL, @@ -185,7 +186,7 @@ target_ulong riscv_load_kernel(const char *kernel_filename, hwaddr riscv_load_initrd(const char *filename, uint64_t mem_size, uint64_t kernel_entry, hwaddr *start) { - int size; + ssize_t size; /* * We want to put the initrd far enough into RAM that when the diff --git a/include/hw/loader.h b/include/hw/loader.h index 5572108ba5e5..70248e0da779 100644 --- a/include/hw/loader.h +++ b/include/hw/loader.h @@ -40,8 +40,8 @@ ssize_t load_image_size(const char *filename, void *addr, size_t size); * * Returns the size of the loaded image on success, -1 otherwise. */ -int load_image_targphys_as(const char *filename, - hwaddr addr, uint64_t max_sz, AddressSpace *as); +ssize_t load_image_targphys_as(const char *filename, + hwaddr addr, uint64_t max_sz, AddressSpace *as); /**load_targphys_hex_as: * @filename: Path to the .hex file @@ -53,14 +53,15 @@ int load_image_targphys_as(const char *filename, * * Returns the size of the loaded .hex file on success, -1 otherwise. */ -int load_targphys_hex_as(const char *filename, hwaddr *entry, AddressSpace *as); +ssize_t load_targphys_hex_as(const char *filename, hwaddr *entry, + AddressSpace *as); /** load_image_targphys: * Same as load_image_targphys_as(), but doesn't allow the caller to specify * an AddressSpace. */ -int load_image_targphys(const char *filename, hwaddr, - uint64_t max_sz); +ssize_t load_image_targphys(const char *filename, hwaddr, + uint64_t max_sz); /** * load_image_mr: load an image into a memory region @@ -73,7 +74,7 @@ int load_image_targphys(const char *filename, hwaddr, * If the file is larger than the memory region's size the call will fail. * Returns -1 on failure, or the size of the file. */ -int load_image_mr(const char *filename, MemoryRegion *mr); +ssize_t load_image_mr(const char *filename, MemoryRegion *mr); /* This is the limit on the maximum uncompressed image size that * load_image_gzipped_buffer() and load_image_gzipped() will read. It prevents @@ -81,9 +82,9 @@ int load_image_mr(const char *filename, MemoryRegion *mr); */ #define LOAD_IMAGE_MAX_GUNZIP_BYTES (256 << 20) -int load_image_gzipped_buffer(const char *filename, uint64_t max_sz, - uint8_t **buffer); -int load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz); +ssize_t load_image_gzipped_buffer(const char *filename, uint64_t max_sz, + uint8_t **buffer); +ssize_t load_image_gzipped(const char *filename, hwaddr addr, uint64_t max_sz); #define ELF_LOAD_FAILED -1 #define ELF_LOAD_NOT_ELF -2 @@ -183,8 +184,8 @@ ssize_t load_elf(const char *filename, */ void load_elf_hdr(const char *filename, void *hdr, bool *is64, Error **errp); -int load_aout(const char *filename, hwaddr addr, int max_sz, - int bswap_needed, hwaddr target_page_size); +ssize_t load_aout(const char *filename, hwaddr addr, int max_sz, + int bswap_needed, hwaddr target_page_size); #define LOAD_UIMAGE_LOADADDR_INVALID (-1) @@ -205,19 +206,19 @@ int load_aout(const char *filename, hwaddr addr, int max_sz, * * Returns the size of the loaded image on success, -1 otherwise. */ -int load_uimage_as(const char *filename, hwaddr *ep, - hwaddr *loadaddr, int *is_linux, - uint64_t (*translate_fn)(void *, uint64_t), - void *translate_opaque, AddressSpace *as); +ssize_t load_uimage_as(const char *filename, hwaddr *ep, + hwaddr *loadaddr, int *is_linux, + uint64_t (*translate_fn)(void *, uint64_t), + void *translate_opaque, AddressSpace *as); /** load_uimage: * Same as load_uimage_as(), but doesn't allow the caller to specify an * AddressSpace. */ -int load_uimage(const char *filename, hwaddr *ep, - hwaddr *loadaddr, int *is_linux, - uint64_t (*translate_fn)(void *, uint64_t), - void *translate_opaque); +ssize_t load_uimage(const char *filename, hwaddr *ep, + hwaddr *loadaddr, int *is_linux, + uint64_t (*translate_fn)(void *, uint64_t), + void *translate_opaque); /** * load_ramdisk_as: @@ -232,15 +233,15 @@ int load_uimage(const char *filename, hwaddr *ep, * * Returns the size of the loaded image on success, -1 otherwise. */ -int load_ramdisk_as(const char *filename, hwaddr addr, uint64_t max_sz, - AddressSpace *as); +ssize_t load_ramdisk_as(const char *filename, hwaddr addr, uint64_t max_sz, + AddressSpace *as); /** * load_ramdisk: * Same as load_ramdisk_as(), but doesn't allow the caller to specify * an AddressSpace. */ -int load_ramdisk(const char *filename, hwaddr addr, uint64_t max_sz); +ssize_t load_ramdisk(const char *filename, hwaddr addr, uint64_t max_sz); ssize_t gunzip(void *dst, size_t dstlen, uint8_t *src, size_t srclen); @@ -253,9 +254,9 @@ void pstrcpy_targphys(const char *name, extern bool option_rom_has_mr; extern bool rom_file_has_mr; -int rom_add_file(const char *file, const char *fw_dir, - hwaddr addr, int32_t bootindex, - bool option_rom, MemoryRegion *mr, AddressSpace *as); +ssize_t rom_add_file(const char *file, const char *fw_dir, + hwaddr addr, int32_t bootindex, + bool option_rom, MemoryRegion *mr, AddressSpace *as); MemoryRegion *rom_add_blob(const char *name, const void *blob, size_t len, size_t max_len, hwaddr addr, const char *fw_file_name, @@ -336,8 +337,8 @@ void hmp_info_roms(Monitor *mon, const QDict *qdict); #define rom_add_blob_fixed_as(_f, _b, _l, _a, _as) \ rom_add_blob(_f, _b, _l, _l, _a, NULL, NULL, NULL, _as, true) -int rom_add_vga(const char *file); -int rom_add_option(const char *file, int32_t bootindex); +ssize_t rom_add_vga(const char *file); +ssize_t rom_add_option(const char *file, int32_t bootindex); /* This is the usual maximum in uboot, so if a uImage overflows this, it would * overflow on real hardware too. */ From 8f42415fc1d1bb462f2001bf5e2ad3b78f14b2e3 Mon Sep 17 00:00:00 2001 From: Andrew Bresticker Date: Tue, 31 May 2022 17:05:44 -0400 Subject: [PATCH 67/90] target/riscv: Wake on VS-level external interrupts Whether or not VSEIP is pending isn't reflected in env->mip and must instead be determined from hstatus.vgein and hgeip. As a result a CPU in WFI won't wake on a VSEIP, which violates the WFI behavior as specified in the privileged ISA. Just use riscv_cpu_all_pending() instead, which already accounts for VSEIP. Signed-off-by: Andrew Bresticker Reviewed-by: Alistair Francis Message-Id: <20220531210544.181322-1-abrestic@rivosinc.com> Signed-off-by: Alistair Francis --- target/riscv/cpu.c | 2 +- target/riscv/cpu.h | 1 + target/riscv/cpu_helper.c | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index bcbba3fbd58e..0497af45cc95 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -391,7 +391,7 @@ static bool riscv_cpu_has_work(CPUState *cs) * Definition of the WFI instruction requires it to ignore the privilege * mode and delegation registers, but respect individual enables */ - return (env->mip & env->mie) != 0; + return riscv_cpu_all_pending(env) != 0; #else return true; #endif diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 890d33cebb2e..194a58d760d5 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -489,6 +489,7 @@ int riscv_cpu_gdb_read_register(CPUState *cpu, GByteArray *buf, int reg); int riscv_cpu_gdb_write_register(CPUState *cpu, uint8_t *buf, int reg); int riscv_cpu_hviprio_index2irq(int index, int *out_irq, int *out_rdzero); uint8_t riscv_cpu_default_priority(int irq); +uint64_t riscv_cpu_all_pending(CPURISCVState *env); int riscv_cpu_mirq_pending(CPURISCVState *env); int riscv_cpu_sirq_pending(CPURISCVState *env); int riscv_cpu_vsirq_pending(CPURISCVState *env); diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index d99fac9d2ddb..16c6045459fd 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -340,7 +340,7 @@ static int riscv_cpu_pending_to_irq(CPURISCVState *env, return best_irq; } -static uint64_t riscv_cpu_all_pending(CPURISCVState *env) +uint64_t riscv_cpu_all_pending(CPURISCVState *env) { uint32_t gein = get_field(env->hstatus, HSTATUS_VGEIN); uint64_t vsgein = (env->hgeip & (1ULL << gein)) ? MIP_VSEIP : 0; From d1d8541217ce8a23e9e751cd868c7d618817134a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20P=C3=A9trot?= Date: Thu, 2 Jun 2022 17:52:46 +0200 Subject: [PATCH 68/90] target/riscv/debug.c: keep experimental rv128 support working MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an MXL_RV128 case in two switches so that no error is triggered when using the -cpu x-rv128 option. Signed-off-by: Frédéric Pétrot Acked-by: Alistair Francis Reviewed-by: Bin Meng Message-Id: <20220602155246.38837-1-frederic.petrot@univ-grenoble-alpes.fr> Signed-off-by: Alistair Francis --- target/riscv/debug.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/riscv/debug.c b/target/riscv/debug.c index 2f2a51c732a9..fc6e13222f2e 100644 --- a/target/riscv/debug.c +++ b/target/riscv/debug.c @@ -77,6 +77,7 @@ static inline target_ulong trigger_type(CPURISCVState *env, tdata1 = RV32_TYPE(type); break; case MXL_RV64: + case MXL_RV128: tdata1 = RV64_TYPE(type); break; default: @@ -123,6 +124,7 @@ static target_ulong tdata1_validate(CPURISCVState *env, target_ulong val, tdata1 = RV32_TYPE(t); break; case MXL_RV64: + case MXL_RV128: type = extract64(val, 60, 4); dmode = extract64(val, 59, 1); tdata1 = RV64_TYPE(t); From 8a085fb2ad812bf06145779bbb6a18a3e7439771 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 69/90] target/riscv: rvv: Prune redundant ESZ, DSZ parameter passed No functional change intended in this commit. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Reviewed-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-1@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 1132 +++++++++++++++++----------------- 1 file changed, 565 insertions(+), 567 deletions(-) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 576b14e5a36f..85dd611cd9ca 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -710,7 +710,6 @@ RVVCALL(OPIVV2, vsub_vv_d, OP_SSS_D, H8, H8, H8, DO_SUB) static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2, CPURISCVState *env, uint32_t desc, - uint32_t esz, uint32_t dsz, opivv2_fn *fn) { uint32_t vm = vext_vm(desc); @@ -727,23 +726,23 @@ static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2, } /* generate the helpers for OPIVV */ -#define GEN_VEXT_VV(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VV(NAME) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ - do_vext_vv(vd, v0, vs1, vs2, env, desc, ESZ, DSZ, \ + do_vext_vv(vd, v0, vs1, vs2, env, desc, \ do_##NAME); \ } -GEN_VEXT_VV(vadd_vv_b, 1, 1) -GEN_VEXT_VV(vadd_vv_h, 2, 2) -GEN_VEXT_VV(vadd_vv_w, 4, 4) -GEN_VEXT_VV(vadd_vv_d, 8, 8) -GEN_VEXT_VV(vsub_vv_b, 1, 1) -GEN_VEXT_VV(vsub_vv_h, 2, 2) -GEN_VEXT_VV(vsub_vv_w, 4, 4) -GEN_VEXT_VV(vsub_vv_d, 8, 8) +GEN_VEXT_VV(vadd_vv_b) +GEN_VEXT_VV(vadd_vv_h) +GEN_VEXT_VV(vadd_vv_w) +GEN_VEXT_VV(vadd_vv_d) +GEN_VEXT_VV(vsub_vv_b) +GEN_VEXT_VV(vsub_vv_h) +GEN_VEXT_VV(vsub_vv_w) +GEN_VEXT_VV(vsub_vv_d) typedef void opivx2_fn(void *vd, target_long s1, void *vs2, int i); @@ -773,7 +772,6 @@ RVVCALL(OPIVX2, vrsub_vx_d, OP_SSS_D, H8, H8, DO_RSUB) static void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2, CPURISCVState *env, uint32_t desc, - uint32_t esz, uint32_t dsz, opivx2_fn fn) { uint32_t vm = vext_vm(desc); @@ -790,27 +788,27 @@ static void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2, } /* generate the helpers for OPIVX */ -#define GEN_VEXT_VX(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VX(NAME) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ - do_vext_vx(vd, v0, s1, vs2, env, desc, ESZ, DSZ, \ + do_vext_vx(vd, v0, s1, vs2, env, desc, \ do_##NAME); \ } -GEN_VEXT_VX(vadd_vx_b, 1, 1) -GEN_VEXT_VX(vadd_vx_h, 2, 2) -GEN_VEXT_VX(vadd_vx_w, 4, 4) -GEN_VEXT_VX(vadd_vx_d, 8, 8) -GEN_VEXT_VX(vsub_vx_b, 1, 1) -GEN_VEXT_VX(vsub_vx_h, 2, 2) -GEN_VEXT_VX(vsub_vx_w, 4, 4) -GEN_VEXT_VX(vsub_vx_d, 8, 8) -GEN_VEXT_VX(vrsub_vx_b, 1, 1) -GEN_VEXT_VX(vrsub_vx_h, 2, 2) -GEN_VEXT_VX(vrsub_vx_w, 4, 4) -GEN_VEXT_VX(vrsub_vx_d, 8, 8) +GEN_VEXT_VX(vadd_vx_b) +GEN_VEXT_VX(vadd_vx_h) +GEN_VEXT_VX(vadd_vx_w) +GEN_VEXT_VX(vadd_vx_d) +GEN_VEXT_VX(vsub_vx_b) +GEN_VEXT_VX(vsub_vx_h) +GEN_VEXT_VX(vsub_vx_w) +GEN_VEXT_VX(vsub_vx_d) +GEN_VEXT_VX(vrsub_vx_b) +GEN_VEXT_VX(vrsub_vx_h) +GEN_VEXT_VX(vrsub_vx_w) +GEN_VEXT_VX(vrsub_vx_d) void HELPER(vec_rsubs8)(void *d, void *a, uint64_t b, uint32_t desc) { @@ -889,30 +887,30 @@ RVVCALL(OPIVV2, vwadd_wv_w, WOP_WSSS_W, H8, H4, H4, DO_ADD) RVVCALL(OPIVV2, vwsub_wv_b, WOP_WSSS_B, H2, H1, H1, DO_SUB) RVVCALL(OPIVV2, vwsub_wv_h, WOP_WSSS_H, H4, H2, H2, DO_SUB) RVVCALL(OPIVV2, vwsub_wv_w, WOP_WSSS_W, H8, H4, H4, DO_SUB) -GEN_VEXT_VV(vwaddu_vv_b, 1, 2) -GEN_VEXT_VV(vwaddu_vv_h, 2, 4) -GEN_VEXT_VV(vwaddu_vv_w, 4, 8) -GEN_VEXT_VV(vwsubu_vv_b, 1, 2) -GEN_VEXT_VV(vwsubu_vv_h, 2, 4) -GEN_VEXT_VV(vwsubu_vv_w, 4, 8) -GEN_VEXT_VV(vwadd_vv_b, 1, 2) -GEN_VEXT_VV(vwadd_vv_h, 2, 4) -GEN_VEXT_VV(vwadd_vv_w, 4, 8) -GEN_VEXT_VV(vwsub_vv_b, 1, 2) -GEN_VEXT_VV(vwsub_vv_h, 2, 4) -GEN_VEXT_VV(vwsub_vv_w, 4, 8) -GEN_VEXT_VV(vwaddu_wv_b, 1, 2) -GEN_VEXT_VV(vwaddu_wv_h, 2, 4) -GEN_VEXT_VV(vwaddu_wv_w, 4, 8) -GEN_VEXT_VV(vwsubu_wv_b, 1, 2) -GEN_VEXT_VV(vwsubu_wv_h, 2, 4) -GEN_VEXT_VV(vwsubu_wv_w, 4, 8) -GEN_VEXT_VV(vwadd_wv_b, 1, 2) -GEN_VEXT_VV(vwadd_wv_h, 2, 4) -GEN_VEXT_VV(vwadd_wv_w, 4, 8) -GEN_VEXT_VV(vwsub_wv_b, 1, 2) -GEN_VEXT_VV(vwsub_wv_h, 2, 4) -GEN_VEXT_VV(vwsub_wv_w, 4, 8) +GEN_VEXT_VV(vwaddu_vv_b) +GEN_VEXT_VV(vwaddu_vv_h) +GEN_VEXT_VV(vwaddu_vv_w) +GEN_VEXT_VV(vwsubu_vv_b) +GEN_VEXT_VV(vwsubu_vv_h) +GEN_VEXT_VV(vwsubu_vv_w) +GEN_VEXT_VV(vwadd_vv_b) +GEN_VEXT_VV(vwadd_vv_h) +GEN_VEXT_VV(vwadd_vv_w) +GEN_VEXT_VV(vwsub_vv_b) +GEN_VEXT_VV(vwsub_vv_h) +GEN_VEXT_VV(vwsub_vv_w) +GEN_VEXT_VV(vwaddu_wv_b) +GEN_VEXT_VV(vwaddu_wv_h) +GEN_VEXT_VV(vwaddu_wv_w) +GEN_VEXT_VV(vwsubu_wv_b) +GEN_VEXT_VV(vwsubu_wv_h) +GEN_VEXT_VV(vwsubu_wv_w) +GEN_VEXT_VV(vwadd_wv_b) +GEN_VEXT_VV(vwadd_wv_h) +GEN_VEXT_VV(vwadd_wv_w) +GEN_VEXT_VV(vwsub_wv_b) +GEN_VEXT_VV(vwsub_wv_h) +GEN_VEXT_VV(vwsub_wv_w) RVVCALL(OPIVX2, vwaddu_vx_b, WOP_UUU_B, H2, H1, DO_ADD) RVVCALL(OPIVX2, vwaddu_vx_h, WOP_UUU_H, H4, H2, DO_ADD) @@ -938,30 +936,30 @@ RVVCALL(OPIVX2, vwadd_wx_w, WOP_WSSS_W, H8, H4, DO_ADD) RVVCALL(OPIVX2, vwsub_wx_b, WOP_WSSS_B, H2, H1, DO_SUB) RVVCALL(OPIVX2, vwsub_wx_h, WOP_WSSS_H, H4, H2, DO_SUB) RVVCALL(OPIVX2, vwsub_wx_w, WOP_WSSS_W, H8, H4, DO_SUB) -GEN_VEXT_VX(vwaddu_vx_b, 1, 2) -GEN_VEXT_VX(vwaddu_vx_h, 2, 4) -GEN_VEXT_VX(vwaddu_vx_w, 4, 8) -GEN_VEXT_VX(vwsubu_vx_b, 1, 2) -GEN_VEXT_VX(vwsubu_vx_h, 2, 4) -GEN_VEXT_VX(vwsubu_vx_w, 4, 8) -GEN_VEXT_VX(vwadd_vx_b, 1, 2) -GEN_VEXT_VX(vwadd_vx_h, 2, 4) -GEN_VEXT_VX(vwadd_vx_w, 4, 8) -GEN_VEXT_VX(vwsub_vx_b, 1, 2) -GEN_VEXT_VX(vwsub_vx_h, 2, 4) -GEN_VEXT_VX(vwsub_vx_w, 4, 8) -GEN_VEXT_VX(vwaddu_wx_b, 1, 2) -GEN_VEXT_VX(vwaddu_wx_h, 2, 4) -GEN_VEXT_VX(vwaddu_wx_w, 4, 8) -GEN_VEXT_VX(vwsubu_wx_b, 1, 2) -GEN_VEXT_VX(vwsubu_wx_h, 2, 4) -GEN_VEXT_VX(vwsubu_wx_w, 4, 8) -GEN_VEXT_VX(vwadd_wx_b, 1, 2) -GEN_VEXT_VX(vwadd_wx_h, 2, 4) -GEN_VEXT_VX(vwadd_wx_w, 4, 8) -GEN_VEXT_VX(vwsub_wx_b, 1, 2) -GEN_VEXT_VX(vwsub_wx_h, 2, 4) -GEN_VEXT_VX(vwsub_wx_w, 4, 8) +GEN_VEXT_VX(vwaddu_vx_b) +GEN_VEXT_VX(vwaddu_vx_h) +GEN_VEXT_VX(vwaddu_vx_w) +GEN_VEXT_VX(vwsubu_vx_b) +GEN_VEXT_VX(vwsubu_vx_h) +GEN_VEXT_VX(vwsubu_vx_w) +GEN_VEXT_VX(vwadd_vx_b) +GEN_VEXT_VX(vwadd_vx_h) +GEN_VEXT_VX(vwadd_vx_w) +GEN_VEXT_VX(vwsub_vx_b) +GEN_VEXT_VX(vwsub_vx_h) +GEN_VEXT_VX(vwsub_vx_w) +GEN_VEXT_VX(vwaddu_wx_b) +GEN_VEXT_VX(vwaddu_wx_h) +GEN_VEXT_VX(vwaddu_wx_w) +GEN_VEXT_VX(vwsubu_wx_b) +GEN_VEXT_VX(vwsubu_wx_h) +GEN_VEXT_VX(vwsubu_wx_w) +GEN_VEXT_VX(vwadd_wx_b) +GEN_VEXT_VX(vwadd_wx_h) +GEN_VEXT_VX(vwadd_wx_w) +GEN_VEXT_VX(vwsub_wx_b) +GEN_VEXT_VX(vwsub_wx_h) +GEN_VEXT_VX(vwsub_wx_w) /* Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions */ #define DO_VADC(N, M, C) (N + M + C) @@ -1091,18 +1089,18 @@ RVVCALL(OPIVV2, vxor_vv_b, OP_SSS_B, H1, H1, H1, DO_XOR) RVVCALL(OPIVV2, vxor_vv_h, OP_SSS_H, H2, H2, H2, DO_XOR) RVVCALL(OPIVV2, vxor_vv_w, OP_SSS_W, H4, H4, H4, DO_XOR) RVVCALL(OPIVV2, vxor_vv_d, OP_SSS_D, H8, H8, H8, DO_XOR) -GEN_VEXT_VV(vand_vv_b, 1, 1) -GEN_VEXT_VV(vand_vv_h, 2, 2) -GEN_VEXT_VV(vand_vv_w, 4, 4) -GEN_VEXT_VV(vand_vv_d, 8, 8) -GEN_VEXT_VV(vor_vv_b, 1, 1) -GEN_VEXT_VV(vor_vv_h, 2, 2) -GEN_VEXT_VV(vor_vv_w, 4, 4) -GEN_VEXT_VV(vor_vv_d, 8, 8) -GEN_VEXT_VV(vxor_vv_b, 1, 1) -GEN_VEXT_VV(vxor_vv_h, 2, 2) -GEN_VEXT_VV(vxor_vv_w, 4, 4) -GEN_VEXT_VV(vxor_vv_d, 8, 8) +GEN_VEXT_VV(vand_vv_b) +GEN_VEXT_VV(vand_vv_h) +GEN_VEXT_VV(vand_vv_w) +GEN_VEXT_VV(vand_vv_d) +GEN_VEXT_VV(vor_vv_b) +GEN_VEXT_VV(vor_vv_h) +GEN_VEXT_VV(vor_vv_w) +GEN_VEXT_VV(vor_vv_d) +GEN_VEXT_VV(vxor_vv_b) +GEN_VEXT_VV(vxor_vv_h) +GEN_VEXT_VV(vxor_vv_w) +GEN_VEXT_VV(vxor_vv_d) RVVCALL(OPIVX2, vand_vx_b, OP_SSS_B, H1, H1, DO_AND) RVVCALL(OPIVX2, vand_vx_h, OP_SSS_H, H2, H2, DO_AND) @@ -1116,18 +1114,18 @@ RVVCALL(OPIVX2, vxor_vx_b, OP_SSS_B, H1, H1, DO_XOR) RVVCALL(OPIVX2, vxor_vx_h, OP_SSS_H, H2, H2, DO_XOR) RVVCALL(OPIVX2, vxor_vx_w, OP_SSS_W, H4, H4, DO_XOR) RVVCALL(OPIVX2, vxor_vx_d, OP_SSS_D, H8, H8, DO_XOR) -GEN_VEXT_VX(vand_vx_b, 1, 1) -GEN_VEXT_VX(vand_vx_h, 2, 2) -GEN_VEXT_VX(vand_vx_w, 4, 4) -GEN_VEXT_VX(vand_vx_d, 8, 8) -GEN_VEXT_VX(vor_vx_b, 1, 1) -GEN_VEXT_VX(vor_vx_h, 2, 2) -GEN_VEXT_VX(vor_vx_w, 4, 4) -GEN_VEXT_VX(vor_vx_d, 8, 8) -GEN_VEXT_VX(vxor_vx_b, 1, 1) -GEN_VEXT_VX(vxor_vx_h, 2, 2) -GEN_VEXT_VX(vxor_vx_w, 4, 4) -GEN_VEXT_VX(vxor_vx_d, 8, 8) +GEN_VEXT_VX(vand_vx_b) +GEN_VEXT_VX(vand_vx_h) +GEN_VEXT_VX(vand_vx_w) +GEN_VEXT_VX(vand_vx_d) +GEN_VEXT_VX(vor_vx_b) +GEN_VEXT_VX(vor_vx_h) +GEN_VEXT_VX(vor_vx_w) +GEN_VEXT_VX(vor_vx_d) +GEN_VEXT_VX(vxor_vx_b) +GEN_VEXT_VX(vxor_vx_h) +GEN_VEXT_VX(vxor_vx_w) +GEN_VEXT_VX(vxor_vx_d) /* Vector Single-Width Bit Shift Instructions */ #define DO_SLL(N, M) (N << (M)) @@ -1348,22 +1346,22 @@ RVVCALL(OPIVV2, vmax_vv_b, OP_SSS_B, H1, H1, H1, DO_MAX) RVVCALL(OPIVV2, vmax_vv_h, OP_SSS_H, H2, H2, H2, DO_MAX) RVVCALL(OPIVV2, vmax_vv_w, OP_SSS_W, H4, H4, H4, DO_MAX) RVVCALL(OPIVV2, vmax_vv_d, OP_SSS_D, H8, H8, H8, DO_MAX) -GEN_VEXT_VV(vminu_vv_b, 1, 1) -GEN_VEXT_VV(vminu_vv_h, 2, 2) -GEN_VEXT_VV(vminu_vv_w, 4, 4) -GEN_VEXT_VV(vminu_vv_d, 8, 8) -GEN_VEXT_VV(vmin_vv_b, 1, 1) -GEN_VEXT_VV(vmin_vv_h, 2, 2) -GEN_VEXT_VV(vmin_vv_w, 4, 4) -GEN_VEXT_VV(vmin_vv_d, 8, 8) -GEN_VEXT_VV(vmaxu_vv_b, 1, 1) -GEN_VEXT_VV(vmaxu_vv_h, 2, 2) -GEN_VEXT_VV(vmaxu_vv_w, 4, 4) -GEN_VEXT_VV(vmaxu_vv_d, 8, 8) -GEN_VEXT_VV(vmax_vv_b, 1, 1) -GEN_VEXT_VV(vmax_vv_h, 2, 2) -GEN_VEXT_VV(vmax_vv_w, 4, 4) -GEN_VEXT_VV(vmax_vv_d, 8, 8) +GEN_VEXT_VV(vminu_vv_b) +GEN_VEXT_VV(vminu_vv_h) +GEN_VEXT_VV(vminu_vv_w) +GEN_VEXT_VV(vminu_vv_d) +GEN_VEXT_VV(vmin_vv_b) +GEN_VEXT_VV(vmin_vv_h) +GEN_VEXT_VV(vmin_vv_w) +GEN_VEXT_VV(vmin_vv_d) +GEN_VEXT_VV(vmaxu_vv_b) +GEN_VEXT_VV(vmaxu_vv_h) +GEN_VEXT_VV(vmaxu_vv_w) +GEN_VEXT_VV(vmaxu_vv_d) +GEN_VEXT_VV(vmax_vv_b) +GEN_VEXT_VV(vmax_vv_h) +GEN_VEXT_VV(vmax_vv_w) +GEN_VEXT_VV(vmax_vv_d) RVVCALL(OPIVX2, vminu_vx_b, OP_UUU_B, H1, H1, DO_MIN) RVVCALL(OPIVX2, vminu_vx_h, OP_UUU_H, H2, H2, DO_MIN) @@ -1381,22 +1379,22 @@ RVVCALL(OPIVX2, vmax_vx_b, OP_SSS_B, H1, H1, DO_MAX) RVVCALL(OPIVX2, vmax_vx_h, OP_SSS_H, H2, H2, DO_MAX) RVVCALL(OPIVX2, vmax_vx_w, OP_SSS_W, H4, H4, DO_MAX) RVVCALL(OPIVX2, vmax_vx_d, OP_SSS_D, H8, H8, DO_MAX) -GEN_VEXT_VX(vminu_vx_b, 1, 1) -GEN_VEXT_VX(vminu_vx_h, 2, 2) -GEN_VEXT_VX(vminu_vx_w, 4, 4) -GEN_VEXT_VX(vminu_vx_d, 8, 8) -GEN_VEXT_VX(vmin_vx_b, 1, 1) -GEN_VEXT_VX(vmin_vx_h, 2, 2) -GEN_VEXT_VX(vmin_vx_w, 4, 4) -GEN_VEXT_VX(vmin_vx_d, 8, 8) -GEN_VEXT_VX(vmaxu_vx_b, 1, 1) -GEN_VEXT_VX(vmaxu_vx_h, 2, 2) -GEN_VEXT_VX(vmaxu_vx_w, 4, 4) -GEN_VEXT_VX(vmaxu_vx_d, 8, 8) -GEN_VEXT_VX(vmax_vx_b, 1, 1) -GEN_VEXT_VX(vmax_vx_h, 2, 2) -GEN_VEXT_VX(vmax_vx_w, 4, 4) -GEN_VEXT_VX(vmax_vx_d, 8, 8) +GEN_VEXT_VX(vminu_vx_b) +GEN_VEXT_VX(vminu_vx_h) +GEN_VEXT_VX(vminu_vx_w) +GEN_VEXT_VX(vminu_vx_d) +GEN_VEXT_VX(vmin_vx_b) +GEN_VEXT_VX(vmin_vx_h) +GEN_VEXT_VX(vmin_vx_w) +GEN_VEXT_VX(vmin_vx_d) +GEN_VEXT_VX(vmaxu_vx_b) +GEN_VEXT_VX(vmaxu_vx_h) +GEN_VEXT_VX(vmaxu_vx_w) +GEN_VEXT_VX(vmaxu_vx_d) +GEN_VEXT_VX(vmax_vx_b) +GEN_VEXT_VX(vmax_vx_h) +GEN_VEXT_VX(vmax_vx_w) +GEN_VEXT_VX(vmax_vx_d) /* Vector Single-Width Integer Multiply Instructions */ #define DO_MUL(N, M) (N * M) @@ -1404,10 +1402,10 @@ RVVCALL(OPIVV2, vmul_vv_b, OP_SSS_B, H1, H1, H1, DO_MUL) RVVCALL(OPIVV2, vmul_vv_h, OP_SSS_H, H2, H2, H2, DO_MUL) RVVCALL(OPIVV2, vmul_vv_w, OP_SSS_W, H4, H4, H4, DO_MUL) RVVCALL(OPIVV2, vmul_vv_d, OP_SSS_D, H8, H8, H8, DO_MUL) -GEN_VEXT_VV(vmul_vv_b, 1, 1) -GEN_VEXT_VV(vmul_vv_h, 2, 2) -GEN_VEXT_VV(vmul_vv_w, 4, 4) -GEN_VEXT_VV(vmul_vv_d, 8, 8) +GEN_VEXT_VV(vmul_vv_b) +GEN_VEXT_VV(vmul_vv_h) +GEN_VEXT_VV(vmul_vv_w) +GEN_VEXT_VV(vmul_vv_d) static int8_t do_mulh_b(int8_t s2, int8_t s1) { @@ -1511,18 +1509,18 @@ RVVCALL(OPIVV2, vmulhsu_vv_b, OP_SUS_B, H1, H1, H1, do_mulhsu_b) RVVCALL(OPIVV2, vmulhsu_vv_h, OP_SUS_H, H2, H2, H2, do_mulhsu_h) RVVCALL(OPIVV2, vmulhsu_vv_w, OP_SUS_W, H4, H4, H4, do_mulhsu_w) RVVCALL(OPIVV2, vmulhsu_vv_d, OP_SUS_D, H8, H8, H8, do_mulhsu_d) -GEN_VEXT_VV(vmulh_vv_b, 1, 1) -GEN_VEXT_VV(vmulh_vv_h, 2, 2) -GEN_VEXT_VV(vmulh_vv_w, 4, 4) -GEN_VEXT_VV(vmulh_vv_d, 8, 8) -GEN_VEXT_VV(vmulhu_vv_b, 1, 1) -GEN_VEXT_VV(vmulhu_vv_h, 2, 2) -GEN_VEXT_VV(vmulhu_vv_w, 4, 4) -GEN_VEXT_VV(vmulhu_vv_d, 8, 8) -GEN_VEXT_VV(vmulhsu_vv_b, 1, 1) -GEN_VEXT_VV(vmulhsu_vv_h, 2, 2) -GEN_VEXT_VV(vmulhsu_vv_w, 4, 4) -GEN_VEXT_VV(vmulhsu_vv_d, 8, 8) +GEN_VEXT_VV(vmulh_vv_b) +GEN_VEXT_VV(vmulh_vv_h) +GEN_VEXT_VV(vmulh_vv_w) +GEN_VEXT_VV(vmulh_vv_d) +GEN_VEXT_VV(vmulhu_vv_b) +GEN_VEXT_VV(vmulhu_vv_h) +GEN_VEXT_VV(vmulhu_vv_w) +GEN_VEXT_VV(vmulhu_vv_d) +GEN_VEXT_VV(vmulhsu_vv_b) +GEN_VEXT_VV(vmulhsu_vv_h) +GEN_VEXT_VV(vmulhsu_vv_w) +GEN_VEXT_VV(vmulhsu_vv_d) RVVCALL(OPIVX2, vmul_vx_b, OP_SSS_B, H1, H1, DO_MUL) RVVCALL(OPIVX2, vmul_vx_h, OP_SSS_H, H2, H2, DO_MUL) @@ -1540,22 +1538,22 @@ RVVCALL(OPIVX2, vmulhsu_vx_b, OP_SUS_B, H1, H1, do_mulhsu_b) RVVCALL(OPIVX2, vmulhsu_vx_h, OP_SUS_H, H2, H2, do_mulhsu_h) RVVCALL(OPIVX2, vmulhsu_vx_w, OP_SUS_W, H4, H4, do_mulhsu_w) RVVCALL(OPIVX2, vmulhsu_vx_d, OP_SUS_D, H8, H8, do_mulhsu_d) -GEN_VEXT_VX(vmul_vx_b, 1, 1) -GEN_VEXT_VX(vmul_vx_h, 2, 2) -GEN_VEXT_VX(vmul_vx_w, 4, 4) -GEN_VEXT_VX(vmul_vx_d, 8, 8) -GEN_VEXT_VX(vmulh_vx_b, 1, 1) -GEN_VEXT_VX(vmulh_vx_h, 2, 2) -GEN_VEXT_VX(vmulh_vx_w, 4, 4) -GEN_VEXT_VX(vmulh_vx_d, 8, 8) -GEN_VEXT_VX(vmulhu_vx_b, 1, 1) -GEN_VEXT_VX(vmulhu_vx_h, 2, 2) -GEN_VEXT_VX(vmulhu_vx_w, 4, 4) -GEN_VEXT_VX(vmulhu_vx_d, 8, 8) -GEN_VEXT_VX(vmulhsu_vx_b, 1, 1) -GEN_VEXT_VX(vmulhsu_vx_h, 2, 2) -GEN_VEXT_VX(vmulhsu_vx_w, 4, 4) -GEN_VEXT_VX(vmulhsu_vx_d, 8, 8) +GEN_VEXT_VX(vmul_vx_b) +GEN_VEXT_VX(vmul_vx_h) +GEN_VEXT_VX(vmul_vx_w) +GEN_VEXT_VX(vmul_vx_d) +GEN_VEXT_VX(vmulh_vx_b) +GEN_VEXT_VX(vmulh_vx_h) +GEN_VEXT_VX(vmulh_vx_w) +GEN_VEXT_VX(vmulh_vx_d) +GEN_VEXT_VX(vmulhu_vx_b) +GEN_VEXT_VX(vmulhu_vx_h) +GEN_VEXT_VX(vmulhu_vx_w) +GEN_VEXT_VX(vmulhu_vx_d) +GEN_VEXT_VX(vmulhsu_vx_b) +GEN_VEXT_VX(vmulhsu_vx_h) +GEN_VEXT_VX(vmulhsu_vx_w) +GEN_VEXT_VX(vmulhsu_vx_d) /* Vector Integer Divide Instructions */ #define DO_DIVU(N, M) (unlikely(M == 0) ? (__typeof(N))(-1) : N / M) @@ -1581,22 +1579,22 @@ RVVCALL(OPIVV2, vrem_vv_b, OP_SSS_B, H1, H1, H1, DO_REM) RVVCALL(OPIVV2, vrem_vv_h, OP_SSS_H, H2, H2, H2, DO_REM) RVVCALL(OPIVV2, vrem_vv_w, OP_SSS_W, H4, H4, H4, DO_REM) RVVCALL(OPIVV2, vrem_vv_d, OP_SSS_D, H8, H8, H8, DO_REM) -GEN_VEXT_VV(vdivu_vv_b, 1, 1) -GEN_VEXT_VV(vdivu_vv_h, 2, 2) -GEN_VEXT_VV(vdivu_vv_w, 4, 4) -GEN_VEXT_VV(vdivu_vv_d, 8, 8) -GEN_VEXT_VV(vdiv_vv_b, 1, 1) -GEN_VEXT_VV(vdiv_vv_h, 2, 2) -GEN_VEXT_VV(vdiv_vv_w, 4, 4) -GEN_VEXT_VV(vdiv_vv_d, 8, 8) -GEN_VEXT_VV(vremu_vv_b, 1, 1) -GEN_VEXT_VV(vremu_vv_h, 2, 2) -GEN_VEXT_VV(vremu_vv_w, 4, 4) -GEN_VEXT_VV(vremu_vv_d, 8, 8) -GEN_VEXT_VV(vrem_vv_b, 1, 1) -GEN_VEXT_VV(vrem_vv_h, 2, 2) -GEN_VEXT_VV(vrem_vv_w, 4, 4) -GEN_VEXT_VV(vrem_vv_d, 8, 8) +GEN_VEXT_VV(vdivu_vv_b) +GEN_VEXT_VV(vdivu_vv_h) +GEN_VEXT_VV(vdivu_vv_w) +GEN_VEXT_VV(vdivu_vv_d) +GEN_VEXT_VV(vdiv_vv_b) +GEN_VEXT_VV(vdiv_vv_h) +GEN_VEXT_VV(vdiv_vv_w) +GEN_VEXT_VV(vdiv_vv_d) +GEN_VEXT_VV(vremu_vv_b) +GEN_VEXT_VV(vremu_vv_h) +GEN_VEXT_VV(vremu_vv_w) +GEN_VEXT_VV(vremu_vv_d) +GEN_VEXT_VV(vrem_vv_b) +GEN_VEXT_VV(vrem_vv_h) +GEN_VEXT_VV(vrem_vv_w) +GEN_VEXT_VV(vrem_vv_d) RVVCALL(OPIVX2, vdivu_vx_b, OP_UUU_B, H1, H1, DO_DIVU) RVVCALL(OPIVX2, vdivu_vx_h, OP_UUU_H, H2, H2, DO_DIVU) @@ -1614,22 +1612,22 @@ RVVCALL(OPIVX2, vrem_vx_b, OP_SSS_B, H1, H1, DO_REM) RVVCALL(OPIVX2, vrem_vx_h, OP_SSS_H, H2, H2, DO_REM) RVVCALL(OPIVX2, vrem_vx_w, OP_SSS_W, H4, H4, DO_REM) RVVCALL(OPIVX2, vrem_vx_d, OP_SSS_D, H8, H8, DO_REM) -GEN_VEXT_VX(vdivu_vx_b, 1, 1) -GEN_VEXT_VX(vdivu_vx_h, 2, 2) -GEN_VEXT_VX(vdivu_vx_w, 4, 4) -GEN_VEXT_VX(vdivu_vx_d, 8, 8) -GEN_VEXT_VX(vdiv_vx_b, 1, 1) -GEN_VEXT_VX(vdiv_vx_h, 2, 2) -GEN_VEXT_VX(vdiv_vx_w, 4, 4) -GEN_VEXT_VX(vdiv_vx_d, 8, 8) -GEN_VEXT_VX(vremu_vx_b, 1, 1) -GEN_VEXT_VX(vremu_vx_h, 2, 2) -GEN_VEXT_VX(vremu_vx_w, 4, 4) -GEN_VEXT_VX(vremu_vx_d, 8, 8) -GEN_VEXT_VX(vrem_vx_b, 1, 1) -GEN_VEXT_VX(vrem_vx_h, 2, 2) -GEN_VEXT_VX(vrem_vx_w, 4, 4) -GEN_VEXT_VX(vrem_vx_d, 8, 8) +GEN_VEXT_VX(vdivu_vx_b) +GEN_VEXT_VX(vdivu_vx_h) +GEN_VEXT_VX(vdivu_vx_w) +GEN_VEXT_VX(vdivu_vx_d) +GEN_VEXT_VX(vdiv_vx_b) +GEN_VEXT_VX(vdiv_vx_h) +GEN_VEXT_VX(vdiv_vx_w) +GEN_VEXT_VX(vdiv_vx_d) +GEN_VEXT_VX(vremu_vx_b) +GEN_VEXT_VX(vremu_vx_h) +GEN_VEXT_VX(vremu_vx_w) +GEN_VEXT_VX(vremu_vx_d) +GEN_VEXT_VX(vrem_vx_b) +GEN_VEXT_VX(vrem_vx_h) +GEN_VEXT_VX(vrem_vx_w) +GEN_VEXT_VX(vrem_vx_d) /* Vector Widening Integer Multiply Instructions */ RVVCALL(OPIVV2, vwmul_vv_b, WOP_SSS_B, H2, H1, H1, DO_MUL) @@ -1641,15 +1639,15 @@ RVVCALL(OPIVV2, vwmulu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_b, WOP_SUS_B, H2, H1, H1, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_h, WOP_SUS_H, H4, H2, H2, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_w, WOP_SUS_W, H8, H4, H4, DO_MUL) -GEN_VEXT_VV(vwmul_vv_b, 1, 2) -GEN_VEXT_VV(vwmul_vv_h, 2, 4) -GEN_VEXT_VV(vwmul_vv_w, 4, 8) -GEN_VEXT_VV(vwmulu_vv_b, 1, 2) -GEN_VEXT_VV(vwmulu_vv_h, 2, 4) -GEN_VEXT_VV(vwmulu_vv_w, 4, 8) -GEN_VEXT_VV(vwmulsu_vv_b, 1, 2) -GEN_VEXT_VV(vwmulsu_vv_h, 2, 4) -GEN_VEXT_VV(vwmulsu_vv_w, 4, 8) +GEN_VEXT_VV(vwmul_vv_b) +GEN_VEXT_VV(vwmul_vv_h) +GEN_VEXT_VV(vwmul_vv_w) +GEN_VEXT_VV(vwmulu_vv_b) +GEN_VEXT_VV(vwmulu_vv_h) +GEN_VEXT_VV(vwmulu_vv_w) +GEN_VEXT_VV(vwmulsu_vv_b) +GEN_VEXT_VV(vwmulsu_vv_h) +GEN_VEXT_VV(vwmulsu_vv_w) RVVCALL(OPIVX2, vwmul_vx_b, WOP_SSS_B, H2, H1, DO_MUL) RVVCALL(OPIVX2, vwmul_vx_h, WOP_SSS_H, H4, H2, DO_MUL) @@ -1660,15 +1658,15 @@ RVVCALL(OPIVX2, vwmulu_vx_w, WOP_UUU_W, H8, H4, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_b, WOP_SUS_B, H2, H1, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_h, WOP_SUS_H, H4, H2, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_w, WOP_SUS_W, H8, H4, DO_MUL) -GEN_VEXT_VX(vwmul_vx_b, 1, 2) -GEN_VEXT_VX(vwmul_vx_h, 2, 4) -GEN_VEXT_VX(vwmul_vx_w, 4, 8) -GEN_VEXT_VX(vwmulu_vx_b, 1, 2) -GEN_VEXT_VX(vwmulu_vx_h, 2, 4) -GEN_VEXT_VX(vwmulu_vx_w, 4, 8) -GEN_VEXT_VX(vwmulsu_vx_b, 1, 2) -GEN_VEXT_VX(vwmulsu_vx_h, 2, 4) -GEN_VEXT_VX(vwmulsu_vx_w, 4, 8) +GEN_VEXT_VX(vwmul_vx_b) +GEN_VEXT_VX(vwmul_vx_h) +GEN_VEXT_VX(vwmul_vx_w) +GEN_VEXT_VX(vwmulu_vx_b) +GEN_VEXT_VX(vwmulu_vx_h) +GEN_VEXT_VX(vwmulu_vx_w) +GEN_VEXT_VX(vwmulsu_vx_b) +GEN_VEXT_VX(vwmulsu_vx_h) +GEN_VEXT_VX(vwmulsu_vx_w) /* Vector Single-Width Integer Multiply-Add Instructions */ #define OPIVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \ @@ -1700,22 +1698,22 @@ RVVCALL(OPIVV3, vnmsub_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSUB) -GEN_VEXT_VV(vmacc_vv_b, 1, 1) -GEN_VEXT_VV(vmacc_vv_h, 2, 2) -GEN_VEXT_VV(vmacc_vv_w, 4, 4) -GEN_VEXT_VV(vmacc_vv_d, 8, 8) -GEN_VEXT_VV(vnmsac_vv_b, 1, 1) -GEN_VEXT_VV(vnmsac_vv_h, 2, 2) -GEN_VEXT_VV(vnmsac_vv_w, 4, 4) -GEN_VEXT_VV(vnmsac_vv_d, 8, 8) -GEN_VEXT_VV(vmadd_vv_b, 1, 1) -GEN_VEXT_VV(vmadd_vv_h, 2, 2) -GEN_VEXT_VV(vmadd_vv_w, 4, 4) -GEN_VEXT_VV(vmadd_vv_d, 8, 8) -GEN_VEXT_VV(vnmsub_vv_b, 1, 1) -GEN_VEXT_VV(vnmsub_vv_h, 2, 2) -GEN_VEXT_VV(vnmsub_vv_w, 4, 4) -GEN_VEXT_VV(vnmsub_vv_d, 8, 8) +GEN_VEXT_VV(vmacc_vv_b) +GEN_VEXT_VV(vmacc_vv_h) +GEN_VEXT_VV(vmacc_vv_w) +GEN_VEXT_VV(vmacc_vv_d) +GEN_VEXT_VV(vnmsac_vv_b) +GEN_VEXT_VV(vnmsac_vv_h) +GEN_VEXT_VV(vnmsac_vv_w) +GEN_VEXT_VV(vnmsac_vv_d) +GEN_VEXT_VV(vmadd_vv_b) +GEN_VEXT_VV(vmadd_vv_h) +GEN_VEXT_VV(vmadd_vv_w) +GEN_VEXT_VV(vmadd_vv_d) +GEN_VEXT_VV(vnmsub_vv_b) +GEN_VEXT_VV(vnmsub_vv_h) +GEN_VEXT_VV(vnmsub_vv_w) +GEN_VEXT_VV(vnmsub_vv_d) #define OPIVX3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, target_long s1, void *vs2, int i) \ @@ -1741,22 +1739,22 @@ RVVCALL(OPIVX3, vnmsub_vx_b, OP_SSS_B, H1, H1, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_h, OP_SSS_H, H2, H2, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_w, OP_SSS_W, H4, H4, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_d, OP_SSS_D, H8, H8, DO_NMSUB) -GEN_VEXT_VX(vmacc_vx_b, 1, 1) -GEN_VEXT_VX(vmacc_vx_h, 2, 2) -GEN_VEXT_VX(vmacc_vx_w, 4, 4) -GEN_VEXT_VX(vmacc_vx_d, 8, 8) -GEN_VEXT_VX(vnmsac_vx_b, 1, 1) -GEN_VEXT_VX(vnmsac_vx_h, 2, 2) -GEN_VEXT_VX(vnmsac_vx_w, 4, 4) -GEN_VEXT_VX(vnmsac_vx_d, 8, 8) -GEN_VEXT_VX(vmadd_vx_b, 1, 1) -GEN_VEXT_VX(vmadd_vx_h, 2, 2) -GEN_VEXT_VX(vmadd_vx_w, 4, 4) -GEN_VEXT_VX(vmadd_vx_d, 8, 8) -GEN_VEXT_VX(vnmsub_vx_b, 1, 1) -GEN_VEXT_VX(vnmsub_vx_h, 2, 2) -GEN_VEXT_VX(vnmsub_vx_w, 4, 4) -GEN_VEXT_VX(vnmsub_vx_d, 8, 8) +GEN_VEXT_VX(vmacc_vx_b) +GEN_VEXT_VX(vmacc_vx_h) +GEN_VEXT_VX(vmacc_vx_w) +GEN_VEXT_VX(vmacc_vx_d) +GEN_VEXT_VX(vnmsac_vx_b) +GEN_VEXT_VX(vnmsac_vx_h) +GEN_VEXT_VX(vnmsac_vx_w) +GEN_VEXT_VX(vnmsac_vx_d) +GEN_VEXT_VX(vmadd_vx_b) +GEN_VEXT_VX(vmadd_vx_h) +GEN_VEXT_VX(vmadd_vx_w) +GEN_VEXT_VX(vmadd_vx_d) +GEN_VEXT_VX(vnmsub_vx_b) +GEN_VEXT_VX(vnmsub_vx_h) +GEN_VEXT_VX(vnmsub_vx_w) +GEN_VEXT_VX(vnmsub_vx_d) /* Vector Widening Integer Multiply-Add Instructions */ RVVCALL(OPIVV3, vwmaccu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MACC) @@ -1768,15 +1766,15 @@ RVVCALL(OPIVV3, vwmacc_vv_w, WOP_SSS_W, H8, H4, H4, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_b, WOP_SSU_B, H2, H1, H1, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_h, WOP_SSU_H, H4, H2, H2, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_w, WOP_SSU_W, H8, H4, H4, DO_MACC) -GEN_VEXT_VV(vwmaccu_vv_b, 1, 2) -GEN_VEXT_VV(vwmaccu_vv_h, 2, 4) -GEN_VEXT_VV(vwmaccu_vv_w, 4, 8) -GEN_VEXT_VV(vwmacc_vv_b, 1, 2) -GEN_VEXT_VV(vwmacc_vv_h, 2, 4) -GEN_VEXT_VV(vwmacc_vv_w, 4, 8) -GEN_VEXT_VV(vwmaccsu_vv_b, 1, 2) -GEN_VEXT_VV(vwmaccsu_vv_h, 2, 4) -GEN_VEXT_VV(vwmaccsu_vv_w, 4, 8) +GEN_VEXT_VV(vwmaccu_vv_b) +GEN_VEXT_VV(vwmaccu_vv_h) +GEN_VEXT_VV(vwmaccu_vv_w) +GEN_VEXT_VV(vwmacc_vv_b) +GEN_VEXT_VV(vwmacc_vv_h) +GEN_VEXT_VV(vwmacc_vv_w) +GEN_VEXT_VV(vwmaccsu_vv_b) +GEN_VEXT_VV(vwmaccsu_vv_h) +GEN_VEXT_VV(vwmaccsu_vv_w) RVVCALL(OPIVX3, vwmaccu_vx_b, WOP_UUU_B, H2, H1, DO_MACC) RVVCALL(OPIVX3, vwmaccu_vx_h, WOP_UUU_H, H4, H2, DO_MACC) @@ -1790,18 +1788,18 @@ RVVCALL(OPIVX3, vwmaccsu_vx_w, WOP_SSU_W, H8, H4, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_b, WOP_SUS_B, H2, H1, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_h, WOP_SUS_H, H4, H2, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_w, WOP_SUS_W, H8, H4, DO_MACC) -GEN_VEXT_VX(vwmaccu_vx_b, 1, 2) -GEN_VEXT_VX(vwmaccu_vx_h, 2, 4) -GEN_VEXT_VX(vwmaccu_vx_w, 4, 8) -GEN_VEXT_VX(vwmacc_vx_b, 1, 2) -GEN_VEXT_VX(vwmacc_vx_h, 2, 4) -GEN_VEXT_VX(vwmacc_vx_w, 4, 8) -GEN_VEXT_VX(vwmaccsu_vx_b, 1, 2) -GEN_VEXT_VX(vwmaccsu_vx_h, 2, 4) -GEN_VEXT_VX(vwmaccsu_vx_w, 4, 8) -GEN_VEXT_VX(vwmaccus_vx_b, 1, 2) -GEN_VEXT_VX(vwmaccus_vx_h, 2, 4) -GEN_VEXT_VX(vwmaccus_vx_w, 4, 8) +GEN_VEXT_VX(vwmaccu_vx_b) +GEN_VEXT_VX(vwmaccu_vx_h) +GEN_VEXT_VX(vwmaccu_vx_w) +GEN_VEXT_VX(vwmacc_vx_b) +GEN_VEXT_VX(vwmacc_vx_h) +GEN_VEXT_VX(vwmacc_vx_w) +GEN_VEXT_VX(vwmaccsu_vx_b) +GEN_VEXT_VX(vwmaccsu_vx_h) +GEN_VEXT_VX(vwmaccsu_vx_w) +GEN_VEXT_VX(vwmaccus_vx_b) +GEN_VEXT_VX(vwmaccus_vx_h) +GEN_VEXT_VX(vwmaccus_vx_w) /* Vector Integer Merge and Move Instructions */ #define GEN_VEXT_VMV_VV(NAME, ETYPE, H) \ @@ -1922,7 +1920,7 @@ vext_vv_rm_1(void *vd, void *v0, void *vs1, void *vs2, static inline void vext_vv_rm_2(void *vd, void *v0, void *vs1, void *vs2, CPURISCVState *env, - uint32_t desc, uint32_t esz, uint32_t dsz, + uint32_t desc, opivv2_rm_fn *fn) { uint32_t vm = vext_vm(desc); @@ -1949,11 +1947,11 @@ vext_vv_rm_2(void *vd, void *v0, void *vs1, void *vs2, } /* generate helpers for fixed point instructions with OPIVV format */ -#define GEN_VEXT_VV_RM(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VV_RM(NAME) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ - vext_vv_rm_2(vd, v0, vs1, vs2, env, desc, ESZ, DSZ, \ + vext_vv_rm_2(vd, v0, vs1, vs2, env, desc, \ do_##NAME); \ } @@ -2004,10 +2002,10 @@ RVVCALL(OPIVV2_RM, vsaddu_vv_b, OP_UUU_B, H1, H1, H1, saddu8) RVVCALL(OPIVV2_RM, vsaddu_vv_h, OP_UUU_H, H2, H2, H2, saddu16) RVVCALL(OPIVV2_RM, vsaddu_vv_w, OP_UUU_W, H4, H4, H4, saddu32) RVVCALL(OPIVV2_RM, vsaddu_vv_d, OP_UUU_D, H8, H8, H8, saddu64) -GEN_VEXT_VV_RM(vsaddu_vv_b, 1, 1) -GEN_VEXT_VV_RM(vsaddu_vv_h, 2, 2) -GEN_VEXT_VV_RM(vsaddu_vv_w, 4, 4) -GEN_VEXT_VV_RM(vsaddu_vv_d, 8, 8) +GEN_VEXT_VV_RM(vsaddu_vv_b) +GEN_VEXT_VV_RM(vsaddu_vv_h) +GEN_VEXT_VV_RM(vsaddu_vv_w) +GEN_VEXT_VV_RM(vsaddu_vv_d) typedef void opivx2_rm_fn(void *vd, target_long s1, void *vs2, int i, CPURISCVState *env, int vxrm); @@ -2039,7 +2037,7 @@ vext_vx_rm_1(void *vd, void *v0, target_long s1, void *vs2, static inline void vext_vx_rm_2(void *vd, void *v0, target_long s1, void *vs2, CPURISCVState *env, - uint32_t desc, uint32_t esz, uint32_t dsz, + uint32_t desc, opivx2_rm_fn *fn) { uint32_t vm = vext_vm(desc); @@ -2066,11 +2064,11 @@ vext_vx_rm_2(void *vd, void *v0, target_long s1, void *vs2, } /* generate helpers for fixed point instructions with OPIVX format */ -#define GEN_VEXT_VX_RM(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VX_RM(NAME) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ void *vs2, CPURISCVState *env, uint32_t desc) \ { \ - vext_vx_rm_2(vd, v0, s1, vs2, env, desc, ESZ, DSZ, \ + vext_vx_rm_2(vd, v0, s1, vs2, env, desc, \ do_##NAME); \ } @@ -2078,10 +2076,10 @@ RVVCALL(OPIVX2_RM, vsaddu_vx_b, OP_UUU_B, H1, H1, saddu8) RVVCALL(OPIVX2_RM, vsaddu_vx_h, OP_UUU_H, H2, H2, saddu16) RVVCALL(OPIVX2_RM, vsaddu_vx_w, OP_UUU_W, H4, H4, saddu32) RVVCALL(OPIVX2_RM, vsaddu_vx_d, OP_UUU_D, H8, H8, saddu64) -GEN_VEXT_VX_RM(vsaddu_vx_b, 1, 1) -GEN_VEXT_VX_RM(vsaddu_vx_h, 2, 2) -GEN_VEXT_VX_RM(vsaddu_vx_w, 4, 4) -GEN_VEXT_VX_RM(vsaddu_vx_d, 8, 8) +GEN_VEXT_VX_RM(vsaddu_vx_b) +GEN_VEXT_VX_RM(vsaddu_vx_h) +GEN_VEXT_VX_RM(vsaddu_vx_w) +GEN_VEXT_VX_RM(vsaddu_vx_d) static inline int8_t sadd8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) { @@ -2127,19 +2125,19 @@ RVVCALL(OPIVV2_RM, vsadd_vv_b, OP_SSS_B, H1, H1, H1, sadd8) RVVCALL(OPIVV2_RM, vsadd_vv_h, OP_SSS_H, H2, H2, H2, sadd16) RVVCALL(OPIVV2_RM, vsadd_vv_w, OP_SSS_W, H4, H4, H4, sadd32) RVVCALL(OPIVV2_RM, vsadd_vv_d, OP_SSS_D, H8, H8, H8, sadd64) -GEN_VEXT_VV_RM(vsadd_vv_b, 1, 1) -GEN_VEXT_VV_RM(vsadd_vv_h, 2, 2) -GEN_VEXT_VV_RM(vsadd_vv_w, 4, 4) -GEN_VEXT_VV_RM(vsadd_vv_d, 8, 8) +GEN_VEXT_VV_RM(vsadd_vv_b) +GEN_VEXT_VV_RM(vsadd_vv_h) +GEN_VEXT_VV_RM(vsadd_vv_w) +GEN_VEXT_VV_RM(vsadd_vv_d) RVVCALL(OPIVX2_RM, vsadd_vx_b, OP_SSS_B, H1, H1, sadd8) RVVCALL(OPIVX2_RM, vsadd_vx_h, OP_SSS_H, H2, H2, sadd16) RVVCALL(OPIVX2_RM, vsadd_vx_w, OP_SSS_W, H4, H4, sadd32) RVVCALL(OPIVX2_RM, vsadd_vx_d, OP_SSS_D, H8, H8, sadd64) -GEN_VEXT_VX_RM(vsadd_vx_b, 1, 1) -GEN_VEXT_VX_RM(vsadd_vx_h, 2, 2) -GEN_VEXT_VX_RM(vsadd_vx_w, 4, 4) -GEN_VEXT_VX_RM(vsadd_vx_d, 8, 8) +GEN_VEXT_VX_RM(vsadd_vx_b) +GEN_VEXT_VX_RM(vsadd_vx_h) +GEN_VEXT_VX_RM(vsadd_vx_w) +GEN_VEXT_VX_RM(vsadd_vx_d) static inline uint8_t ssubu8(CPURISCVState *env, int vxrm, uint8_t a, uint8_t b) { @@ -2188,19 +2186,19 @@ RVVCALL(OPIVV2_RM, vssubu_vv_b, OP_UUU_B, H1, H1, H1, ssubu8) RVVCALL(OPIVV2_RM, vssubu_vv_h, OP_UUU_H, H2, H2, H2, ssubu16) RVVCALL(OPIVV2_RM, vssubu_vv_w, OP_UUU_W, H4, H4, H4, ssubu32) RVVCALL(OPIVV2_RM, vssubu_vv_d, OP_UUU_D, H8, H8, H8, ssubu64) -GEN_VEXT_VV_RM(vssubu_vv_b, 1, 1) -GEN_VEXT_VV_RM(vssubu_vv_h, 2, 2) -GEN_VEXT_VV_RM(vssubu_vv_w, 4, 4) -GEN_VEXT_VV_RM(vssubu_vv_d, 8, 8) +GEN_VEXT_VV_RM(vssubu_vv_b) +GEN_VEXT_VV_RM(vssubu_vv_h) +GEN_VEXT_VV_RM(vssubu_vv_w) +GEN_VEXT_VV_RM(vssubu_vv_d) RVVCALL(OPIVX2_RM, vssubu_vx_b, OP_UUU_B, H1, H1, ssubu8) RVVCALL(OPIVX2_RM, vssubu_vx_h, OP_UUU_H, H2, H2, ssubu16) RVVCALL(OPIVX2_RM, vssubu_vx_w, OP_UUU_W, H4, H4, ssubu32) RVVCALL(OPIVX2_RM, vssubu_vx_d, OP_UUU_D, H8, H8, ssubu64) -GEN_VEXT_VX_RM(vssubu_vx_b, 1, 1) -GEN_VEXT_VX_RM(vssubu_vx_h, 2, 2) -GEN_VEXT_VX_RM(vssubu_vx_w, 4, 4) -GEN_VEXT_VX_RM(vssubu_vx_d, 8, 8) +GEN_VEXT_VX_RM(vssubu_vx_b) +GEN_VEXT_VX_RM(vssubu_vx_h) +GEN_VEXT_VX_RM(vssubu_vx_w) +GEN_VEXT_VX_RM(vssubu_vx_d) static inline int8_t ssub8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) { @@ -2246,19 +2244,19 @@ RVVCALL(OPIVV2_RM, vssub_vv_b, OP_SSS_B, H1, H1, H1, ssub8) RVVCALL(OPIVV2_RM, vssub_vv_h, OP_SSS_H, H2, H2, H2, ssub16) RVVCALL(OPIVV2_RM, vssub_vv_w, OP_SSS_W, H4, H4, H4, ssub32) RVVCALL(OPIVV2_RM, vssub_vv_d, OP_SSS_D, H8, H8, H8, ssub64) -GEN_VEXT_VV_RM(vssub_vv_b, 1, 1) -GEN_VEXT_VV_RM(vssub_vv_h, 2, 2) -GEN_VEXT_VV_RM(vssub_vv_w, 4, 4) -GEN_VEXT_VV_RM(vssub_vv_d, 8, 8) +GEN_VEXT_VV_RM(vssub_vv_b) +GEN_VEXT_VV_RM(vssub_vv_h) +GEN_VEXT_VV_RM(vssub_vv_w) +GEN_VEXT_VV_RM(vssub_vv_d) RVVCALL(OPIVX2_RM, vssub_vx_b, OP_SSS_B, H1, H1, ssub8) RVVCALL(OPIVX2_RM, vssub_vx_h, OP_SSS_H, H2, H2, ssub16) RVVCALL(OPIVX2_RM, vssub_vx_w, OP_SSS_W, H4, H4, ssub32) RVVCALL(OPIVX2_RM, vssub_vx_d, OP_SSS_D, H8, H8, ssub64) -GEN_VEXT_VX_RM(vssub_vx_b, 1, 1) -GEN_VEXT_VX_RM(vssub_vx_h, 2, 2) -GEN_VEXT_VX_RM(vssub_vx_w, 4, 4) -GEN_VEXT_VX_RM(vssub_vx_d, 8, 8) +GEN_VEXT_VX_RM(vssub_vx_b) +GEN_VEXT_VX_RM(vssub_vx_h) +GEN_VEXT_VX_RM(vssub_vx_w) +GEN_VEXT_VX_RM(vssub_vx_d) /* Vector Single-Width Averaging Add and Subtract */ static inline uint8_t get_round(int vxrm, uint64_t v, uint8_t shift) @@ -2310,19 +2308,19 @@ RVVCALL(OPIVV2_RM, vaadd_vv_b, OP_SSS_B, H1, H1, H1, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_h, OP_SSS_H, H2, H2, H2, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_w, OP_SSS_W, H4, H4, H4, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_d, OP_SSS_D, H8, H8, H8, aadd64) -GEN_VEXT_VV_RM(vaadd_vv_b, 1, 1) -GEN_VEXT_VV_RM(vaadd_vv_h, 2, 2) -GEN_VEXT_VV_RM(vaadd_vv_w, 4, 4) -GEN_VEXT_VV_RM(vaadd_vv_d, 8, 8) +GEN_VEXT_VV_RM(vaadd_vv_b) +GEN_VEXT_VV_RM(vaadd_vv_h) +GEN_VEXT_VV_RM(vaadd_vv_w) +GEN_VEXT_VV_RM(vaadd_vv_d) RVVCALL(OPIVX2_RM, vaadd_vx_b, OP_SSS_B, H1, H1, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_h, OP_SSS_H, H2, H2, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_w, OP_SSS_W, H4, H4, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_d, OP_SSS_D, H8, H8, aadd64) -GEN_VEXT_VX_RM(vaadd_vx_b, 1, 1) -GEN_VEXT_VX_RM(vaadd_vx_h, 2, 2) -GEN_VEXT_VX_RM(vaadd_vx_w, 4, 4) -GEN_VEXT_VX_RM(vaadd_vx_d, 8, 8) +GEN_VEXT_VX_RM(vaadd_vx_b) +GEN_VEXT_VX_RM(vaadd_vx_h) +GEN_VEXT_VX_RM(vaadd_vx_w) +GEN_VEXT_VX_RM(vaadd_vx_d) static inline uint32_t aaddu32(CPURISCVState *env, int vxrm, uint32_t a, uint32_t b) @@ -2347,19 +2345,19 @@ RVVCALL(OPIVV2_RM, vaaddu_vv_b, OP_UUU_B, H1, H1, H1, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_h, OP_UUU_H, H2, H2, H2, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_w, OP_UUU_W, H4, H4, H4, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_d, OP_UUU_D, H8, H8, H8, aaddu64) -GEN_VEXT_VV_RM(vaaddu_vv_b, 1, 1) -GEN_VEXT_VV_RM(vaaddu_vv_h, 2, 2) -GEN_VEXT_VV_RM(vaaddu_vv_w, 4, 4) -GEN_VEXT_VV_RM(vaaddu_vv_d, 8, 8) +GEN_VEXT_VV_RM(vaaddu_vv_b) +GEN_VEXT_VV_RM(vaaddu_vv_h) +GEN_VEXT_VV_RM(vaaddu_vv_w) +GEN_VEXT_VV_RM(vaaddu_vv_d) RVVCALL(OPIVX2_RM, vaaddu_vx_b, OP_UUU_B, H1, H1, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_h, OP_UUU_H, H2, H2, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_w, OP_UUU_W, H4, H4, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_d, OP_UUU_D, H8, H8, aaddu64) -GEN_VEXT_VX_RM(vaaddu_vx_b, 1, 1) -GEN_VEXT_VX_RM(vaaddu_vx_h, 2, 2) -GEN_VEXT_VX_RM(vaaddu_vx_w, 4, 4) -GEN_VEXT_VX_RM(vaaddu_vx_d, 8, 8) +GEN_VEXT_VX_RM(vaaddu_vx_b) +GEN_VEXT_VX_RM(vaaddu_vx_h) +GEN_VEXT_VX_RM(vaaddu_vx_w) +GEN_VEXT_VX_RM(vaaddu_vx_d) static inline int32_t asub32(CPURISCVState *env, int vxrm, int32_t a, int32_t b) { @@ -2383,19 +2381,19 @@ RVVCALL(OPIVV2_RM, vasub_vv_b, OP_SSS_B, H1, H1, H1, asub32) RVVCALL(OPIVV2_RM, vasub_vv_h, OP_SSS_H, H2, H2, H2, asub32) RVVCALL(OPIVV2_RM, vasub_vv_w, OP_SSS_W, H4, H4, H4, asub32) RVVCALL(OPIVV2_RM, vasub_vv_d, OP_SSS_D, H8, H8, H8, asub64) -GEN_VEXT_VV_RM(vasub_vv_b, 1, 1) -GEN_VEXT_VV_RM(vasub_vv_h, 2, 2) -GEN_VEXT_VV_RM(vasub_vv_w, 4, 4) -GEN_VEXT_VV_RM(vasub_vv_d, 8, 8) +GEN_VEXT_VV_RM(vasub_vv_b) +GEN_VEXT_VV_RM(vasub_vv_h) +GEN_VEXT_VV_RM(vasub_vv_w) +GEN_VEXT_VV_RM(vasub_vv_d) RVVCALL(OPIVX2_RM, vasub_vx_b, OP_SSS_B, H1, H1, asub32) RVVCALL(OPIVX2_RM, vasub_vx_h, OP_SSS_H, H2, H2, asub32) RVVCALL(OPIVX2_RM, vasub_vx_w, OP_SSS_W, H4, H4, asub32) RVVCALL(OPIVX2_RM, vasub_vx_d, OP_SSS_D, H8, H8, asub64) -GEN_VEXT_VX_RM(vasub_vx_b, 1, 1) -GEN_VEXT_VX_RM(vasub_vx_h, 2, 2) -GEN_VEXT_VX_RM(vasub_vx_w, 4, 4) -GEN_VEXT_VX_RM(vasub_vx_d, 8, 8) +GEN_VEXT_VX_RM(vasub_vx_b) +GEN_VEXT_VX_RM(vasub_vx_h) +GEN_VEXT_VX_RM(vasub_vx_w) +GEN_VEXT_VX_RM(vasub_vx_d) static inline uint32_t asubu32(CPURISCVState *env, int vxrm, uint32_t a, uint32_t b) @@ -2420,19 +2418,19 @@ RVVCALL(OPIVV2_RM, vasubu_vv_b, OP_UUU_B, H1, H1, H1, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_h, OP_UUU_H, H2, H2, H2, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_w, OP_UUU_W, H4, H4, H4, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_d, OP_UUU_D, H8, H8, H8, asubu64) -GEN_VEXT_VV_RM(vasubu_vv_b, 1, 1) -GEN_VEXT_VV_RM(vasubu_vv_h, 2, 2) -GEN_VEXT_VV_RM(vasubu_vv_w, 4, 4) -GEN_VEXT_VV_RM(vasubu_vv_d, 8, 8) +GEN_VEXT_VV_RM(vasubu_vv_b) +GEN_VEXT_VV_RM(vasubu_vv_h) +GEN_VEXT_VV_RM(vasubu_vv_w) +GEN_VEXT_VV_RM(vasubu_vv_d) RVVCALL(OPIVX2_RM, vasubu_vx_b, OP_UUU_B, H1, H1, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_h, OP_UUU_H, H2, H2, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_w, OP_UUU_W, H4, H4, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_d, OP_UUU_D, H8, H8, asubu64) -GEN_VEXT_VX_RM(vasubu_vx_b, 1, 1) -GEN_VEXT_VX_RM(vasubu_vx_h, 2, 2) -GEN_VEXT_VX_RM(vasubu_vx_w, 4, 4) -GEN_VEXT_VX_RM(vasubu_vx_d, 8, 8) +GEN_VEXT_VX_RM(vasubu_vx_b) +GEN_VEXT_VX_RM(vasubu_vx_h) +GEN_VEXT_VX_RM(vasubu_vx_w) +GEN_VEXT_VX_RM(vasubu_vx_d) /* Vector Single-Width Fractional Multiply with Rounding and Saturation */ static inline int8_t vsmul8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) @@ -2527,19 +2525,19 @@ RVVCALL(OPIVV2_RM, vsmul_vv_b, OP_SSS_B, H1, H1, H1, vsmul8) RVVCALL(OPIVV2_RM, vsmul_vv_h, OP_SSS_H, H2, H2, H2, vsmul16) RVVCALL(OPIVV2_RM, vsmul_vv_w, OP_SSS_W, H4, H4, H4, vsmul32) RVVCALL(OPIVV2_RM, vsmul_vv_d, OP_SSS_D, H8, H8, H8, vsmul64) -GEN_VEXT_VV_RM(vsmul_vv_b, 1, 1) -GEN_VEXT_VV_RM(vsmul_vv_h, 2, 2) -GEN_VEXT_VV_RM(vsmul_vv_w, 4, 4) -GEN_VEXT_VV_RM(vsmul_vv_d, 8, 8) +GEN_VEXT_VV_RM(vsmul_vv_b) +GEN_VEXT_VV_RM(vsmul_vv_h) +GEN_VEXT_VV_RM(vsmul_vv_w) +GEN_VEXT_VV_RM(vsmul_vv_d) RVVCALL(OPIVX2_RM, vsmul_vx_b, OP_SSS_B, H1, H1, vsmul8) RVVCALL(OPIVX2_RM, vsmul_vx_h, OP_SSS_H, H2, H2, vsmul16) RVVCALL(OPIVX2_RM, vsmul_vx_w, OP_SSS_W, H4, H4, vsmul32) RVVCALL(OPIVX2_RM, vsmul_vx_d, OP_SSS_D, H8, H8, vsmul64) -GEN_VEXT_VX_RM(vsmul_vx_b, 1, 1) -GEN_VEXT_VX_RM(vsmul_vx_h, 2, 2) -GEN_VEXT_VX_RM(vsmul_vx_w, 4, 4) -GEN_VEXT_VX_RM(vsmul_vx_d, 8, 8) +GEN_VEXT_VX_RM(vsmul_vx_b) +GEN_VEXT_VX_RM(vsmul_vx_h) +GEN_VEXT_VX_RM(vsmul_vx_w) +GEN_VEXT_VX_RM(vsmul_vx_d) /* Vector Single-Width Scaling Shift Instructions */ static inline uint8_t @@ -2586,19 +2584,19 @@ RVVCALL(OPIVV2_RM, vssrl_vv_b, OP_UUU_B, H1, H1, H1, vssrl8) RVVCALL(OPIVV2_RM, vssrl_vv_h, OP_UUU_H, H2, H2, H2, vssrl16) RVVCALL(OPIVV2_RM, vssrl_vv_w, OP_UUU_W, H4, H4, H4, vssrl32) RVVCALL(OPIVV2_RM, vssrl_vv_d, OP_UUU_D, H8, H8, H8, vssrl64) -GEN_VEXT_VV_RM(vssrl_vv_b, 1, 1) -GEN_VEXT_VV_RM(vssrl_vv_h, 2, 2) -GEN_VEXT_VV_RM(vssrl_vv_w, 4, 4) -GEN_VEXT_VV_RM(vssrl_vv_d, 8, 8) +GEN_VEXT_VV_RM(vssrl_vv_b) +GEN_VEXT_VV_RM(vssrl_vv_h) +GEN_VEXT_VV_RM(vssrl_vv_w) +GEN_VEXT_VV_RM(vssrl_vv_d) RVVCALL(OPIVX2_RM, vssrl_vx_b, OP_UUU_B, H1, H1, vssrl8) RVVCALL(OPIVX2_RM, vssrl_vx_h, OP_UUU_H, H2, H2, vssrl16) RVVCALL(OPIVX2_RM, vssrl_vx_w, OP_UUU_W, H4, H4, vssrl32) RVVCALL(OPIVX2_RM, vssrl_vx_d, OP_UUU_D, H8, H8, vssrl64) -GEN_VEXT_VX_RM(vssrl_vx_b, 1, 1) -GEN_VEXT_VX_RM(vssrl_vx_h, 2, 2) -GEN_VEXT_VX_RM(vssrl_vx_w, 4, 4) -GEN_VEXT_VX_RM(vssrl_vx_d, 8, 8) +GEN_VEXT_VX_RM(vssrl_vx_b) +GEN_VEXT_VX_RM(vssrl_vx_h) +GEN_VEXT_VX_RM(vssrl_vx_w) +GEN_VEXT_VX_RM(vssrl_vx_d) static inline int8_t vssra8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) @@ -2645,19 +2643,19 @@ RVVCALL(OPIVV2_RM, vssra_vv_b, OP_SSS_B, H1, H1, H1, vssra8) RVVCALL(OPIVV2_RM, vssra_vv_h, OP_SSS_H, H2, H2, H2, vssra16) RVVCALL(OPIVV2_RM, vssra_vv_w, OP_SSS_W, H4, H4, H4, vssra32) RVVCALL(OPIVV2_RM, vssra_vv_d, OP_SSS_D, H8, H8, H8, vssra64) -GEN_VEXT_VV_RM(vssra_vv_b, 1, 1) -GEN_VEXT_VV_RM(vssra_vv_h, 2, 2) -GEN_VEXT_VV_RM(vssra_vv_w, 4, 4) -GEN_VEXT_VV_RM(vssra_vv_d, 8, 8) +GEN_VEXT_VV_RM(vssra_vv_b) +GEN_VEXT_VV_RM(vssra_vv_h) +GEN_VEXT_VV_RM(vssra_vv_w) +GEN_VEXT_VV_RM(vssra_vv_d) RVVCALL(OPIVX2_RM, vssra_vx_b, OP_SSS_B, H1, H1, vssra8) RVVCALL(OPIVX2_RM, vssra_vx_h, OP_SSS_H, H2, H2, vssra16) RVVCALL(OPIVX2_RM, vssra_vx_w, OP_SSS_W, H4, H4, vssra32) RVVCALL(OPIVX2_RM, vssra_vx_d, OP_SSS_D, H8, H8, vssra64) -GEN_VEXT_VX_RM(vssra_vx_b, 1, 1) -GEN_VEXT_VX_RM(vssra_vx_h, 2, 2) -GEN_VEXT_VX_RM(vssra_vx_w, 4, 4) -GEN_VEXT_VX_RM(vssra_vx_d, 8, 8) +GEN_VEXT_VX_RM(vssra_vx_b) +GEN_VEXT_VX_RM(vssra_vx_h) +GEN_VEXT_VX_RM(vssra_vx_w) +GEN_VEXT_VX_RM(vssra_vx_d) /* Vector Narrowing Fixed-Point Clip Instructions */ static inline int8_t @@ -2720,16 +2718,16 @@ vnclip32(CPURISCVState *env, int vxrm, int64_t a, int32_t b) RVVCALL(OPIVV2_RM, vnclip_wv_b, NOP_SSS_B, H1, H2, H1, vnclip8) RVVCALL(OPIVV2_RM, vnclip_wv_h, NOP_SSS_H, H2, H4, H2, vnclip16) RVVCALL(OPIVV2_RM, vnclip_wv_w, NOP_SSS_W, H4, H8, H4, vnclip32) -GEN_VEXT_VV_RM(vnclip_wv_b, 1, 1) -GEN_VEXT_VV_RM(vnclip_wv_h, 2, 2) -GEN_VEXT_VV_RM(vnclip_wv_w, 4, 4) +GEN_VEXT_VV_RM(vnclip_wv_b) +GEN_VEXT_VV_RM(vnclip_wv_h) +GEN_VEXT_VV_RM(vnclip_wv_w) RVVCALL(OPIVX2_RM, vnclip_wx_b, NOP_SSS_B, H1, H2, vnclip8) RVVCALL(OPIVX2_RM, vnclip_wx_h, NOP_SSS_H, H2, H4, vnclip16) RVVCALL(OPIVX2_RM, vnclip_wx_w, NOP_SSS_W, H4, H8, vnclip32) -GEN_VEXT_VX_RM(vnclip_wx_b, 1, 1) -GEN_VEXT_VX_RM(vnclip_wx_h, 2, 2) -GEN_VEXT_VX_RM(vnclip_wx_w, 4, 4) +GEN_VEXT_VX_RM(vnclip_wx_b) +GEN_VEXT_VX_RM(vnclip_wx_h) +GEN_VEXT_VX_RM(vnclip_wx_w) static inline uint8_t vnclipu8(CPURISCVState *env, int vxrm, uint16_t a, uint8_t b) @@ -2782,16 +2780,16 @@ vnclipu32(CPURISCVState *env, int vxrm, uint64_t a, uint32_t b) RVVCALL(OPIVV2_RM, vnclipu_wv_b, NOP_UUU_B, H1, H2, H1, vnclipu8) RVVCALL(OPIVV2_RM, vnclipu_wv_h, NOP_UUU_H, H2, H4, H2, vnclipu16) RVVCALL(OPIVV2_RM, vnclipu_wv_w, NOP_UUU_W, H4, H8, H4, vnclipu32) -GEN_VEXT_VV_RM(vnclipu_wv_b, 1, 1) -GEN_VEXT_VV_RM(vnclipu_wv_h, 2, 2) -GEN_VEXT_VV_RM(vnclipu_wv_w, 4, 4) +GEN_VEXT_VV_RM(vnclipu_wv_b) +GEN_VEXT_VV_RM(vnclipu_wv_h) +GEN_VEXT_VV_RM(vnclipu_wv_w) RVVCALL(OPIVX2_RM, vnclipu_wx_b, NOP_UUU_B, H1, H2, vnclipu8) RVVCALL(OPIVX2_RM, vnclipu_wx_h, NOP_UUU_H, H2, H4, vnclipu16) RVVCALL(OPIVX2_RM, vnclipu_wx_w, NOP_UUU_W, H4, H8, vnclipu32) -GEN_VEXT_VX_RM(vnclipu_wx_b, 1, 1) -GEN_VEXT_VX_RM(vnclipu_wx_h, 2, 2) -GEN_VEXT_VX_RM(vnclipu_wx_w, 4, 4) +GEN_VEXT_VX_RM(vnclipu_wx_b) +GEN_VEXT_VX_RM(vnclipu_wx_h) +GEN_VEXT_VX_RM(vnclipu_wx_w) /* *** Vector Float Point Arithmetic Instructions @@ -2806,7 +2804,7 @@ static void do_##NAME(void *vd, void *vs1, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, s1, &env->fp_status); \ } -#define GEN_VEXT_VV_ENV(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VV_ENV(NAME) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ @@ -2827,9 +2825,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ RVVCALL(OPFVV2, vfadd_vv_h, OP_UUU_H, H2, H2, H2, float16_add) RVVCALL(OPFVV2, vfadd_vv_w, OP_UUU_W, H4, H4, H4, float32_add) RVVCALL(OPFVV2, vfadd_vv_d, OP_UUU_D, H8, H8, H8, float64_add) -GEN_VEXT_VV_ENV(vfadd_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfadd_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfadd_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfadd_vv_h) +GEN_VEXT_VV_ENV(vfadd_vv_w) +GEN_VEXT_VV_ENV(vfadd_vv_d) #define OPFVF2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ @@ -2839,7 +2837,7 @@ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1, &env->fp_status);\ } -#define GEN_VEXT_VF(NAME, ESZ, DSZ) \ +#define GEN_VEXT_VF(NAME) \ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ @@ -2860,22 +2858,22 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, \ RVVCALL(OPFVF2, vfadd_vf_h, OP_UUU_H, H2, H2, float16_add) RVVCALL(OPFVF2, vfadd_vf_w, OP_UUU_W, H4, H4, float32_add) RVVCALL(OPFVF2, vfadd_vf_d, OP_UUU_D, H8, H8, float64_add) -GEN_VEXT_VF(vfadd_vf_h, 2, 2) -GEN_VEXT_VF(vfadd_vf_w, 4, 4) -GEN_VEXT_VF(vfadd_vf_d, 8, 8) +GEN_VEXT_VF(vfadd_vf_h) +GEN_VEXT_VF(vfadd_vf_w) +GEN_VEXT_VF(vfadd_vf_d) RVVCALL(OPFVV2, vfsub_vv_h, OP_UUU_H, H2, H2, H2, float16_sub) RVVCALL(OPFVV2, vfsub_vv_w, OP_UUU_W, H4, H4, H4, float32_sub) RVVCALL(OPFVV2, vfsub_vv_d, OP_UUU_D, H8, H8, H8, float64_sub) -GEN_VEXT_VV_ENV(vfsub_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfsub_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfsub_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfsub_vv_h) +GEN_VEXT_VV_ENV(vfsub_vv_w) +GEN_VEXT_VV_ENV(vfsub_vv_d) RVVCALL(OPFVF2, vfsub_vf_h, OP_UUU_H, H2, H2, float16_sub) RVVCALL(OPFVF2, vfsub_vf_w, OP_UUU_W, H4, H4, float32_sub) RVVCALL(OPFVF2, vfsub_vf_d, OP_UUU_D, H8, H8, float64_sub) -GEN_VEXT_VF(vfsub_vf_h, 2, 2) -GEN_VEXT_VF(vfsub_vf_w, 4, 4) -GEN_VEXT_VF(vfsub_vf_d, 8, 8) +GEN_VEXT_VF(vfsub_vf_h) +GEN_VEXT_VF(vfsub_vf_w) +GEN_VEXT_VF(vfsub_vf_d) static uint16_t float16_rsub(uint16_t a, uint16_t b, float_status *s) { @@ -2895,9 +2893,9 @@ static uint64_t float64_rsub(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVF2, vfrsub_vf_h, OP_UUU_H, H2, H2, float16_rsub) RVVCALL(OPFVF2, vfrsub_vf_w, OP_UUU_W, H4, H4, float32_rsub) RVVCALL(OPFVF2, vfrsub_vf_d, OP_UUU_D, H8, H8, float64_rsub) -GEN_VEXT_VF(vfrsub_vf_h, 2, 2) -GEN_VEXT_VF(vfrsub_vf_w, 4, 4) -GEN_VEXT_VF(vfrsub_vf_d, 8, 8) +GEN_VEXT_VF(vfrsub_vf_h) +GEN_VEXT_VF(vfrsub_vf_w) +GEN_VEXT_VF(vfrsub_vf_d) /* Vector Widening Floating-Point Add/Subtract Instructions */ static uint32_t vfwadd16(uint16_t a, uint16_t b, float_status *s) @@ -2915,12 +2913,12 @@ static uint64_t vfwadd32(uint32_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwadd_vv_h, WOP_UUU_H, H4, H2, H2, vfwadd16) RVVCALL(OPFVV2, vfwadd_vv_w, WOP_UUU_W, H8, H4, H4, vfwadd32) -GEN_VEXT_VV_ENV(vfwadd_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwadd_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwadd_vv_h) +GEN_VEXT_VV_ENV(vfwadd_vv_w) RVVCALL(OPFVF2, vfwadd_vf_h, WOP_UUU_H, H4, H2, vfwadd16) RVVCALL(OPFVF2, vfwadd_vf_w, WOP_UUU_W, H8, H4, vfwadd32) -GEN_VEXT_VF(vfwadd_vf_h, 2, 4) -GEN_VEXT_VF(vfwadd_vf_w, 4, 8) +GEN_VEXT_VF(vfwadd_vf_h) +GEN_VEXT_VF(vfwadd_vf_w) static uint32_t vfwsub16(uint16_t a, uint16_t b, float_status *s) { @@ -2937,12 +2935,12 @@ static uint64_t vfwsub32(uint32_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwsub_vv_h, WOP_UUU_H, H4, H2, H2, vfwsub16) RVVCALL(OPFVV2, vfwsub_vv_w, WOP_UUU_W, H8, H4, H4, vfwsub32) -GEN_VEXT_VV_ENV(vfwsub_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwsub_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwsub_vv_h) +GEN_VEXT_VV_ENV(vfwsub_vv_w) RVVCALL(OPFVF2, vfwsub_vf_h, WOP_UUU_H, H4, H2, vfwsub16) RVVCALL(OPFVF2, vfwsub_vf_w, WOP_UUU_W, H8, H4, vfwsub32) -GEN_VEXT_VF(vfwsub_vf_h, 2, 4) -GEN_VEXT_VF(vfwsub_vf_w, 4, 8) +GEN_VEXT_VF(vfwsub_vf_h) +GEN_VEXT_VF(vfwsub_vf_w) static uint32_t vfwaddw16(uint32_t a, uint16_t b, float_status *s) { @@ -2956,12 +2954,12 @@ static uint64_t vfwaddw32(uint64_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwadd_wv_h, WOP_WUUU_H, H4, H2, H2, vfwaddw16) RVVCALL(OPFVV2, vfwadd_wv_w, WOP_WUUU_W, H8, H4, H4, vfwaddw32) -GEN_VEXT_VV_ENV(vfwadd_wv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwadd_wv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwadd_wv_h) +GEN_VEXT_VV_ENV(vfwadd_wv_w) RVVCALL(OPFVF2, vfwadd_wf_h, WOP_WUUU_H, H4, H2, vfwaddw16) RVVCALL(OPFVF2, vfwadd_wf_w, WOP_WUUU_W, H8, H4, vfwaddw32) -GEN_VEXT_VF(vfwadd_wf_h, 2, 4) -GEN_VEXT_VF(vfwadd_wf_w, 4, 8) +GEN_VEXT_VF(vfwadd_wf_h) +GEN_VEXT_VF(vfwadd_wf_w) static uint32_t vfwsubw16(uint32_t a, uint16_t b, float_status *s) { @@ -2975,39 +2973,39 @@ static uint64_t vfwsubw32(uint64_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwsub_wv_h, WOP_WUUU_H, H4, H2, H2, vfwsubw16) RVVCALL(OPFVV2, vfwsub_wv_w, WOP_WUUU_W, H8, H4, H4, vfwsubw32) -GEN_VEXT_VV_ENV(vfwsub_wv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwsub_wv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwsub_wv_h) +GEN_VEXT_VV_ENV(vfwsub_wv_w) RVVCALL(OPFVF2, vfwsub_wf_h, WOP_WUUU_H, H4, H2, vfwsubw16) RVVCALL(OPFVF2, vfwsub_wf_w, WOP_WUUU_W, H8, H4, vfwsubw32) -GEN_VEXT_VF(vfwsub_wf_h, 2, 4) -GEN_VEXT_VF(vfwsub_wf_w, 4, 8) +GEN_VEXT_VF(vfwsub_wf_h) +GEN_VEXT_VF(vfwsub_wf_w) /* Vector Single-Width Floating-Point Multiply/Divide Instructions */ RVVCALL(OPFVV2, vfmul_vv_h, OP_UUU_H, H2, H2, H2, float16_mul) RVVCALL(OPFVV2, vfmul_vv_w, OP_UUU_W, H4, H4, H4, float32_mul) RVVCALL(OPFVV2, vfmul_vv_d, OP_UUU_D, H8, H8, H8, float64_mul) -GEN_VEXT_VV_ENV(vfmul_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmul_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmul_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmul_vv_h) +GEN_VEXT_VV_ENV(vfmul_vv_w) +GEN_VEXT_VV_ENV(vfmul_vv_d) RVVCALL(OPFVF2, vfmul_vf_h, OP_UUU_H, H2, H2, float16_mul) RVVCALL(OPFVF2, vfmul_vf_w, OP_UUU_W, H4, H4, float32_mul) RVVCALL(OPFVF2, vfmul_vf_d, OP_UUU_D, H8, H8, float64_mul) -GEN_VEXT_VF(vfmul_vf_h, 2, 2) -GEN_VEXT_VF(vfmul_vf_w, 4, 4) -GEN_VEXT_VF(vfmul_vf_d, 8, 8) +GEN_VEXT_VF(vfmul_vf_h) +GEN_VEXT_VF(vfmul_vf_w) +GEN_VEXT_VF(vfmul_vf_d) RVVCALL(OPFVV2, vfdiv_vv_h, OP_UUU_H, H2, H2, H2, float16_div) RVVCALL(OPFVV2, vfdiv_vv_w, OP_UUU_W, H4, H4, H4, float32_div) RVVCALL(OPFVV2, vfdiv_vv_d, OP_UUU_D, H8, H8, H8, float64_div) -GEN_VEXT_VV_ENV(vfdiv_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfdiv_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfdiv_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfdiv_vv_h) +GEN_VEXT_VV_ENV(vfdiv_vv_w) +GEN_VEXT_VV_ENV(vfdiv_vv_d) RVVCALL(OPFVF2, vfdiv_vf_h, OP_UUU_H, H2, H2, float16_div) RVVCALL(OPFVF2, vfdiv_vf_w, OP_UUU_W, H4, H4, float32_div) RVVCALL(OPFVF2, vfdiv_vf_d, OP_UUU_D, H8, H8, float64_div) -GEN_VEXT_VF(vfdiv_vf_h, 2, 2) -GEN_VEXT_VF(vfdiv_vf_w, 4, 4) -GEN_VEXT_VF(vfdiv_vf_d, 8, 8) +GEN_VEXT_VF(vfdiv_vf_h) +GEN_VEXT_VF(vfdiv_vf_w) +GEN_VEXT_VF(vfdiv_vf_d) static uint16_t float16_rdiv(uint16_t a, uint16_t b, float_status *s) { @@ -3027,9 +3025,9 @@ static uint64_t float64_rdiv(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVF2, vfrdiv_vf_h, OP_UUU_H, H2, H2, float16_rdiv) RVVCALL(OPFVF2, vfrdiv_vf_w, OP_UUU_W, H4, H4, float32_rdiv) RVVCALL(OPFVF2, vfrdiv_vf_d, OP_UUU_D, H8, H8, float64_rdiv) -GEN_VEXT_VF(vfrdiv_vf_h, 2, 2) -GEN_VEXT_VF(vfrdiv_vf_w, 4, 4) -GEN_VEXT_VF(vfrdiv_vf_d, 8, 8) +GEN_VEXT_VF(vfrdiv_vf_h) +GEN_VEXT_VF(vfrdiv_vf_w) +GEN_VEXT_VF(vfrdiv_vf_d) /* Vector Widening Floating-Point Multiply */ static uint32_t vfwmul16(uint16_t a, uint16_t b, float_status *s) @@ -3046,12 +3044,12 @@ static uint64_t vfwmul32(uint32_t a, uint32_t b, float_status *s) } RVVCALL(OPFVV2, vfwmul_vv_h, WOP_UUU_H, H4, H2, H2, vfwmul16) RVVCALL(OPFVV2, vfwmul_vv_w, WOP_UUU_W, H8, H4, H4, vfwmul32) -GEN_VEXT_VV_ENV(vfwmul_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwmul_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwmul_vv_h) +GEN_VEXT_VV_ENV(vfwmul_vv_w) RVVCALL(OPFVF2, vfwmul_vf_h, WOP_UUU_H, H4, H2, vfwmul16) RVVCALL(OPFVF2, vfwmul_vf_w, WOP_UUU_W, H8, H4, vfwmul32) -GEN_VEXT_VF(vfwmul_vf_h, 2, 4) -GEN_VEXT_VF(vfwmul_vf_w, 4, 8) +GEN_VEXT_VF(vfwmul_vf_h) +GEN_VEXT_VF(vfwmul_vf_w) /* Vector Single-Width Floating-Point Fused Multiply-Add Instructions */ #define OPFVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \ @@ -3082,9 +3080,9 @@ static uint64_t fmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmacc_vv_h, OP_UUU_H, H2, H2, H2, fmacc16) RVVCALL(OPFVV3, vfmacc_vv_w, OP_UUU_W, H4, H4, H4, fmacc32) RVVCALL(OPFVV3, vfmacc_vv_d, OP_UUU_D, H8, H8, H8, fmacc64) -GEN_VEXT_VV_ENV(vfmacc_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmacc_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmacc_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmacc_vv_h) +GEN_VEXT_VV_ENV(vfmacc_vv_w) +GEN_VEXT_VV_ENV(vfmacc_vv_d) #define OPFVF3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ @@ -3098,9 +3096,9 @@ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ RVVCALL(OPFVF3, vfmacc_vf_h, OP_UUU_H, H2, H2, fmacc16) RVVCALL(OPFVF3, vfmacc_vf_w, OP_UUU_W, H4, H4, fmacc32) RVVCALL(OPFVF3, vfmacc_vf_d, OP_UUU_D, H8, H8, fmacc64) -GEN_VEXT_VF(vfmacc_vf_h, 2, 2) -GEN_VEXT_VF(vfmacc_vf_w, 4, 4) -GEN_VEXT_VF(vfmacc_vf_d, 8, 8) +GEN_VEXT_VF(vfmacc_vf_h) +GEN_VEXT_VF(vfmacc_vf_w) +GEN_VEXT_VF(vfmacc_vf_d) static uint16_t fnmacc16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3123,15 +3121,15 @@ static uint64_t fnmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmacc_vv_h, OP_UUU_H, H2, H2, H2, fnmacc16) RVVCALL(OPFVV3, vfnmacc_vv_w, OP_UUU_W, H4, H4, H4, fnmacc32) RVVCALL(OPFVV3, vfnmacc_vv_d, OP_UUU_D, H8, H8, H8, fnmacc64) -GEN_VEXT_VV_ENV(vfnmacc_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfnmacc_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfnmacc_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfnmacc_vv_h) +GEN_VEXT_VV_ENV(vfnmacc_vv_w) +GEN_VEXT_VV_ENV(vfnmacc_vv_d) RVVCALL(OPFVF3, vfnmacc_vf_h, OP_UUU_H, H2, H2, fnmacc16) RVVCALL(OPFVF3, vfnmacc_vf_w, OP_UUU_W, H4, H4, fnmacc32) RVVCALL(OPFVF3, vfnmacc_vf_d, OP_UUU_D, H8, H8, fnmacc64) -GEN_VEXT_VF(vfnmacc_vf_h, 2, 2) -GEN_VEXT_VF(vfnmacc_vf_w, 4, 4) -GEN_VEXT_VF(vfnmacc_vf_d, 8, 8) +GEN_VEXT_VF(vfnmacc_vf_h) +GEN_VEXT_VF(vfnmacc_vf_w) +GEN_VEXT_VF(vfnmacc_vf_d) static uint16_t fmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3151,15 +3149,15 @@ static uint64_t fmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmsac_vv_h, OP_UUU_H, H2, H2, H2, fmsac16) RVVCALL(OPFVV3, vfmsac_vv_w, OP_UUU_W, H4, H4, H4, fmsac32) RVVCALL(OPFVV3, vfmsac_vv_d, OP_UUU_D, H8, H8, H8, fmsac64) -GEN_VEXT_VV_ENV(vfmsac_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmsac_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmsac_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmsac_vv_h) +GEN_VEXT_VV_ENV(vfmsac_vv_w) +GEN_VEXT_VV_ENV(vfmsac_vv_d) RVVCALL(OPFVF3, vfmsac_vf_h, OP_UUU_H, H2, H2, fmsac16) RVVCALL(OPFVF3, vfmsac_vf_w, OP_UUU_W, H4, H4, fmsac32) RVVCALL(OPFVF3, vfmsac_vf_d, OP_UUU_D, H8, H8, fmsac64) -GEN_VEXT_VF(vfmsac_vf_h, 2, 2) -GEN_VEXT_VF(vfmsac_vf_w, 4, 4) -GEN_VEXT_VF(vfmsac_vf_d, 8, 8) +GEN_VEXT_VF(vfmsac_vf_h) +GEN_VEXT_VF(vfmsac_vf_w) +GEN_VEXT_VF(vfmsac_vf_d) static uint16_t fnmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3179,15 +3177,15 @@ static uint64_t fnmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmsac_vv_h, OP_UUU_H, H2, H2, H2, fnmsac16) RVVCALL(OPFVV3, vfnmsac_vv_w, OP_UUU_W, H4, H4, H4, fnmsac32) RVVCALL(OPFVV3, vfnmsac_vv_d, OP_UUU_D, H8, H8, H8, fnmsac64) -GEN_VEXT_VV_ENV(vfnmsac_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfnmsac_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfnmsac_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfnmsac_vv_h) +GEN_VEXT_VV_ENV(vfnmsac_vv_w) +GEN_VEXT_VV_ENV(vfnmsac_vv_d) RVVCALL(OPFVF3, vfnmsac_vf_h, OP_UUU_H, H2, H2, fnmsac16) RVVCALL(OPFVF3, vfnmsac_vf_w, OP_UUU_W, H4, H4, fnmsac32) RVVCALL(OPFVF3, vfnmsac_vf_d, OP_UUU_D, H8, H8, fnmsac64) -GEN_VEXT_VF(vfnmsac_vf_h, 2, 2) -GEN_VEXT_VF(vfnmsac_vf_w, 4, 4) -GEN_VEXT_VF(vfnmsac_vf_d, 8, 8) +GEN_VEXT_VF(vfnmsac_vf_h) +GEN_VEXT_VF(vfnmsac_vf_w) +GEN_VEXT_VF(vfnmsac_vf_d) static uint16_t fmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3207,15 +3205,15 @@ static uint64_t fmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmadd_vv_h, OP_UUU_H, H2, H2, H2, fmadd16) RVVCALL(OPFVV3, vfmadd_vv_w, OP_UUU_W, H4, H4, H4, fmadd32) RVVCALL(OPFVV3, vfmadd_vv_d, OP_UUU_D, H8, H8, H8, fmadd64) -GEN_VEXT_VV_ENV(vfmadd_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmadd_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmadd_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmadd_vv_h) +GEN_VEXT_VV_ENV(vfmadd_vv_w) +GEN_VEXT_VV_ENV(vfmadd_vv_d) RVVCALL(OPFVF3, vfmadd_vf_h, OP_UUU_H, H2, H2, fmadd16) RVVCALL(OPFVF3, vfmadd_vf_w, OP_UUU_W, H4, H4, fmadd32) RVVCALL(OPFVF3, vfmadd_vf_d, OP_UUU_D, H8, H8, fmadd64) -GEN_VEXT_VF(vfmadd_vf_h, 2, 2) -GEN_VEXT_VF(vfmadd_vf_w, 4, 4) -GEN_VEXT_VF(vfmadd_vf_d, 8, 8) +GEN_VEXT_VF(vfmadd_vf_h) +GEN_VEXT_VF(vfmadd_vf_w) +GEN_VEXT_VF(vfmadd_vf_d) static uint16_t fnmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3238,15 +3236,15 @@ static uint64_t fnmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmadd_vv_h, OP_UUU_H, H2, H2, H2, fnmadd16) RVVCALL(OPFVV3, vfnmadd_vv_w, OP_UUU_W, H4, H4, H4, fnmadd32) RVVCALL(OPFVV3, vfnmadd_vv_d, OP_UUU_D, H8, H8, H8, fnmadd64) -GEN_VEXT_VV_ENV(vfnmadd_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfnmadd_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfnmadd_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfnmadd_vv_h) +GEN_VEXT_VV_ENV(vfnmadd_vv_w) +GEN_VEXT_VV_ENV(vfnmadd_vv_d) RVVCALL(OPFVF3, vfnmadd_vf_h, OP_UUU_H, H2, H2, fnmadd16) RVVCALL(OPFVF3, vfnmadd_vf_w, OP_UUU_W, H4, H4, fnmadd32) RVVCALL(OPFVF3, vfnmadd_vf_d, OP_UUU_D, H8, H8, fnmadd64) -GEN_VEXT_VF(vfnmadd_vf_h, 2, 2) -GEN_VEXT_VF(vfnmadd_vf_w, 4, 4) -GEN_VEXT_VF(vfnmadd_vf_d, 8, 8) +GEN_VEXT_VF(vfnmadd_vf_h) +GEN_VEXT_VF(vfnmadd_vf_w) +GEN_VEXT_VF(vfnmadd_vf_d) static uint16_t fmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3266,15 +3264,15 @@ static uint64_t fmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmsub_vv_h, OP_UUU_H, H2, H2, H2, fmsub16) RVVCALL(OPFVV3, vfmsub_vv_w, OP_UUU_W, H4, H4, H4, fmsub32) RVVCALL(OPFVV3, vfmsub_vv_d, OP_UUU_D, H8, H8, H8, fmsub64) -GEN_VEXT_VV_ENV(vfmsub_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmsub_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmsub_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmsub_vv_h) +GEN_VEXT_VV_ENV(vfmsub_vv_w) +GEN_VEXT_VV_ENV(vfmsub_vv_d) RVVCALL(OPFVF3, vfmsub_vf_h, OP_UUU_H, H2, H2, fmsub16) RVVCALL(OPFVF3, vfmsub_vf_w, OP_UUU_W, H4, H4, fmsub32) RVVCALL(OPFVF3, vfmsub_vf_d, OP_UUU_D, H8, H8, fmsub64) -GEN_VEXT_VF(vfmsub_vf_h, 2, 2) -GEN_VEXT_VF(vfmsub_vf_w, 4, 4) -GEN_VEXT_VF(vfmsub_vf_d, 8, 8) +GEN_VEXT_VF(vfmsub_vf_h) +GEN_VEXT_VF(vfmsub_vf_w) +GEN_VEXT_VF(vfmsub_vf_d) static uint16_t fnmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3294,15 +3292,15 @@ static uint64_t fnmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmsub_vv_h, OP_UUU_H, H2, H2, H2, fnmsub16) RVVCALL(OPFVV3, vfnmsub_vv_w, OP_UUU_W, H4, H4, H4, fnmsub32) RVVCALL(OPFVV3, vfnmsub_vv_d, OP_UUU_D, H8, H8, H8, fnmsub64) -GEN_VEXT_VV_ENV(vfnmsub_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfnmsub_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfnmsub_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfnmsub_vv_h) +GEN_VEXT_VV_ENV(vfnmsub_vv_w) +GEN_VEXT_VV_ENV(vfnmsub_vv_d) RVVCALL(OPFVF3, vfnmsub_vf_h, OP_UUU_H, H2, H2, fnmsub16) RVVCALL(OPFVF3, vfnmsub_vf_w, OP_UUU_W, H4, H4, fnmsub32) RVVCALL(OPFVF3, vfnmsub_vf_d, OP_UUU_D, H8, H8, fnmsub64) -GEN_VEXT_VF(vfnmsub_vf_h, 2, 2) -GEN_VEXT_VF(vfnmsub_vf_w, 4, 4) -GEN_VEXT_VF(vfnmsub_vf_d, 8, 8) +GEN_VEXT_VF(vfnmsub_vf_h) +GEN_VEXT_VF(vfnmsub_vf_w) +GEN_VEXT_VF(vfnmsub_vf_d) /* Vector Widening Floating-Point Fused Multiply-Add Instructions */ static uint32_t fwmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s) @@ -3319,12 +3317,12 @@ static uint64_t fwmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwmacc16) RVVCALL(OPFVV3, vfwmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwmacc32) -GEN_VEXT_VV_ENV(vfwmacc_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwmacc_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwmacc_vv_h) +GEN_VEXT_VV_ENV(vfwmacc_vv_w) RVVCALL(OPFVF3, vfwmacc_vf_h, WOP_UUU_H, H4, H2, fwmacc16) RVVCALL(OPFVF3, vfwmacc_vf_w, WOP_UUU_W, H8, H4, fwmacc32) -GEN_VEXT_VF(vfwmacc_vf_h, 2, 4) -GEN_VEXT_VF(vfwmacc_vf_w, 4, 8) +GEN_VEXT_VF(vfwmacc_vf_h) +GEN_VEXT_VF(vfwmacc_vf_w) static uint32_t fwnmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3342,12 +3340,12 @@ static uint64_t fwnmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwnmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwnmacc16) RVVCALL(OPFVV3, vfwnmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwnmacc32) -GEN_VEXT_VV_ENV(vfwnmacc_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwnmacc_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwnmacc_vv_h) +GEN_VEXT_VV_ENV(vfwnmacc_vv_w) RVVCALL(OPFVF3, vfwnmacc_vf_h, WOP_UUU_H, H4, H2, fwnmacc16) RVVCALL(OPFVF3, vfwnmacc_vf_w, WOP_UUU_W, H8, H4, fwnmacc32) -GEN_VEXT_VF(vfwnmacc_vf_h, 2, 4) -GEN_VEXT_VF(vfwnmacc_vf_w, 4, 8) +GEN_VEXT_VF(vfwnmacc_vf_h) +GEN_VEXT_VF(vfwnmacc_vf_w) static uint32_t fwmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3365,12 +3363,12 @@ static uint64_t fwmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwmsac16) RVVCALL(OPFVV3, vfwmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwmsac32) -GEN_VEXT_VV_ENV(vfwmsac_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwmsac_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwmsac_vv_h) +GEN_VEXT_VV_ENV(vfwmsac_vv_w) RVVCALL(OPFVF3, vfwmsac_vf_h, WOP_UUU_H, H4, H2, fwmsac16) RVVCALL(OPFVF3, vfwmsac_vf_w, WOP_UUU_W, H8, H4, fwmsac32) -GEN_VEXT_VF(vfwmsac_vf_h, 2, 4) -GEN_VEXT_VF(vfwmsac_vf_w, 4, 8) +GEN_VEXT_VF(vfwmsac_vf_h) +GEN_VEXT_VF(vfwmsac_vf_w) static uint32_t fwnmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3388,12 +3386,12 @@ static uint64_t fwnmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwnmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwnmsac16) RVVCALL(OPFVV3, vfwnmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwnmsac32) -GEN_VEXT_VV_ENV(vfwnmsac_vv_h, 2, 4) -GEN_VEXT_VV_ENV(vfwnmsac_vv_w, 4, 8) +GEN_VEXT_VV_ENV(vfwnmsac_vv_h) +GEN_VEXT_VV_ENV(vfwnmsac_vv_w) RVVCALL(OPFVF3, vfwnmsac_vf_h, WOP_UUU_H, H4, H2, fwnmsac16) RVVCALL(OPFVF3, vfwnmsac_vf_w, WOP_UUU_W, H8, H4, fwnmsac32) -GEN_VEXT_VF(vfwnmsac_vf_h, 2, 4) -GEN_VEXT_VF(vfwnmsac_vf_w, 4, 8) +GEN_VEXT_VF(vfwnmsac_vf_h) +GEN_VEXT_VF(vfwnmsac_vf_w) /* Vector Floating-Point Square-Root Instruction */ /* (TD, T2, TX2) */ @@ -3409,7 +3407,7 @@ static void do_##NAME(void *vd, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, &env->fp_status); \ } -#define GEN_VEXT_V_ENV(NAME, ESZ, DSZ) \ +#define GEN_VEXT_V_ENV(NAME) \ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ @@ -3432,9 +3430,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ RVVCALL(OPFVV1, vfsqrt_v_h, OP_UU_H, H2, H2, float16_sqrt) RVVCALL(OPFVV1, vfsqrt_v_w, OP_UU_W, H4, H4, float32_sqrt) RVVCALL(OPFVV1, vfsqrt_v_d, OP_UU_D, H8, H8, float64_sqrt) -GEN_VEXT_V_ENV(vfsqrt_v_h, 2, 2) -GEN_VEXT_V_ENV(vfsqrt_v_w, 4, 4) -GEN_VEXT_V_ENV(vfsqrt_v_d, 8, 8) +GEN_VEXT_V_ENV(vfsqrt_v_h) +GEN_VEXT_V_ENV(vfsqrt_v_w) +GEN_VEXT_V_ENV(vfsqrt_v_d) /* * Vector Floating-Point Reciprocal Square-Root Estimate Instruction @@ -3614,9 +3612,9 @@ static float64 frsqrt7_d(float64 f, float_status *s) RVVCALL(OPFVV1, vfrsqrt7_v_h, OP_UU_H, H2, H2, frsqrt7_h) RVVCALL(OPFVV1, vfrsqrt7_v_w, OP_UU_W, H4, H4, frsqrt7_s) RVVCALL(OPFVV1, vfrsqrt7_v_d, OP_UU_D, H8, H8, frsqrt7_d) -GEN_VEXT_V_ENV(vfrsqrt7_v_h, 2, 2) -GEN_VEXT_V_ENV(vfrsqrt7_v_w, 4, 4) -GEN_VEXT_V_ENV(vfrsqrt7_v_d, 8, 8) +GEN_VEXT_V_ENV(vfrsqrt7_v_h) +GEN_VEXT_V_ENV(vfrsqrt7_v_w) +GEN_VEXT_V_ENV(vfrsqrt7_v_d) /* * Vector Floating-Point Reciprocal Estimate Instruction @@ -3805,36 +3803,36 @@ static float64 frec7_d(float64 f, float_status *s) RVVCALL(OPFVV1, vfrec7_v_h, OP_UU_H, H2, H2, frec7_h) RVVCALL(OPFVV1, vfrec7_v_w, OP_UU_W, H4, H4, frec7_s) RVVCALL(OPFVV1, vfrec7_v_d, OP_UU_D, H8, H8, frec7_d) -GEN_VEXT_V_ENV(vfrec7_v_h, 2, 2) -GEN_VEXT_V_ENV(vfrec7_v_w, 4, 4) -GEN_VEXT_V_ENV(vfrec7_v_d, 8, 8) +GEN_VEXT_V_ENV(vfrec7_v_h) +GEN_VEXT_V_ENV(vfrec7_v_w) +GEN_VEXT_V_ENV(vfrec7_v_d) /* Vector Floating-Point MIN/MAX Instructions */ RVVCALL(OPFVV2, vfmin_vv_h, OP_UUU_H, H2, H2, H2, float16_minimum_number) RVVCALL(OPFVV2, vfmin_vv_w, OP_UUU_W, H4, H4, H4, float32_minimum_number) RVVCALL(OPFVV2, vfmin_vv_d, OP_UUU_D, H8, H8, H8, float64_minimum_number) -GEN_VEXT_VV_ENV(vfmin_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmin_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmin_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmin_vv_h) +GEN_VEXT_VV_ENV(vfmin_vv_w) +GEN_VEXT_VV_ENV(vfmin_vv_d) RVVCALL(OPFVF2, vfmin_vf_h, OP_UUU_H, H2, H2, float16_minimum_number) RVVCALL(OPFVF2, vfmin_vf_w, OP_UUU_W, H4, H4, float32_minimum_number) RVVCALL(OPFVF2, vfmin_vf_d, OP_UUU_D, H8, H8, float64_minimum_number) -GEN_VEXT_VF(vfmin_vf_h, 2, 2) -GEN_VEXT_VF(vfmin_vf_w, 4, 4) -GEN_VEXT_VF(vfmin_vf_d, 8, 8) +GEN_VEXT_VF(vfmin_vf_h) +GEN_VEXT_VF(vfmin_vf_w) +GEN_VEXT_VF(vfmin_vf_d) RVVCALL(OPFVV2, vfmax_vv_h, OP_UUU_H, H2, H2, H2, float16_maximum_number) RVVCALL(OPFVV2, vfmax_vv_w, OP_UUU_W, H4, H4, H4, float32_maximum_number) RVVCALL(OPFVV2, vfmax_vv_d, OP_UUU_D, H8, H8, H8, float64_maximum_number) -GEN_VEXT_VV_ENV(vfmax_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfmax_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfmax_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfmax_vv_h) +GEN_VEXT_VV_ENV(vfmax_vv_w) +GEN_VEXT_VV_ENV(vfmax_vv_d) RVVCALL(OPFVF2, vfmax_vf_h, OP_UUU_H, H2, H2, float16_maximum_number) RVVCALL(OPFVF2, vfmax_vf_w, OP_UUU_W, H4, H4, float32_maximum_number) RVVCALL(OPFVF2, vfmax_vf_d, OP_UUU_D, H8, H8, float64_maximum_number) -GEN_VEXT_VF(vfmax_vf_h, 2, 2) -GEN_VEXT_VF(vfmax_vf_w, 4, 4) -GEN_VEXT_VF(vfmax_vf_d, 8, 8) +GEN_VEXT_VF(vfmax_vf_h) +GEN_VEXT_VF(vfmax_vf_w) +GEN_VEXT_VF(vfmax_vf_d) /* Vector Floating-Point Sign-Injection Instructions */ static uint16_t fsgnj16(uint16_t a, uint16_t b, float_status *s) @@ -3855,15 +3853,15 @@ static uint64_t fsgnj64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnj_vv_h, OP_UUU_H, H2, H2, H2, fsgnj16) RVVCALL(OPFVV2, vfsgnj_vv_w, OP_UUU_W, H4, H4, H4, fsgnj32) RVVCALL(OPFVV2, vfsgnj_vv_d, OP_UUU_D, H8, H8, H8, fsgnj64) -GEN_VEXT_VV_ENV(vfsgnj_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfsgnj_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfsgnj_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfsgnj_vv_h) +GEN_VEXT_VV_ENV(vfsgnj_vv_w) +GEN_VEXT_VV_ENV(vfsgnj_vv_d) RVVCALL(OPFVF2, vfsgnj_vf_h, OP_UUU_H, H2, H2, fsgnj16) RVVCALL(OPFVF2, vfsgnj_vf_w, OP_UUU_W, H4, H4, fsgnj32) RVVCALL(OPFVF2, vfsgnj_vf_d, OP_UUU_D, H8, H8, fsgnj64) -GEN_VEXT_VF(vfsgnj_vf_h, 2, 2) -GEN_VEXT_VF(vfsgnj_vf_w, 4, 4) -GEN_VEXT_VF(vfsgnj_vf_d, 8, 8) +GEN_VEXT_VF(vfsgnj_vf_h) +GEN_VEXT_VF(vfsgnj_vf_w) +GEN_VEXT_VF(vfsgnj_vf_d) static uint16_t fsgnjn16(uint16_t a, uint16_t b, float_status *s) { @@ -3883,15 +3881,15 @@ static uint64_t fsgnjn64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnjn_vv_h, OP_UUU_H, H2, H2, H2, fsgnjn16) RVVCALL(OPFVV2, vfsgnjn_vv_w, OP_UUU_W, H4, H4, H4, fsgnjn32) RVVCALL(OPFVV2, vfsgnjn_vv_d, OP_UUU_D, H8, H8, H8, fsgnjn64) -GEN_VEXT_VV_ENV(vfsgnjn_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfsgnjn_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfsgnjn_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfsgnjn_vv_h) +GEN_VEXT_VV_ENV(vfsgnjn_vv_w) +GEN_VEXT_VV_ENV(vfsgnjn_vv_d) RVVCALL(OPFVF2, vfsgnjn_vf_h, OP_UUU_H, H2, H2, fsgnjn16) RVVCALL(OPFVF2, vfsgnjn_vf_w, OP_UUU_W, H4, H4, fsgnjn32) RVVCALL(OPFVF2, vfsgnjn_vf_d, OP_UUU_D, H8, H8, fsgnjn64) -GEN_VEXT_VF(vfsgnjn_vf_h, 2, 2) -GEN_VEXT_VF(vfsgnjn_vf_w, 4, 4) -GEN_VEXT_VF(vfsgnjn_vf_d, 8, 8) +GEN_VEXT_VF(vfsgnjn_vf_h) +GEN_VEXT_VF(vfsgnjn_vf_w) +GEN_VEXT_VF(vfsgnjn_vf_d) static uint16_t fsgnjx16(uint16_t a, uint16_t b, float_status *s) { @@ -3911,15 +3909,15 @@ static uint64_t fsgnjx64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnjx_vv_h, OP_UUU_H, H2, H2, H2, fsgnjx16) RVVCALL(OPFVV2, vfsgnjx_vv_w, OP_UUU_W, H4, H4, H4, fsgnjx32) RVVCALL(OPFVV2, vfsgnjx_vv_d, OP_UUU_D, H8, H8, H8, fsgnjx64) -GEN_VEXT_VV_ENV(vfsgnjx_vv_h, 2, 2) -GEN_VEXT_VV_ENV(vfsgnjx_vv_w, 4, 4) -GEN_VEXT_VV_ENV(vfsgnjx_vv_d, 8, 8) +GEN_VEXT_VV_ENV(vfsgnjx_vv_h) +GEN_VEXT_VV_ENV(vfsgnjx_vv_w) +GEN_VEXT_VV_ENV(vfsgnjx_vv_d) RVVCALL(OPFVF2, vfsgnjx_vf_h, OP_UUU_H, H2, H2, fsgnjx16) RVVCALL(OPFVF2, vfsgnjx_vf_w, OP_UUU_W, H4, H4, fsgnjx32) RVVCALL(OPFVF2, vfsgnjx_vf_d, OP_UUU_D, H8, H8, fsgnjx64) -GEN_VEXT_VF(vfsgnjx_vf_h, 2, 2) -GEN_VEXT_VF(vfsgnjx_vf_w, 4, 4) -GEN_VEXT_VF(vfsgnjx_vf_d, 8, 8) +GEN_VEXT_VF(vfsgnjx_vf_h) +GEN_VEXT_VF(vfsgnjx_vf_w) +GEN_VEXT_VF(vfsgnjx_vf_d) /* Vector Floating-Point Compare Instructions */ #define GEN_VEXT_CMP_VV_ENV(NAME, ETYPE, H, DO_OP) \ @@ -4063,7 +4061,7 @@ static void do_##NAME(void *vd, void *vs2, int i) \ *((TD *)vd + HD(i)) = OP(s2); \ } -#define GEN_VEXT_V(NAME, ESZ, DSZ) \ +#define GEN_VEXT_V(NAME) \ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ @@ -4140,9 +4138,9 @@ target_ulong fclass_d(uint64_t frs1) RVVCALL(OPIVV1, vfclass_v_h, OP_UU_H, H2, H2, fclass_h) RVVCALL(OPIVV1, vfclass_v_w, OP_UU_W, H4, H4, fclass_s) RVVCALL(OPIVV1, vfclass_v_d, OP_UU_D, H8, H8, fclass_d) -GEN_VEXT_V(vfclass_v_h, 2, 2) -GEN_VEXT_V(vfclass_v_w, 4, 4) -GEN_VEXT_V(vfclass_v_d, 8, 8) +GEN_VEXT_V(vfclass_v_h) +GEN_VEXT_V(vfclass_v_w) +GEN_VEXT_V(vfclass_v_d) /* Vector Floating-Point Merge Instruction */ #define GEN_VFMERGE_VF(NAME, ETYPE, H) \ @@ -4170,33 +4168,33 @@ GEN_VFMERGE_VF(vfmerge_vfm_d, int64_t, H8) RVVCALL(OPFVV1, vfcvt_xu_f_v_h, OP_UU_H, H2, H2, float16_to_uint16) RVVCALL(OPFVV1, vfcvt_xu_f_v_w, OP_UU_W, H4, H4, float32_to_uint32) RVVCALL(OPFVV1, vfcvt_xu_f_v_d, OP_UU_D, H8, H8, float64_to_uint64) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_h, 2, 2) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_w, 4, 4) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_d, 8, 8) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_h) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_w) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_d) /* vfcvt.x.f.v vd, vs2, vm # Convert float to signed integer. */ RVVCALL(OPFVV1, vfcvt_x_f_v_h, OP_UU_H, H2, H2, float16_to_int16) RVVCALL(OPFVV1, vfcvt_x_f_v_w, OP_UU_W, H4, H4, float32_to_int32) RVVCALL(OPFVV1, vfcvt_x_f_v_d, OP_UU_D, H8, H8, float64_to_int64) -GEN_VEXT_V_ENV(vfcvt_x_f_v_h, 2, 2) -GEN_VEXT_V_ENV(vfcvt_x_f_v_w, 4, 4) -GEN_VEXT_V_ENV(vfcvt_x_f_v_d, 8, 8) +GEN_VEXT_V_ENV(vfcvt_x_f_v_h) +GEN_VEXT_V_ENV(vfcvt_x_f_v_w) +GEN_VEXT_V_ENV(vfcvt_x_f_v_d) /* vfcvt.f.xu.v vd, vs2, vm # Convert unsigned integer to float. */ RVVCALL(OPFVV1, vfcvt_f_xu_v_h, OP_UU_H, H2, H2, uint16_to_float16) RVVCALL(OPFVV1, vfcvt_f_xu_v_w, OP_UU_W, H4, H4, uint32_to_float32) RVVCALL(OPFVV1, vfcvt_f_xu_v_d, OP_UU_D, H8, H8, uint64_to_float64) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_h, 2, 2) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_w, 4, 4) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_d, 8, 8) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_h) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_w) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_d) /* vfcvt.f.x.v vd, vs2, vm # Convert integer to float. */ RVVCALL(OPFVV1, vfcvt_f_x_v_h, OP_UU_H, H2, H2, int16_to_float16) RVVCALL(OPFVV1, vfcvt_f_x_v_w, OP_UU_W, H4, H4, int32_to_float32) RVVCALL(OPFVV1, vfcvt_f_x_v_d, OP_UU_D, H8, H8, int64_to_float64) -GEN_VEXT_V_ENV(vfcvt_f_x_v_h, 2, 2) -GEN_VEXT_V_ENV(vfcvt_f_x_v_w, 4, 4) -GEN_VEXT_V_ENV(vfcvt_f_x_v_d, 8, 8) +GEN_VEXT_V_ENV(vfcvt_f_x_v_h) +GEN_VEXT_V_ENV(vfcvt_f_x_v_w) +GEN_VEXT_V_ENV(vfcvt_f_x_v_d) /* Widening Floating-Point/Integer Type-Convert Instructions */ /* (TD, T2, TX2) */ @@ -4206,30 +4204,30 @@ GEN_VEXT_V_ENV(vfcvt_f_x_v_d, 8, 8) /* vfwcvt.xu.f.v vd, vs2, vm # Convert float to double-width unsigned integer.*/ RVVCALL(OPFVV1, vfwcvt_xu_f_v_h, WOP_UU_H, H4, H2, float16_to_uint32) RVVCALL(OPFVV1, vfwcvt_xu_f_v_w, WOP_UU_W, H8, H4, float32_to_uint64) -GEN_VEXT_V_ENV(vfwcvt_xu_f_v_h, 2, 4) -GEN_VEXT_V_ENV(vfwcvt_xu_f_v_w, 4, 8) +GEN_VEXT_V_ENV(vfwcvt_xu_f_v_h) +GEN_VEXT_V_ENV(vfwcvt_xu_f_v_w) /* vfwcvt.x.f.v vd, vs2, vm # Convert float to double-width signed integer. */ RVVCALL(OPFVV1, vfwcvt_x_f_v_h, WOP_UU_H, H4, H2, float16_to_int32) RVVCALL(OPFVV1, vfwcvt_x_f_v_w, WOP_UU_W, H8, H4, float32_to_int64) -GEN_VEXT_V_ENV(vfwcvt_x_f_v_h, 2, 4) -GEN_VEXT_V_ENV(vfwcvt_x_f_v_w, 4, 8) +GEN_VEXT_V_ENV(vfwcvt_x_f_v_h) +GEN_VEXT_V_ENV(vfwcvt_x_f_v_w) /* vfwcvt.f.xu.v vd, vs2, vm # Convert unsigned integer to double-width float */ RVVCALL(OPFVV1, vfwcvt_f_xu_v_b, WOP_UU_B, H2, H1, uint8_to_float16) RVVCALL(OPFVV1, vfwcvt_f_xu_v_h, WOP_UU_H, H4, H2, uint16_to_float32) RVVCALL(OPFVV1, vfwcvt_f_xu_v_w, WOP_UU_W, H8, H4, uint32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_b, 1, 2) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_h, 2, 4) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_w, 4, 8) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_b) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_h) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_w) /* vfwcvt.f.x.v vd, vs2, vm # Convert integer to double-width float. */ RVVCALL(OPFVV1, vfwcvt_f_x_v_b, WOP_UU_B, H2, H1, int8_to_float16) RVVCALL(OPFVV1, vfwcvt_f_x_v_h, WOP_UU_H, H4, H2, int16_to_float32) RVVCALL(OPFVV1, vfwcvt_f_x_v_w, WOP_UU_W, H8, H4, int32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_b, 1, 2) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_h, 2, 4) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_w, 4, 8) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_b) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_h) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_w) /* * vfwcvt.f.f.v vd, vs2, vm @@ -4242,8 +4240,8 @@ static uint32_t vfwcvtffv16(uint16_t a, float_status *s) RVVCALL(OPFVV1, vfwcvt_f_f_v_h, WOP_UU_H, H4, H2, vfwcvtffv16) RVVCALL(OPFVV1, vfwcvt_f_f_v_w, WOP_UU_W, H8, H4, float32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_f_v_h, 2, 4) -GEN_VEXT_V_ENV(vfwcvt_f_f_v_w, 4, 8) +GEN_VEXT_V_ENV(vfwcvt_f_f_v_h) +GEN_VEXT_V_ENV(vfwcvt_f_f_v_w) /* Narrowing Floating-Point/Integer Type-Convert Instructions */ /* (TD, T2, TX2) */ @@ -4254,29 +4252,29 @@ GEN_VEXT_V_ENV(vfwcvt_f_f_v_w, 4, 8) RVVCALL(OPFVV1, vfncvt_xu_f_w_b, NOP_UU_B, H1, H2, float16_to_uint8) RVVCALL(OPFVV1, vfncvt_xu_f_w_h, NOP_UU_H, H2, H4, float32_to_uint16) RVVCALL(OPFVV1, vfncvt_xu_f_w_w, NOP_UU_W, H4, H8, float64_to_uint32) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_b, 1, 1) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_h, 2, 2) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_w, 4, 4) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_b) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_h) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_w) /* vfncvt.x.f.v vd, vs2, vm # Convert double-width float to signed integer. */ RVVCALL(OPFVV1, vfncvt_x_f_w_b, NOP_UU_B, H1, H2, float16_to_int8) RVVCALL(OPFVV1, vfncvt_x_f_w_h, NOP_UU_H, H2, H4, float32_to_int16) RVVCALL(OPFVV1, vfncvt_x_f_w_w, NOP_UU_W, H4, H8, float64_to_int32) -GEN_VEXT_V_ENV(vfncvt_x_f_w_b, 1, 1) -GEN_VEXT_V_ENV(vfncvt_x_f_w_h, 2, 2) -GEN_VEXT_V_ENV(vfncvt_x_f_w_w, 4, 4) +GEN_VEXT_V_ENV(vfncvt_x_f_w_b) +GEN_VEXT_V_ENV(vfncvt_x_f_w_h) +GEN_VEXT_V_ENV(vfncvt_x_f_w_w) /* vfncvt.f.xu.v vd, vs2, vm # Convert double-width unsigned integer to float */ RVVCALL(OPFVV1, vfncvt_f_xu_w_h, NOP_UU_H, H2, H4, uint32_to_float16) RVVCALL(OPFVV1, vfncvt_f_xu_w_w, NOP_UU_W, H4, H8, uint64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_xu_w_h, 2, 2) -GEN_VEXT_V_ENV(vfncvt_f_xu_w_w, 4, 4) +GEN_VEXT_V_ENV(vfncvt_f_xu_w_h) +GEN_VEXT_V_ENV(vfncvt_f_xu_w_w) /* vfncvt.f.x.v vd, vs2, vm # Convert double-width integer to float. */ RVVCALL(OPFVV1, vfncvt_f_x_w_h, NOP_UU_H, H2, H4, int32_to_float16) RVVCALL(OPFVV1, vfncvt_f_x_w_w, NOP_UU_W, H4, H8, int64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_x_w_h, 2, 2) -GEN_VEXT_V_ENV(vfncvt_f_x_w_w, 4, 4) +GEN_VEXT_V_ENV(vfncvt_f_x_w_h) +GEN_VEXT_V_ENV(vfncvt_f_x_w_w) /* vfncvt.f.f.v vd, vs2, vm # Convert double float to single-width float. */ static uint16_t vfncvtffv16(uint32_t a, float_status *s) @@ -4286,8 +4284,8 @@ static uint16_t vfncvtffv16(uint32_t a, float_status *s) RVVCALL(OPFVV1, vfncvt_f_f_w_h, NOP_UU_H, H2, H4, vfncvtffv16) RVVCALL(OPFVV1, vfncvt_f_f_w_w, NOP_UU_W, H4, H8, float64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_f_w_h, 2, 2) -GEN_VEXT_V_ENV(vfncvt_f_f_w_w, 4, 4) +GEN_VEXT_V_ENV(vfncvt_f_f_w_h) +GEN_VEXT_V_ENV(vfncvt_f_f_w_w) /* *** Vector Reduction Operations From 25eae0486db6e24f86550e3c42af2b17cc43ddbf Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 70/90] target/riscv: rvv: Prune redundant access_type parameter passed No functional change intended in this commit. Signed-off-by: eop Chen Reviewed-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-2@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 85dd611cd9ca..60840325c485 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -231,7 +231,7 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride, CPURISCVState *env, uint32_t desc, uint32_t vm, vext_ldst_elem_fn *ldst_elem, - uint32_t esz, uintptr_t ra, MMUAccessType access_type) + uint32_t esz, uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); @@ -259,7 +259,7 @@ void HELPER(NAME)(void *vd, void * v0, target_ulong base, \ { \ uint32_t vm = vext_vm(desc); \ vext_ldst_stride(vd, v0, base, stride, env, desc, vm, LOAD_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), MMU_DATA_LOAD); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_LD_STRIDE(vlse8_v, int8_t, lde_b) @@ -274,7 +274,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ { \ uint32_t vm = vext_vm(desc); \ vext_ldst_stride(vd, v0, base, stride, env, desc, vm, STORE_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), MMU_DATA_STORE); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_ST_STRIDE(vsse8_v, int8_t, ste_b) @@ -290,7 +290,7 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d) static void vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn *ldst_elem, uint32_t esz, uint32_t evl, - uintptr_t ra, MMUAccessType access_type) + uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); @@ -319,14 +319,14 @@ void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \ { \ uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \ vext_ldst_stride(vd, v0, base, stride, env, desc, false, LOAD_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), MMU_DATA_LOAD); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } \ \ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_us(vd, base, env, desc, LOAD_FN, \ - ctzl(sizeof(ETYPE)), env->vl, GETPC(), MMU_DATA_LOAD); \ + ctzl(sizeof(ETYPE)), env->vl, GETPC()); \ } GEN_VEXT_LD_US(vle8_v, int8_t, lde_b) @@ -340,14 +340,14 @@ void HELPER(NAME##_mask)(void *vd, void *v0, target_ulong base, \ { \ uint32_t stride = vext_nf(desc) << ctzl(sizeof(ETYPE)); \ vext_ldst_stride(vd, v0, base, stride, env, desc, false, STORE_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), MMU_DATA_STORE); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } \ \ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_us(vd, base, env, desc, STORE_FN, \ - ctzl(sizeof(ETYPE)), env->vl, GETPC(), MMU_DATA_STORE); \ + ctzl(sizeof(ETYPE)), env->vl, GETPC()); \ } GEN_VEXT_ST_US(vse8_v, int8_t, ste_b) @@ -364,7 +364,7 @@ void HELPER(vlm_v)(void *vd, void *v0, target_ulong base, /* evl = ceil(vl/8) */ uint8_t evl = (env->vl + 7) >> 3; vext_ldst_us(vd, base, env, desc, lde_b, - 0, evl, GETPC(), MMU_DATA_LOAD); + 0, evl, GETPC()); } void HELPER(vsm_v)(void *vd, void *v0, target_ulong base, @@ -373,7 +373,7 @@ void HELPER(vsm_v)(void *vd, void *v0, target_ulong base, /* evl = ceil(vl/8) */ uint8_t evl = (env->vl + 7) >> 3; vext_ldst_us(vd, base, env, desc, ste_b, - 0, evl, GETPC(), MMU_DATA_STORE); + 0, evl, GETPC()); } /* @@ -399,7 +399,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, void *vs2, CPURISCVState *env, uint32_t desc, vext_get_index_addr get_index_addr, vext_ldst_elem_fn *ldst_elem, - uint32_t esz, uintptr_t ra, MMUAccessType access_type) + uint32_t esz, uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); @@ -427,7 +427,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ void *vs2, CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ - LOAD_FN, ctzl(sizeof(ETYPE)), GETPC(), MMU_DATA_LOAD); \ + LOAD_FN, ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_LD_INDEX(vlxei8_8_v, int8_t, idx_b, lde_b) @@ -453,7 +453,7 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong base, \ { \ vext_ldst_index(vd, v0, base, vs2, env, desc, INDEX_FN, \ STORE_FN, ctzl(sizeof(ETYPE)), \ - GETPC(), MMU_DATA_STORE); \ + GETPC()); \ } GEN_VEXT_ST_INDEX(vsxei8_8_v, int8_t, idx_b, ste_b) @@ -576,8 +576,7 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d) */ static void vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, - vext_ldst_elem_fn *ldst_elem, uint32_t esz, uintptr_t ra, - MMUAccessType access_type) + vext_ldst_elem_fn *ldst_elem, uint32_t esz, uintptr_t ra) { uint32_t i, k, off, pos; uint32_t nf = vext_nf(desc); @@ -612,8 +611,7 @@ void HELPER(NAME)(void *vd, target_ulong base, \ CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_whole(vd, base, env, desc, LOAD_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), \ - MMU_DATA_LOAD); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_LD_WHOLE(vl1re8_v, int8_t, lde_b) @@ -638,8 +636,7 @@ void HELPER(NAME)(void *vd, target_ulong base, \ CPURISCVState *env, uint32_t desc) \ { \ vext_ldst_whole(vd, base, env, desc, STORE_FN, \ - ctzl(sizeof(ETYPE)), GETPC(), \ - MMU_DATA_STORE); \ + ctzl(sizeof(ETYPE)), GETPC()); \ } GEN_VEXT_ST_WHOLE(vs1r_v, int8_t, ste_b) From c7b8a4213b0dba710e31d633e17e661151c3d23a Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 71/90] target/riscv: rvv: Rename ambiguous esz No functional change intended in this commit. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Reviewed-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-3@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 76 ++++++++++++++++++------------------ 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 60840325c485..3b79b9cbc249 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -125,9 +125,9 @@ static inline int32_t vext_lmul(uint32_t desc) /* * Get the maximum number of elements can be operated. * - * esz: log2 of element size in bytes. + * log2_esz: log2 of element size in bytes. */ -static inline uint32_t vext_max_elems(uint32_t desc, uint32_t esz) +static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz) { /* * As simd_desc support at most 2048 bytes, the max vlen is 1024 bits. @@ -136,7 +136,7 @@ static inline uint32_t vext_max_elems(uint32_t desc, uint32_t esz) uint32_t vlenb = simd_maxsz(desc); /* Return VLMAX */ - int scale = vext_lmul(desc) - esz; + int scale = vext_lmul(desc) - log2_esz; return scale < 0 ? vlenb >> -scale : vlenb << scale; } @@ -231,11 +231,11 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, target_ulong stride, CPURISCVState *env, uint32_t desc, uint32_t vm, vext_ldst_elem_fn *ldst_elem, - uint32_t esz, uintptr_t ra) + uint32_t log2_esz, uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); - uint32_t max_elems = vext_max_elems(desc, esz); + uint32_t max_elems = vext_max_elems(desc, log2_esz); for (i = env->vstart; i < env->vl; i++, env->vstart++) { if (!vm && !vext_elem_mask(v0, i)) { @@ -244,7 +244,7 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, k = 0; while (k < nf) { - target_ulong addr = base + stride * i + (k << esz); + target_ulong addr = base + stride * i + (k << log2_esz); ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra); k++; } @@ -289,18 +289,18 @@ GEN_VEXT_ST_STRIDE(vsse64_v, int64_t, ste_d) /* unmasked unit-stride load and store operation*/ static void vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, - vext_ldst_elem_fn *ldst_elem, uint32_t esz, uint32_t evl, + vext_ldst_elem_fn *ldst_elem, uint32_t log2_esz, uint32_t evl, uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); - uint32_t max_elems = vext_max_elems(desc, esz); + uint32_t max_elems = vext_max_elems(desc, log2_esz); /* load bytes from guest memory */ for (i = env->vstart; i < evl; i++, env->vstart++) { k = 0; while (k < nf) { - target_ulong addr = base + ((i * nf + k) << esz); + target_ulong addr = base + ((i * nf + k) << log2_esz); ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra); k++; } @@ -399,12 +399,12 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, void *vs2, CPURISCVState *env, uint32_t desc, vext_get_index_addr get_index_addr, vext_ldst_elem_fn *ldst_elem, - uint32_t esz, uintptr_t ra) + uint32_t log2_esz, uintptr_t ra) { uint32_t i, k; uint32_t nf = vext_nf(desc); uint32_t vm = vext_vm(desc); - uint32_t max_elems = vext_max_elems(desc, esz); + uint32_t max_elems = vext_max_elems(desc, log2_esz); /* load bytes from guest memory */ for (i = env->vstart; i < env->vl; i++, env->vstart++) { @@ -414,7 +414,7 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, k = 0; while (k < nf) { - abi_ptr addr = get_index_addr(base, i, vs2) + (k << esz); + abi_ptr addr = get_index_addr(base, i, vs2) + (k << log2_esz); ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra); k++; } @@ -480,13 +480,13 @@ static inline void vext_ldff(void *vd, void *v0, target_ulong base, CPURISCVState *env, uint32_t desc, vext_ldst_elem_fn *ldst_elem, - uint32_t esz, uintptr_t ra) + uint32_t log2_esz, uintptr_t ra) { void *host; uint32_t i, k, vl = 0; uint32_t nf = vext_nf(desc); uint32_t vm = vext_vm(desc); - uint32_t max_elems = vext_max_elems(desc, esz); + uint32_t max_elems = vext_max_elems(desc, log2_esz); target_ulong addr, offset, remain; /* probe every access*/ @@ -494,12 +494,12 @@ vext_ldff(void *vd, void *v0, target_ulong base, if (!vm && !vext_elem_mask(v0, i)) { continue; } - addr = adjust_addr(env, base + i * (nf << esz)); + addr = adjust_addr(env, base + i * (nf << log2_esz)); if (i == 0) { - probe_pages(env, addr, nf << esz, ra, MMU_DATA_LOAD); + probe_pages(env, addr, nf << log2_esz, ra, MMU_DATA_LOAD); } else { /* if it triggers an exception, no need to check watchpoint */ - remain = nf << esz; + remain = nf << log2_esz; while (remain > 0) { offset = -(addr | TARGET_PAGE_MASK); host = tlb_vaddr_to_host(env, addr, MMU_DATA_LOAD, @@ -536,7 +536,7 @@ vext_ldff(void *vd, void *v0, target_ulong base, continue; } while (k < nf) { - target_ulong addr = base + ((i * nf + k) << esz); + target_ulong addr = base + ((i * nf + k) << log2_esz); ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra); k++; } @@ -576,12 +576,12 @@ GEN_VEXT_LDFF(vle64ff_v, int64_t, lde_d) */ static void vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, - vext_ldst_elem_fn *ldst_elem, uint32_t esz, uintptr_t ra) + vext_ldst_elem_fn *ldst_elem, uint32_t log2_esz, uintptr_t ra) { uint32_t i, k, off, pos; uint32_t nf = vext_nf(desc); uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; - uint32_t max_elems = vlenb >> esz; + uint32_t max_elems = vlenb >> log2_esz; k = env->vstart / max_elems; off = env->vstart % max_elems; @@ -589,7 +589,7 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, if (off) { /* load/store rest of elements of current segment pointed by vstart */ for (pos = off; pos < max_elems; pos++, env->vstart++) { - target_ulong addr = base + ((pos + k * max_elems) << esz); + target_ulong addr = base + ((pos + k * max_elems) << log2_esz); ldst_elem(env, adjust_addr(env, addr), pos + k * max_elems, vd, ra); } k++; @@ -598,7 +598,7 @@ vext_ldst_whole(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, /* load/store elements for rest of segments */ for (; k < nf; k++) { for (i = 0; i < max_elems; i++, env->vstart++) { - target_ulong addr = base + ((i + k * max_elems) << esz); + target_ulong addr = base + ((i + k * max_elems) << log2_esz); ldst_elem(env, adjust_addr(env, addr), i + k * max_elems, vd, ra); } } @@ -4688,11 +4688,11 @@ GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_h, uint16_t, H2) GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_w, uint32_t, H4) GEN_VEXT_VSLIDEDOWN_VX(vslidedown_vx_d, uint64_t, H8) -#define GEN_VEXT_VSLIE1UP(ESZ, H) \ -static void vslide1up_##ESZ(void *vd, void *v0, target_ulong s1, void *vs2, \ - CPURISCVState *env, uint32_t desc) \ +#define GEN_VEXT_VSLIE1UP(BITWIDTH, H) \ +static void vslide1up_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ + void *vs2, CPURISCVState *env, uint32_t desc) \ { \ - typedef uint##ESZ##_t ETYPE; \ + typedef uint##BITWIDTH##_t ETYPE; \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ uint32_t i; \ @@ -4715,11 +4715,11 @@ GEN_VEXT_VSLIE1UP(16, H2) GEN_VEXT_VSLIE1UP(32, H4) GEN_VEXT_VSLIE1UP(64, H8) -#define GEN_VEXT_VSLIDE1UP_VX(NAME, ESZ) \ +#define GEN_VEXT_VSLIDE1UP_VX(NAME, BITWIDTH) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ - vslide1up_##ESZ(vd, v0, s1, vs2, env, desc); \ + vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vslide1up.vx vd, vs2, rs1, vm # vd[0]=x[rs1], vd[i+1] = vs2[i] */ @@ -4728,11 +4728,11 @@ GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_h, 16) GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_w, 32) GEN_VEXT_VSLIDE1UP_VX(vslide1up_vx_d, 64) -#define GEN_VEXT_VSLIDE1DOWN(ESZ, H) \ -static void vslide1down_##ESZ(void *vd, void *v0, target_ulong s1, void *vs2, \ - CPURISCVState *env, uint32_t desc) \ +#define GEN_VEXT_VSLIDE1DOWN(BITWIDTH, H) \ +static void vslide1down_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ + void *vs2, CPURISCVState *env, uint32_t desc) \ { \ - typedef uint##ESZ##_t ETYPE; \ + typedef uint##BITWIDTH##_t ETYPE; \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ uint32_t i; \ @@ -4755,11 +4755,11 @@ GEN_VEXT_VSLIDE1DOWN(16, H2) GEN_VEXT_VSLIDE1DOWN(32, H4) GEN_VEXT_VSLIDE1DOWN(64, H8) -#define GEN_VEXT_VSLIDE1DOWN_VX(NAME, ESZ) \ +#define GEN_VEXT_VSLIDE1DOWN_VX(NAME, BITWIDTH) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ - vslide1down_##ESZ(vd, v0, s1, vs2, env, desc); \ + vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vslide1down.vx vd, vs2, rs1, vm # vd[i] = vs2[i+1], vd[vl-1]=x[rs1] */ @@ -4769,11 +4769,11 @@ GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_w, 32) GEN_VEXT_VSLIDE1DOWN_VX(vslide1down_vx_d, 64) /* Vector Floating-Point Slide Instructions */ -#define GEN_VEXT_VFSLIDE1UP_VF(NAME, ESZ) \ +#define GEN_VEXT_VFSLIDE1UP_VF(NAME, BITWIDTH) \ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ - vslide1up_##ESZ(vd, v0, s1, vs2, env, desc); \ + vslide1up_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vfslide1up.vf vd, vs2, rs1, vm # vd[0]=f[rs1], vd[i+1] = vs2[i] */ @@ -4781,11 +4781,11 @@ GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_h, 16) GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_w, 32) GEN_VEXT_VFSLIDE1UP_VF(vfslide1up_vf_d, 64) -#define GEN_VEXT_VFSLIDE1DOWN_VF(NAME, ESZ) \ +#define GEN_VEXT_VFSLIDE1DOWN_VF(NAME, BITWIDTH) \ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ - vslide1down_##ESZ(vd, v0, s1, vs2, env, desc); \ + vslide1down_##BITWIDTH(vd, v0, s1, vs2, env, desc); \ } /* vfslide1down.vf vd, vs2, rs1, vm # vd[i] = vs2[i+1], vd[vl-1]=f[rs1] */ From 41d3d7f76aa7060c0cbc1c8b3a9767a5997b231a Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 72/90] target/riscv: rvv: Early exit when vstart >= vl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit According to v-spec (section 5.4): When vstart ≥ vl, there are no body elements, and no elements are updated in any destination vector register group, including that no tail elements are updated with agnostic values. vmsbf.m, vmsif.m, vmsof.m, viota.m, vcompress instructions themselves require vstart to be zero. So they don't need the early exit. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-4@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 27 +++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 391c61fe937a..6750f5d04ada 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -652,6 +652,7 @@ static bool ldst_us_trans(uint32_t vd, uint32_t rs1, uint32_t data, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -818,6 +819,7 @@ static bool ldst_stride_trans(uint32_t vd, uint32_t rs1, uint32_t rs2, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -925,6 +927,7 @@ static bool ldst_index_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -1067,6 +1070,7 @@ static bool ldff_trans(uint32_t vd, uint32_t rs1, uint32_t data, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -1225,6 +1229,7 @@ do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, } tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); if (a->vm && s->vl_eq_vlmax) { gvec_fn(s->sew, vreg_ofs(s, a->rd), @@ -1272,6 +1277,7 @@ static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -1436,6 +1442,7 @@ static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -1522,6 +1529,7 @@ static bool do_opivv_widen(DisasContext *s, arg_rmrr *a, uint32_t data = 0; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -1602,6 +1610,7 @@ static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a, uint32_t data = 0; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -1679,6 +1688,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ }; \ TCGLabel *over = gen_new_label(); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -1860,6 +1870,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ }; \ TCGLabel *over = gen_new_label(); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2070,6 +2081,7 @@ static bool trans_vmv_v_v(DisasContext *s, arg_vmv_v_v *a) }; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); tcg_gen_gvec_2_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), cpu_env, s->cfg_ptr->vlen / 8, @@ -2093,6 +2105,7 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) TCGv s1; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); s1 = get_gpr(s, a->rs1, EXT_SIGN); @@ -2148,6 +2161,7 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) }; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); s1 = tcg_constant_i64(simm); dest = tcg_temp_new_ptr(); @@ -2300,6 +2314,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2330,6 +2345,7 @@ static bool opfvf_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); dest = tcg_temp_new_ptr(); mask = tcg_temp_new_ptr(); @@ -2418,6 +2434,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over);\ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2492,6 +2509,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2613,6 +2631,7 @@ static bool do_opfv(DisasContext *s, arg_rmr *a, TCGLabel *over = gen_new_label(); gen_set_rm(s, rm); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -2726,6 +2745,7 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) }; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); t1 = tcg_temp_new_i64(); /* NaN-box f[rs1] */ @@ -2814,6 +2834,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, FRM); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2865,6 +2886,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, RISCV_FRM_DYN); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ @@ -2930,6 +2952,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, FRM); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ @@ -2983,6 +3006,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ TCGLabel *over = gen_new_label(); \ gen_set_rm(s, FRM); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ @@ -3070,6 +3094,7 @@ static bool trans_##NAME(DisasContext *s, arg_r *a) \ gen_helper_gvec_4_ptr *fn = gen_helper_##NAME; \ TCGLabel *over = gen_new_label(); \ tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); \ + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ @@ -3238,6 +3263,7 @@ static bool trans_vid_v(DisasContext *s, arg_vid_v *a) uint32_t data = 0; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); @@ -3748,6 +3774,7 @@ static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq) gen_helper_gvec_3_ptr *fn; TCGLabel *over = gen_new_label(); tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); + tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); static gen_helper_gvec_3_ptr * const fns[6][4] = { { From f1eed927fb3a1212af8e324cf242cf6f4bd6fd04 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 73/90] target/riscv: rvv: Add tail agnostic for vv instructions According to v-spec, tail agnostic behavior can be either kept as undisturbed or set elements' bits to all 1s. To distinguish the difference of tail policies, QEMU should be able to simulate the tail agnostic behavior as "set tail elements' bits to all 1s". There are multiple possibility for agnostic elements according to v-spec. The main intent of this patch-set tries to add option that can distinguish between tail policies. Setting agnostic elements to all 1s allows QEMU to express this. This is the first commit regarding the optional tail agnostic behavior. Follow-up commits will add this optional behavior for all rvv instructions. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-5@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/cpu.h | 2 + target/riscv/cpu_helper.c | 2 + target/riscv/insn_trans/trans_rvv.c.inc | 3 +- target/riscv/internals.h | 5 +- target/riscv/translate.c | 2 + target/riscv/vector_helper.c | 296 +++++++++++++----------- 6 files changed, 178 insertions(+), 132 deletions(-) diff --git a/target/riscv/cpu.h b/target/riscv/cpu.h index 194a58d760d5..7d6397acdfb1 100644 --- a/target/riscv/cpu.h +++ b/target/riscv/cpu.h @@ -412,6 +412,7 @@ struct RISCVCPUConfig { bool ext_zve32f; bool ext_zve64f; bool ext_zmmul; + bool rvv_ta_all_1s; uint32_t mvendorid; uint64_t marchid; @@ -567,6 +568,7 @@ FIELD(TB_FLAGS, XL, 20, 2) /* If PointerMasking should be applied */ FIELD(TB_FLAGS, PM_MASK_ENABLED, 22, 1) FIELD(TB_FLAGS, PM_BASE_ENABLED, 23, 1) +FIELD(TB_FLAGS, VTA, 24, 1) #ifdef TARGET_RISCV32 #define riscv_cpu_mxl(env) ((void)(env), MXL_RV32) diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index 16c6045459fd..4a6700c89086 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -65,6 +65,8 @@ void cpu_get_tb_cpu_state(CPURISCVState *env, target_ulong *pc, flags = FIELD_DP32(flags, TB_FLAGS, LMUL, FIELD_EX64(env->vtype, VTYPE, VLMUL)); flags = FIELD_DP32(flags, TB_FLAGS, VL_EQ_VLMAX, vl_eq_vlmax); + flags = FIELD_DP32(flags, TB_FLAGS, VTA, + FIELD_EX64(env->vtype, VTYPE, VTA)); } else { flags = FIELD_DP32(flags, TB_FLAGS, VILL, 1); } diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 6750f5d04ada..bf9886a93d00 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -1231,7 +1231,7 @@ do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), vreg_ofs(s, a->rs1), MAXSZ(s), MAXSZ(s)); @@ -1240,6 +1240,7 @@ do_opivv_gvec(DisasContext *s, arg_rmrr *a, GVecGen3Fn *gvec_fn, data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), cpu_env, s->cfg_ptr->vlen / 8, diff --git a/target/riscv/internals.h b/target/riscv/internals.h index dbb322bfa7e5..512c6c30cf35 100644 --- a/target/riscv/internals.h +++ b/target/riscv/internals.h @@ -24,8 +24,9 @@ /* share data between vector helpers and decode code */ FIELD(VDATA, VM, 0, 1) FIELD(VDATA, LMUL, 1, 3) -FIELD(VDATA, NF, 4, 4) -FIELD(VDATA, WD, 4, 1) +FIELD(VDATA, VTA, 4, 1) +FIELD(VDATA, NF, 5, 4) +FIELD(VDATA, WD, 5, 1) /* float point classify helpers */ target_ulong fclass_h(uint64_t frs1); diff --git a/target/riscv/translate.c b/target/riscv/translate.c index 55a4713af260..59f0ee9a5059 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -94,6 +94,7 @@ typedef struct DisasContext { */ int8_t lmul; uint8_t sew; + uint8_t vta; target_ulong vstart; bool vl_eq_vlmax; uint8_t ntemp; @@ -1099,6 +1100,7 @@ static void riscv_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) ctx->vill = FIELD_EX32(tb_flags, TB_FLAGS, VILL); ctx->sew = FIELD_EX32(tb_flags, TB_FLAGS, SEW); ctx->lmul = sextract32(FIELD_EX32(tb_flags, TB_FLAGS, LMUL), 0, 3); + ctx->vta = FIELD_EX32(tb_flags, TB_FLAGS, VTA) && cpu->cfg.rvv_ta_all_1s; ctx->vstart = env->vstart; ctx->vl_eq_vlmax = FIELD_EX32(tb_flags, TB_FLAGS, VL_EQ_VLMAX); ctx->misa_mxl_max = env->misa_mxl_max; diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 3b79b9cbc249..e2a2979badad 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -122,6 +122,11 @@ static inline int32_t vext_lmul(uint32_t desc) return sextract32(FIELD_EX32(simd_data(desc), VDATA, LMUL), 0, 3); } +static inline uint32_t vext_vta(uint32_t desc) +{ + return FIELD_EX32(simd_data(desc), VDATA, VTA); +} + /* * Get the maximum number of elements can be operated. * @@ -140,6 +145,21 @@ static inline uint32_t vext_max_elems(uint32_t desc, uint32_t log2_esz) return scale < 0 ? vlenb >> -scale : vlenb << scale; } +/* + * Get number of total elements, including prestart, body and tail elements. + * Note that when LMUL < 1, the tail includes the elements past VLMAX that + * are held in the same vector register. + */ +static inline uint32_t vext_get_total_elems(CPURISCVState *env, uint32_t desc, + uint32_t esz) +{ + uint32_t vlenb = simd_maxsz(desc); + uint32_t sew = 1 << FIELD_EX64(env->vtype, VTYPE, VSEW); + int8_t emul = ctzl(esz) - ctzl(sew) + vext_lmul(desc) < 0 ? 0 : + ctzl(esz) - ctzl(sew) + vext_lmul(desc); + return (vlenb << emul) / esz; +} + static inline target_ulong adjust_addr(CPURISCVState *env, target_ulong addr) { return (addr & env->cur_pmmask) | env->cur_pmbase; @@ -172,6 +192,20 @@ static void probe_pages(CPURISCVState *env, target_ulong addr, } } +/* set agnostic elements to 1s */ +static void vext_set_elems_1s(void *base, uint32_t is_agnostic, uint32_t cnt, + uint32_t tot) +{ + if (is_agnostic == 0) { + /* policy undisturbed */ + return; + } + if (tot - cnt == 0) { + return ; + } + memset(base + cnt, -1, tot - cnt); +} + static inline void vext_set_elem_mask(void *v0, int index, uint8_t value) { @@ -707,10 +741,12 @@ RVVCALL(OPIVV2, vsub_vv_d, OP_SSS_D, H8, H8, H8, DO_SUB) static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2, CPURISCVState *env, uint32_t desc, - opivv2_fn *fn) + opivv2_fn *fn, uint32_t esz) { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); uint32_t i; for (i = env->vstart; i < vl; i++) { @@ -720,26 +756,28 @@ static void do_vext_vv(void *vd, void *v0, void *vs1, void *vs2, fn(vd, vs1, vs2, i); } env->vstart = 0; + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); } /* generate the helpers for OPIVV */ -#define GEN_VEXT_VV(NAME) \ +#define GEN_VEXT_VV(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ do_vext_vv(vd, v0, vs1, vs2, env, desc, \ - do_##NAME); \ + do_##NAME, ESZ); \ } -GEN_VEXT_VV(vadd_vv_b) -GEN_VEXT_VV(vadd_vv_h) -GEN_VEXT_VV(vadd_vv_w) -GEN_VEXT_VV(vadd_vv_d) -GEN_VEXT_VV(vsub_vv_b) -GEN_VEXT_VV(vsub_vv_h) -GEN_VEXT_VV(vsub_vv_w) -GEN_VEXT_VV(vsub_vv_d) +GEN_VEXT_VV(vadd_vv_b, 1) +GEN_VEXT_VV(vadd_vv_h, 2) +GEN_VEXT_VV(vadd_vv_w, 4) +GEN_VEXT_VV(vadd_vv_d, 8) +GEN_VEXT_VV(vsub_vv_b, 1) +GEN_VEXT_VV(vsub_vv_h, 2) +GEN_VEXT_VV(vsub_vv_w, 4) +GEN_VEXT_VV(vsub_vv_d, 8) typedef void opivx2_fn(void *vd, target_long s1, void *vs2, int i); @@ -884,30 +922,30 @@ RVVCALL(OPIVV2, vwadd_wv_w, WOP_WSSS_W, H8, H4, H4, DO_ADD) RVVCALL(OPIVV2, vwsub_wv_b, WOP_WSSS_B, H2, H1, H1, DO_SUB) RVVCALL(OPIVV2, vwsub_wv_h, WOP_WSSS_H, H4, H2, H2, DO_SUB) RVVCALL(OPIVV2, vwsub_wv_w, WOP_WSSS_W, H8, H4, H4, DO_SUB) -GEN_VEXT_VV(vwaddu_vv_b) -GEN_VEXT_VV(vwaddu_vv_h) -GEN_VEXT_VV(vwaddu_vv_w) -GEN_VEXT_VV(vwsubu_vv_b) -GEN_VEXT_VV(vwsubu_vv_h) -GEN_VEXT_VV(vwsubu_vv_w) -GEN_VEXT_VV(vwadd_vv_b) -GEN_VEXT_VV(vwadd_vv_h) -GEN_VEXT_VV(vwadd_vv_w) -GEN_VEXT_VV(vwsub_vv_b) -GEN_VEXT_VV(vwsub_vv_h) -GEN_VEXT_VV(vwsub_vv_w) -GEN_VEXT_VV(vwaddu_wv_b) -GEN_VEXT_VV(vwaddu_wv_h) -GEN_VEXT_VV(vwaddu_wv_w) -GEN_VEXT_VV(vwsubu_wv_b) -GEN_VEXT_VV(vwsubu_wv_h) -GEN_VEXT_VV(vwsubu_wv_w) -GEN_VEXT_VV(vwadd_wv_b) -GEN_VEXT_VV(vwadd_wv_h) -GEN_VEXT_VV(vwadd_wv_w) -GEN_VEXT_VV(vwsub_wv_b) -GEN_VEXT_VV(vwsub_wv_h) -GEN_VEXT_VV(vwsub_wv_w) +GEN_VEXT_VV(vwaddu_vv_b, 2) +GEN_VEXT_VV(vwaddu_vv_h, 4) +GEN_VEXT_VV(vwaddu_vv_w, 8) +GEN_VEXT_VV(vwsubu_vv_b, 2) +GEN_VEXT_VV(vwsubu_vv_h, 4) +GEN_VEXT_VV(vwsubu_vv_w, 8) +GEN_VEXT_VV(vwadd_vv_b, 2) +GEN_VEXT_VV(vwadd_vv_h, 4) +GEN_VEXT_VV(vwadd_vv_w, 8) +GEN_VEXT_VV(vwsub_vv_b, 2) +GEN_VEXT_VV(vwsub_vv_h, 4) +GEN_VEXT_VV(vwsub_vv_w, 8) +GEN_VEXT_VV(vwaddu_wv_b, 2) +GEN_VEXT_VV(vwaddu_wv_h, 4) +GEN_VEXT_VV(vwaddu_wv_w, 8) +GEN_VEXT_VV(vwsubu_wv_b, 2) +GEN_VEXT_VV(vwsubu_wv_h, 4) +GEN_VEXT_VV(vwsubu_wv_w, 8) +GEN_VEXT_VV(vwadd_wv_b, 2) +GEN_VEXT_VV(vwadd_wv_h, 4) +GEN_VEXT_VV(vwadd_wv_w, 8) +GEN_VEXT_VV(vwsub_wv_b, 2) +GEN_VEXT_VV(vwsub_wv_h, 4) +GEN_VEXT_VV(vwsub_wv_w, 8) RVVCALL(OPIVX2, vwaddu_vx_b, WOP_UUU_B, H2, H1, DO_ADD) RVVCALL(OPIVX2, vwaddu_vx_h, WOP_UUU_H, H4, H2, DO_ADD) @@ -1086,18 +1124,18 @@ RVVCALL(OPIVV2, vxor_vv_b, OP_SSS_B, H1, H1, H1, DO_XOR) RVVCALL(OPIVV2, vxor_vv_h, OP_SSS_H, H2, H2, H2, DO_XOR) RVVCALL(OPIVV2, vxor_vv_w, OP_SSS_W, H4, H4, H4, DO_XOR) RVVCALL(OPIVV2, vxor_vv_d, OP_SSS_D, H8, H8, H8, DO_XOR) -GEN_VEXT_VV(vand_vv_b) -GEN_VEXT_VV(vand_vv_h) -GEN_VEXT_VV(vand_vv_w) -GEN_VEXT_VV(vand_vv_d) -GEN_VEXT_VV(vor_vv_b) -GEN_VEXT_VV(vor_vv_h) -GEN_VEXT_VV(vor_vv_w) -GEN_VEXT_VV(vor_vv_d) -GEN_VEXT_VV(vxor_vv_b) -GEN_VEXT_VV(vxor_vv_h) -GEN_VEXT_VV(vxor_vv_w) -GEN_VEXT_VV(vxor_vv_d) +GEN_VEXT_VV(vand_vv_b, 1) +GEN_VEXT_VV(vand_vv_h, 2) +GEN_VEXT_VV(vand_vv_w, 4) +GEN_VEXT_VV(vand_vv_d, 8) +GEN_VEXT_VV(vor_vv_b, 1) +GEN_VEXT_VV(vor_vv_h, 2) +GEN_VEXT_VV(vor_vv_w, 4) +GEN_VEXT_VV(vor_vv_d, 8) +GEN_VEXT_VV(vxor_vv_b, 1) +GEN_VEXT_VV(vxor_vv_h, 2) +GEN_VEXT_VV(vxor_vv_w, 4) +GEN_VEXT_VV(vxor_vv_d, 8) RVVCALL(OPIVX2, vand_vx_b, OP_SSS_B, H1, H1, DO_AND) RVVCALL(OPIVX2, vand_vx_h, OP_SSS_H, H2, H2, DO_AND) @@ -1343,22 +1381,22 @@ RVVCALL(OPIVV2, vmax_vv_b, OP_SSS_B, H1, H1, H1, DO_MAX) RVVCALL(OPIVV2, vmax_vv_h, OP_SSS_H, H2, H2, H2, DO_MAX) RVVCALL(OPIVV2, vmax_vv_w, OP_SSS_W, H4, H4, H4, DO_MAX) RVVCALL(OPIVV2, vmax_vv_d, OP_SSS_D, H8, H8, H8, DO_MAX) -GEN_VEXT_VV(vminu_vv_b) -GEN_VEXT_VV(vminu_vv_h) -GEN_VEXT_VV(vminu_vv_w) -GEN_VEXT_VV(vminu_vv_d) -GEN_VEXT_VV(vmin_vv_b) -GEN_VEXT_VV(vmin_vv_h) -GEN_VEXT_VV(vmin_vv_w) -GEN_VEXT_VV(vmin_vv_d) -GEN_VEXT_VV(vmaxu_vv_b) -GEN_VEXT_VV(vmaxu_vv_h) -GEN_VEXT_VV(vmaxu_vv_w) -GEN_VEXT_VV(vmaxu_vv_d) -GEN_VEXT_VV(vmax_vv_b) -GEN_VEXT_VV(vmax_vv_h) -GEN_VEXT_VV(vmax_vv_w) -GEN_VEXT_VV(vmax_vv_d) +GEN_VEXT_VV(vminu_vv_b, 1) +GEN_VEXT_VV(vminu_vv_h, 2) +GEN_VEXT_VV(vminu_vv_w, 4) +GEN_VEXT_VV(vminu_vv_d, 8) +GEN_VEXT_VV(vmin_vv_b, 1) +GEN_VEXT_VV(vmin_vv_h, 2) +GEN_VEXT_VV(vmin_vv_w, 4) +GEN_VEXT_VV(vmin_vv_d, 8) +GEN_VEXT_VV(vmaxu_vv_b, 1) +GEN_VEXT_VV(vmaxu_vv_h, 2) +GEN_VEXT_VV(vmaxu_vv_w, 4) +GEN_VEXT_VV(vmaxu_vv_d, 8) +GEN_VEXT_VV(vmax_vv_b, 1) +GEN_VEXT_VV(vmax_vv_h, 2) +GEN_VEXT_VV(vmax_vv_w, 4) +GEN_VEXT_VV(vmax_vv_d, 8) RVVCALL(OPIVX2, vminu_vx_b, OP_UUU_B, H1, H1, DO_MIN) RVVCALL(OPIVX2, vminu_vx_h, OP_UUU_H, H2, H2, DO_MIN) @@ -1399,10 +1437,10 @@ RVVCALL(OPIVV2, vmul_vv_b, OP_SSS_B, H1, H1, H1, DO_MUL) RVVCALL(OPIVV2, vmul_vv_h, OP_SSS_H, H2, H2, H2, DO_MUL) RVVCALL(OPIVV2, vmul_vv_w, OP_SSS_W, H4, H4, H4, DO_MUL) RVVCALL(OPIVV2, vmul_vv_d, OP_SSS_D, H8, H8, H8, DO_MUL) -GEN_VEXT_VV(vmul_vv_b) -GEN_VEXT_VV(vmul_vv_h) -GEN_VEXT_VV(vmul_vv_w) -GEN_VEXT_VV(vmul_vv_d) +GEN_VEXT_VV(vmul_vv_b, 1) +GEN_VEXT_VV(vmul_vv_h, 2) +GEN_VEXT_VV(vmul_vv_w, 4) +GEN_VEXT_VV(vmul_vv_d, 8) static int8_t do_mulh_b(int8_t s2, int8_t s1) { @@ -1506,18 +1544,18 @@ RVVCALL(OPIVV2, vmulhsu_vv_b, OP_SUS_B, H1, H1, H1, do_mulhsu_b) RVVCALL(OPIVV2, vmulhsu_vv_h, OP_SUS_H, H2, H2, H2, do_mulhsu_h) RVVCALL(OPIVV2, vmulhsu_vv_w, OP_SUS_W, H4, H4, H4, do_mulhsu_w) RVVCALL(OPIVV2, vmulhsu_vv_d, OP_SUS_D, H8, H8, H8, do_mulhsu_d) -GEN_VEXT_VV(vmulh_vv_b) -GEN_VEXT_VV(vmulh_vv_h) -GEN_VEXT_VV(vmulh_vv_w) -GEN_VEXT_VV(vmulh_vv_d) -GEN_VEXT_VV(vmulhu_vv_b) -GEN_VEXT_VV(vmulhu_vv_h) -GEN_VEXT_VV(vmulhu_vv_w) -GEN_VEXT_VV(vmulhu_vv_d) -GEN_VEXT_VV(vmulhsu_vv_b) -GEN_VEXT_VV(vmulhsu_vv_h) -GEN_VEXT_VV(vmulhsu_vv_w) -GEN_VEXT_VV(vmulhsu_vv_d) +GEN_VEXT_VV(vmulh_vv_b, 1) +GEN_VEXT_VV(vmulh_vv_h, 2) +GEN_VEXT_VV(vmulh_vv_w, 4) +GEN_VEXT_VV(vmulh_vv_d, 8) +GEN_VEXT_VV(vmulhu_vv_b, 1) +GEN_VEXT_VV(vmulhu_vv_h, 2) +GEN_VEXT_VV(vmulhu_vv_w, 4) +GEN_VEXT_VV(vmulhu_vv_d, 8) +GEN_VEXT_VV(vmulhsu_vv_b, 1) +GEN_VEXT_VV(vmulhsu_vv_h, 2) +GEN_VEXT_VV(vmulhsu_vv_w, 4) +GEN_VEXT_VV(vmulhsu_vv_d, 8) RVVCALL(OPIVX2, vmul_vx_b, OP_SSS_B, H1, H1, DO_MUL) RVVCALL(OPIVX2, vmul_vx_h, OP_SSS_H, H2, H2, DO_MUL) @@ -1576,22 +1614,22 @@ RVVCALL(OPIVV2, vrem_vv_b, OP_SSS_B, H1, H1, H1, DO_REM) RVVCALL(OPIVV2, vrem_vv_h, OP_SSS_H, H2, H2, H2, DO_REM) RVVCALL(OPIVV2, vrem_vv_w, OP_SSS_W, H4, H4, H4, DO_REM) RVVCALL(OPIVV2, vrem_vv_d, OP_SSS_D, H8, H8, H8, DO_REM) -GEN_VEXT_VV(vdivu_vv_b) -GEN_VEXT_VV(vdivu_vv_h) -GEN_VEXT_VV(vdivu_vv_w) -GEN_VEXT_VV(vdivu_vv_d) -GEN_VEXT_VV(vdiv_vv_b) -GEN_VEXT_VV(vdiv_vv_h) -GEN_VEXT_VV(vdiv_vv_w) -GEN_VEXT_VV(vdiv_vv_d) -GEN_VEXT_VV(vremu_vv_b) -GEN_VEXT_VV(vremu_vv_h) -GEN_VEXT_VV(vremu_vv_w) -GEN_VEXT_VV(vremu_vv_d) -GEN_VEXT_VV(vrem_vv_b) -GEN_VEXT_VV(vrem_vv_h) -GEN_VEXT_VV(vrem_vv_w) -GEN_VEXT_VV(vrem_vv_d) +GEN_VEXT_VV(vdivu_vv_b, 1) +GEN_VEXT_VV(vdivu_vv_h, 2) +GEN_VEXT_VV(vdivu_vv_w, 4) +GEN_VEXT_VV(vdivu_vv_d, 8) +GEN_VEXT_VV(vdiv_vv_b, 1) +GEN_VEXT_VV(vdiv_vv_h, 2) +GEN_VEXT_VV(vdiv_vv_w, 4) +GEN_VEXT_VV(vdiv_vv_d, 8) +GEN_VEXT_VV(vremu_vv_b, 1) +GEN_VEXT_VV(vremu_vv_h, 2) +GEN_VEXT_VV(vremu_vv_w, 4) +GEN_VEXT_VV(vremu_vv_d, 8) +GEN_VEXT_VV(vrem_vv_b, 1) +GEN_VEXT_VV(vrem_vv_h, 2) +GEN_VEXT_VV(vrem_vv_w, 4) +GEN_VEXT_VV(vrem_vv_d, 8) RVVCALL(OPIVX2, vdivu_vx_b, OP_UUU_B, H1, H1, DO_DIVU) RVVCALL(OPIVX2, vdivu_vx_h, OP_UUU_H, H2, H2, DO_DIVU) @@ -1636,15 +1674,15 @@ RVVCALL(OPIVV2, vwmulu_vv_w, WOP_UUU_W, H8, H4, H4, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_b, WOP_SUS_B, H2, H1, H1, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_h, WOP_SUS_H, H4, H2, H2, DO_MUL) RVVCALL(OPIVV2, vwmulsu_vv_w, WOP_SUS_W, H8, H4, H4, DO_MUL) -GEN_VEXT_VV(vwmul_vv_b) -GEN_VEXT_VV(vwmul_vv_h) -GEN_VEXT_VV(vwmul_vv_w) -GEN_VEXT_VV(vwmulu_vv_b) -GEN_VEXT_VV(vwmulu_vv_h) -GEN_VEXT_VV(vwmulu_vv_w) -GEN_VEXT_VV(vwmulsu_vv_b) -GEN_VEXT_VV(vwmulsu_vv_h) -GEN_VEXT_VV(vwmulsu_vv_w) +GEN_VEXT_VV(vwmul_vv_b, 2) +GEN_VEXT_VV(vwmul_vv_h, 4) +GEN_VEXT_VV(vwmul_vv_w, 8) +GEN_VEXT_VV(vwmulu_vv_b, 2) +GEN_VEXT_VV(vwmulu_vv_h, 4) +GEN_VEXT_VV(vwmulu_vv_w, 8) +GEN_VEXT_VV(vwmulsu_vv_b, 2) +GEN_VEXT_VV(vwmulsu_vv_h, 4) +GEN_VEXT_VV(vwmulsu_vv_w, 8) RVVCALL(OPIVX2, vwmul_vx_b, WOP_SSS_B, H2, H1, DO_MUL) RVVCALL(OPIVX2, vwmul_vx_h, WOP_SSS_H, H4, H2, DO_MUL) @@ -1695,22 +1733,22 @@ RVVCALL(OPIVV3, vnmsub_vv_b, OP_SSS_B, H1, H1, H1, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_h, OP_SSS_H, H2, H2, H2, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_w, OP_SSS_W, H4, H4, H4, DO_NMSUB) RVVCALL(OPIVV3, vnmsub_vv_d, OP_SSS_D, H8, H8, H8, DO_NMSUB) -GEN_VEXT_VV(vmacc_vv_b) -GEN_VEXT_VV(vmacc_vv_h) -GEN_VEXT_VV(vmacc_vv_w) -GEN_VEXT_VV(vmacc_vv_d) -GEN_VEXT_VV(vnmsac_vv_b) -GEN_VEXT_VV(vnmsac_vv_h) -GEN_VEXT_VV(vnmsac_vv_w) -GEN_VEXT_VV(vnmsac_vv_d) -GEN_VEXT_VV(vmadd_vv_b) -GEN_VEXT_VV(vmadd_vv_h) -GEN_VEXT_VV(vmadd_vv_w) -GEN_VEXT_VV(vmadd_vv_d) -GEN_VEXT_VV(vnmsub_vv_b) -GEN_VEXT_VV(vnmsub_vv_h) -GEN_VEXT_VV(vnmsub_vv_w) -GEN_VEXT_VV(vnmsub_vv_d) +GEN_VEXT_VV(vmacc_vv_b, 1) +GEN_VEXT_VV(vmacc_vv_h, 2) +GEN_VEXT_VV(vmacc_vv_w, 4) +GEN_VEXT_VV(vmacc_vv_d, 8) +GEN_VEXT_VV(vnmsac_vv_b, 1) +GEN_VEXT_VV(vnmsac_vv_h, 2) +GEN_VEXT_VV(vnmsac_vv_w, 4) +GEN_VEXT_VV(vnmsac_vv_d, 8) +GEN_VEXT_VV(vmadd_vv_b, 1) +GEN_VEXT_VV(vmadd_vv_h, 2) +GEN_VEXT_VV(vmadd_vv_w, 4) +GEN_VEXT_VV(vmadd_vv_d, 8) +GEN_VEXT_VV(vnmsub_vv_b, 1) +GEN_VEXT_VV(vnmsub_vv_h, 2) +GEN_VEXT_VV(vnmsub_vv_w, 4) +GEN_VEXT_VV(vnmsub_vv_d, 8) #define OPIVX3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, target_long s1, void *vs2, int i) \ @@ -1763,15 +1801,15 @@ RVVCALL(OPIVV3, vwmacc_vv_w, WOP_SSS_W, H8, H4, H4, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_b, WOP_SSU_B, H2, H1, H1, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_h, WOP_SSU_H, H4, H2, H2, DO_MACC) RVVCALL(OPIVV3, vwmaccsu_vv_w, WOP_SSU_W, H8, H4, H4, DO_MACC) -GEN_VEXT_VV(vwmaccu_vv_b) -GEN_VEXT_VV(vwmaccu_vv_h) -GEN_VEXT_VV(vwmaccu_vv_w) -GEN_VEXT_VV(vwmacc_vv_b) -GEN_VEXT_VV(vwmacc_vv_h) -GEN_VEXT_VV(vwmacc_vv_w) -GEN_VEXT_VV(vwmaccsu_vv_b) -GEN_VEXT_VV(vwmaccsu_vv_h) -GEN_VEXT_VV(vwmaccsu_vv_w) +GEN_VEXT_VV(vwmaccu_vv_b, 2) +GEN_VEXT_VV(vwmaccu_vv_h, 4) +GEN_VEXT_VV(vwmaccu_vv_w, 8) +GEN_VEXT_VV(vwmacc_vv_b, 2) +GEN_VEXT_VV(vwmacc_vv_h, 4) +GEN_VEXT_VV(vwmacc_vv_w, 8) +GEN_VEXT_VV(vwmaccsu_vv_b, 2) +GEN_VEXT_VV(vwmaccsu_vv_h, 4) +GEN_VEXT_VV(vwmaccsu_vv_w, 8) RVVCALL(OPIVX3, vwmaccu_vx_b, WOP_UUU_B, H2, H1, DO_MACC) RVVCALL(OPIVX3, vwmaccu_vx_h, WOP_UUU_H, H4, H2, DO_MACC) From 752614cab8e61bb6ba96cee1ec127eba6c35398e Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 74/90] target/riscv: rvv: Add tail agnostic for vector load / store instructions Destination register of unit-stride mask load and store instructions are always written with a tail-agnostic policy. A vector segment load / store instruction may contain fractional lmul with nf * lmul > 1. The rest of the elements in the last register should be treated as tail elements. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-6@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 6 +++ target/riscv/translate.c | 2 + target/riscv/vector_helper.c | 60 +++++++++++++++++++++++++ 3 files changed, 68 insertions(+) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index bf9886a93d00..cd73fd61199a 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -711,6 +711,7 @@ static bool ld_us_op(DisasContext *s, arg_r2nfvm *a, uint8_t eew) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, emul); data = FIELD_DP32(data, VDATA, NF, a->nf); + data = FIELD_DP32(data, VDATA, VTA, s->vta); return ldst_us_trans(a->rd, a->rs1, data, fn, s, false); } @@ -774,6 +775,8 @@ static bool ld_us_mask_op(DisasContext *s, arg_vlm_v *a, uint8_t eew) /* EMUL = 1, NFIELDS = 1 */ data = FIELD_DP32(data, VDATA, LMUL, 0); data = FIELD_DP32(data, VDATA, NF, 1); + /* Mask destination register are always tail-agnostic */ + data = FIELD_DP32(data, VDATA, VTA, s->cfg_vta_all_1s); return ldst_us_trans(a->rd, a->rs1, data, fn, s, false); } @@ -862,6 +865,7 @@ static bool ld_stride_op(DisasContext *s, arg_rnfvm *a, uint8_t eew) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, emul); data = FIELD_DP32(data, VDATA, NF, a->nf); + data = FIELD_DP32(data, VDATA, VTA, s->vta); return ldst_stride_trans(a->rd, a->rs1, a->rs2, data, fn, s, false); } @@ -991,6 +995,7 @@ static bool ld_index_op(DisasContext *s, arg_rnfvm *a, uint8_t eew) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, emul); data = FIELD_DP32(data, VDATA, NF, a->nf); + data = FIELD_DP32(data, VDATA, VTA, s->vta); return ldst_index_trans(a->rd, a->rs1, a->rs2, data, fn, s, false); } @@ -1108,6 +1113,7 @@ static bool ldff_op(DisasContext *s, arg_r2nfvm *a, uint8_t eew) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, emul); data = FIELD_DP32(data, VDATA, NF, a->nf); + data = FIELD_DP32(data, VDATA, VTA, s->vta); return ldff_trans(a->rd, a->rs1, data, fn, s); } diff --git a/target/riscv/translate.c b/target/riscv/translate.c index 59f0ee9a5059..b151c20674d6 100644 --- a/target/riscv/translate.c +++ b/target/riscv/translate.c @@ -95,6 +95,7 @@ typedef struct DisasContext { int8_t lmul; uint8_t sew; uint8_t vta; + bool cfg_vta_all_1s; target_ulong vstart; bool vl_eq_vlmax; uint8_t ntemp; @@ -1101,6 +1102,7 @@ static void riscv_tr_init_disas_context(DisasContextBase *dcbase, CPUState *cs) ctx->sew = FIELD_EX32(tb_flags, TB_FLAGS, SEW); ctx->lmul = sextract32(FIELD_EX32(tb_flags, TB_FLAGS, LMUL), 0, 3); ctx->vta = FIELD_EX32(tb_flags, TB_FLAGS, VTA) && cpu->cfg.rvv_ta_all_1s; + ctx->cfg_vta_all_1s = cpu->cfg.rvv_ta_all_1s; ctx->vstart = env->vstart; ctx->vl_eq_vlmax = FIELD_EX32(tb_flags, TB_FLAGS, VL_EQ_VLMAX); ctx->misa_mxl_max = env->misa_mxl_max; diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index e2a2979badad..ee28e1b92d31 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -270,6 +270,9 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, uint32_t i, k; uint32_t nf = vext_nf(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); + uint32_t esz = 1 << log2_esz; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); for (i = env->vstart; i < env->vl; i++, env->vstart++) { if (!vm && !vext_elem_mask(v0, i)) { @@ -284,6 +287,18 @@ vext_ldst_stride(void *vd, void *v0, target_ulong base, } } env->vstart = 0; + /* set tail elements to 1s */ + for (k = 0; k < nf; ++k) { + vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, + (k * max_elems + max_elems) * esz); + } + if (nf * max_elems % total_elems != 0) { + uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; + uint32_t registers_used = + ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; + vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, + registers_used * vlenb); + } } #define GEN_VEXT_LD_STRIDE(NAME, ETYPE, LOAD_FN) \ @@ -329,6 +344,9 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, uint32_t i, k; uint32_t nf = vext_nf(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); + uint32_t esz = 1 << log2_esz; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); /* load bytes from guest memory */ for (i = env->vstart; i < evl; i++, env->vstart++) { @@ -340,6 +358,18 @@ vext_ldst_us(void *vd, target_ulong base, CPURISCVState *env, uint32_t desc, } } env->vstart = 0; + /* set tail elements to 1s */ + for (k = 0; k < nf; ++k) { + vext_set_elems_1s(vd, vta, (k * max_elems + evl) * esz, + (k * max_elems + max_elems) * esz); + } + if (nf * max_elems % total_elems != 0) { + uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; + uint32_t registers_used = + ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; + vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, + registers_used * vlenb); + } } /* @@ -439,6 +469,9 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, uint32_t nf = vext_nf(desc); uint32_t vm = vext_vm(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); + uint32_t esz = 1 << log2_esz; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); /* load bytes from guest memory */ for (i = env->vstart; i < env->vl; i++, env->vstart++) { @@ -454,6 +487,18 @@ vext_ldst_index(void *vd, void *v0, target_ulong base, } } env->vstart = 0; + /* set tail elements to 1s */ + for (k = 0; k < nf; ++k) { + vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, + (k * max_elems + max_elems) * esz); + } + if (nf * max_elems % total_elems != 0) { + uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; + uint32_t registers_used = + ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; + vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, + registers_used * vlenb); + } } #define GEN_VEXT_LD_INDEX(NAME, ETYPE, INDEX_FN, LOAD_FN) \ @@ -521,6 +566,9 @@ vext_ldff(void *vd, void *v0, target_ulong base, uint32_t nf = vext_nf(desc); uint32_t vm = vext_vm(desc); uint32_t max_elems = vext_max_elems(desc, log2_esz); + uint32_t esz = 1 << log2_esz; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); target_ulong addr, offset, remain; /* probe every access*/ @@ -576,6 +624,18 @@ vext_ldff(void *vd, void *v0, target_ulong base, } } env->vstart = 0; + /* set tail elements to 1s */ + for (k = 0; k < nf; ++k) { + vext_set_elems_1s(vd, vta, (k * max_elems + env->vl) * esz, + (k * max_elems + max_elems) * esz); + } + if (nf * max_elems % total_elems != 0) { + uint32_t vlenb = env_archcpu(env)->cfg.vlen >> 3; + uint32_t registers_used = + ((nf * max_elems) * esz + (vlenb - 1)) / vlenb; + vext_set_elems_1s(vd, vta, (nf * max_elems) * esz, + registers_used * vlenb); + } } #define GEN_VEXT_LDFF(NAME, ETYPE, LOAD_FN) \ From 5c19fc156e5cea2516085c487eb72cdb331c54b6 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:38 +0000 Subject: [PATCH 75/90] target/riscv: rvv: Add tail agnostic for vx, vvm, vxm instructions `vmadc` and `vmsbc` produces a mask value, they always operate with a tail agnostic policy. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-7@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 13 +- target/riscv/internals.h | 5 +- target/riscv/vector_helper.c | 314 +++++++++++++----------- 3 files changed, 190 insertions(+), 142 deletions(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index cd73fd61199a..22edf6228dfb 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -1293,6 +1293,8 @@ static bool opivx_trans(uint32_t vd, uint32_t rs1, uint32_t vs2, uint32_t vm, data = FIELD_DP32(data, VDATA, VM, vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); + data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, data)); @@ -1328,7 +1330,7 @@ do_opivx_gvec(DisasContext *s, arg_rmrr *a, GVecGen2sFn *gvec_fn, return false; } - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { TCGv_i64 src1 = tcg_temp_new_i64(); tcg_gen_ext_tl_i64(src1, get_gpr(s, a->rs1, EXT_SIGN)); @@ -1458,6 +1460,8 @@ static bool opivi_trans(uint32_t vd, uint32_t imm, uint32_t vs2, uint32_t vm, data = FIELD_DP32(data, VDATA, VM, vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); + data = FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s); desc = tcg_constant_i32(simd_desc(s->cfg_ptr->vlen / 8, s->cfg_ptr->vlen / 8, data)); @@ -1486,7 +1490,7 @@ do_opivi_gvec(DisasContext *s, arg_rmrr *a, GVecGen2iFn *gvec_fn, return false; } - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { gvec_fn(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs2), extract_imm(s, a->rs1, imm_mode), MAXSZ(s), MAXSZ(s)); mark_vs_dirty(s); @@ -1540,6 +1544,7 @@ static bool do_opivv_widen(DisasContext *s, arg_rmrr *a, data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), @@ -1621,6 +1626,7 @@ static bool do_opiwv_widen(DisasContext *s, arg_rmrr *a, data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), @@ -1699,6 +1705,9 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ + data = \ + FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ diff --git a/target/riscv/internals.h b/target/riscv/internals.h index 512c6c30cf35..193ce57a6dc0 100644 --- a/target/riscv/internals.h +++ b/target/riscv/internals.h @@ -25,8 +25,9 @@ FIELD(VDATA, VM, 0, 1) FIELD(VDATA, LMUL, 1, 3) FIELD(VDATA, VTA, 4, 1) -FIELD(VDATA, NF, 5, 4) -FIELD(VDATA, WD, 5, 1) +FIELD(VDATA, VTA_ALL_1S, 5, 1) +FIELD(VDATA, NF, 6, 4) +FIELD(VDATA, WD, 6, 1) /* float point classify helpers */ target_ulong fclass_h(uint64_t frs1); diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index ee28e1b92d31..7cdb5d12af29 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -127,6 +127,11 @@ static inline uint32_t vext_vta(uint32_t desc) return FIELD_EX32(simd_data(desc), VDATA, VTA); } +static inline uint32_t vext_vta_all_1s(uint32_t desc) +{ + return FIELD_EX32(simd_data(desc), VDATA, VTA_ALL_1S); +} + /* * Get the maximum number of elements can be operated. * @@ -867,10 +872,12 @@ RVVCALL(OPIVX2, vrsub_vx_d, OP_SSS_D, H8, H8, DO_RSUB) static void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2, CPURISCVState *env, uint32_t desc, - opivx2_fn fn) + opivx2_fn fn, uint32_t esz) { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); uint32_t i; for (i = env->vstart; i < vl; i++) { @@ -880,30 +887,32 @@ static void do_vext_vx(void *vd, void *v0, target_long s1, void *vs2, fn(vd, s1, vs2, i); } env->vstart = 0; + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); } /* generate the helpers for OPIVX */ -#define GEN_VEXT_VX(NAME) \ +#define GEN_VEXT_VX(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ do_vext_vx(vd, v0, s1, vs2, env, desc, \ - do_##NAME); \ -} - -GEN_VEXT_VX(vadd_vx_b) -GEN_VEXT_VX(vadd_vx_h) -GEN_VEXT_VX(vadd_vx_w) -GEN_VEXT_VX(vadd_vx_d) -GEN_VEXT_VX(vsub_vx_b) -GEN_VEXT_VX(vsub_vx_h) -GEN_VEXT_VX(vsub_vx_w) -GEN_VEXT_VX(vsub_vx_d) -GEN_VEXT_VX(vrsub_vx_b) -GEN_VEXT_VX(vrsub_vx_h) -GEN_VEXT_VX(vrsub_vx_w) -GEN_VEXT_VX(vrsub_vx_d) + do_##NAME, ESZ); \ +} + +GEN_VEXT_VX(vadd_vx_b, 1) +GEN_VEXT_VX(vadd_vx_h, 2) +GEN_VEXT_VX(vadd_vx_w, 4) +GEN_VEXT_VX(vadd_vx_d, 8) +GEN_VEXT_VX(vsub_vx_b, 1) +GEN_VEXT_VX(vsub_vx_h, 2) +GEN_VEXT_VX(vsub_vx_w, 4) +GEN_VEXT_VX(vsub_vx_d, 8) +GEN_VEXT_VX(vrsub_vx_b, 1) +GEN_VEXT_VX(vrsub_vx_h, 2) +GEN_VEXT_VX(vrsub_vx_w, 4) +GEN_VEXT_VX(vrsub_vx_d, 8) void HELPER(vec_rsubs8)(void *d, void *a, uint64_t b, uint32_t desc) { @@ -1031,30 +1040,30 @@ RVVCALL(OPIVX2, vwadd_wx_w, WOP_WSSS_W, H8, H4, DO_ADD) RVVCALL(OPIVX2, vwsub_wx_b, WOP_WSSS_B, H2, H1, DO_SUB) RVVCALL(OPIVX2, vwsub_wx_h, WOP_WSSS_H, H4, H2, DO_SUB) RVVCALL(OPIVX2, vwsub_wx_w, WOP_WSSS_W, H8, H4, DO_SUB) -GEN_VEXT_VX(vwaddu_vx_b) -GEN_VEXT_VX(vwaddu_vx_h) -GEN_VEXT_VX(vwaddu_vx_w) -GEN_VEXT_VX(vwsubu_vx_b) -GEN_VEXT_VX(vwsubu_vx_h) -GEN_VEXT_VX(vwsubu_vx_w) -GEN_VEXT_VX(vwadd_vx_b) -GEN_VEXT_VX(vwadd_vx_h) -GEN_VEXT_VX(vwadd_vx_w) -GEN_VEXT_VX(vwsub_vx_b) -GEN_VEXT_VX(vwsub_vx_h) -GEN_VEXT_VX(vwsub_vx_w) -GEN_VEXT_VX(vwaddu_wx_b) -GEN_VEXT_VX(vwaddu_wx_h) -GEN_VEXT_VX(vwaddu_wx_w) -GEN_VEXT_VX(vwsubu_wx_b) -GEN_VEXT_VX(vwsubu_wx_h) -GEN_VEXT_VX(vwsubu_wx_w) -GEN_VEXT_VX(vwadd_wx_b) -GEN_VEXT_VX(vwadd_wx_h) -GEN_VEXT_VX(vwadd_wx_w) -GEN_VEXT_VX(vwsub_wx_b) -GEN_VEXT_VX(vwsub_wx_h) -GEN_VEXT_VX(vwsub_wx_w) +GEN_VEXT_VX(vwaddu_vx_b, 2) +GEN_VEXT_VX(vwaddu_vx_h, 4) +GEN_VEXT_VX(vwaddu_vx_w, 8) +GEN_VEXT_VX(vwsubu_vx_b, 2) +GEN_VEXT_VX(vwsubu_vx_h, 4) +GEN_VEXT_VX(vwsubu_vx_w, 8) +GEN_VEXT_VX(vwadd_vx_b, 2) +GEN_VEXT_VX(vwadd_vx_h, 4) +GEN_VEXT_VX(vwadd_vx_w, 8) +GEN_VEXT_VX(vwsub_vx_b, 2) +GEN_VEXT_VX(vwsub_vx_h, 4) +GEN_VEXT_VX(vwsub_vx_w, 8) +GEN_VEXT_VX(vwaddu_wx_b, 2) +GEN_VEXT_VX(vwaddu_wx_h, 4) +GEN_VEXT_VX(vwaddu_wx_w, 8) +GEN_VEXT_VX(vwsubu_wx_b, 2) +GEN_VEXT_VX(vwsubu_wx_h, 4) +GEN_VEXT_VX(vwsubu_wx_w, 8) +GEN_VEXT_VX(vwadd_wx_b, 2) +GEN_VEXT_VX(vwadd_wx_h, 4) +GEN_VEXT_VX(vwadd_wx_w, 8) +GEN_VEXT_VX(vwsub_wx_b, 2) +GEN_VEXT_VX(vwsub_wx_h, 4) +GEN_VEXT_VX(vwsub_wx_w, 8) /* Vector Integer Add-with-Carry / Subtract-with-Borrow Instructions */ #define DO_VADC(N, M, C) (N + M + C) @@ -1065,6 +1074,10 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1075,6 +1088,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ *((ETYPE *)vd + H(i)) = DO_OP(s2, s1, carry); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VADC_VVM(vadc_vvm_b, uint8_t, H1, DO_VADC) @@ -1092,6 +1107,9 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1101,6 +1119,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ *((ETYPE *)vd + H(i)) = DO_OP(s2, (ETYPE)(target_long)s1, carry);\ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VADC_VXM(vadc_vxm_b, uint8_t, H1, DO_VADC) @@ -1123,6 +1143,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1132,6 +1154,13 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ vext_set_elem_mask(vd, i, DO_OP(s2, s1, carry)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_VMADC_VVM(vmadc_vvm_b, uint8_t, H1, DO_MADC) @@ -1150,6 +1179,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1159,6 +1190,13 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ DO_OP(s2, (ETYPE)(target_long)s1, carry)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_VMADC_VXM(vmadc_vxm_b, uint8_t, H1, DO_MADC) @@ -1209,18 +1247,18 @@ RVVCALL(OPIVX2, vxor_vx_b, OP_SSS_B, H1, H1, DO_XOR) RVVCALL(OPIVX2, vxor_vx_h, OP_SSS_H, H2, H2, DO_XOR) RVVCALL(OPIVX2, vxor_vx_w, OP_SSS_W, H4, H4, DO_XOR) RVVCALL(OPIVX2, vxor_vx_d, OP_SSS_D, H8, H8, DO_XOR) -GEN_VEXT_VX(vand_vx_b) -GEN_VEXT_VX(vand_vx_h) -GEN_VEXT_VX(vand_vx_w) -GEN_VEXT_VX(vand_vx_d) -GEN_VEXT_VX(vor_vx_b) -GEN_VEXT_VX(vor_vx_h) -GEN_VEXT_VX(vor_vx_w) -GEN_VEXT_VX(vor_vx_d) -GEN_VEXT_VX(vxor_vx_b) -GEN_VEXT_VX(vxor_vx_h) -GEN_VEXT_VX(vxor_vx_w) -GEN_VEXT_VX(vxor_vx_d) +GEN_VEXT_VX(vand_vx_b, 1) +GEN_VEXT_VX(vand_vx_h, 2) +GEN_VEXT_VX(vand_vx_w, 4) +GEN_VEXT_VX(vand_vx_d, 8) +GEN_VEXT_VX(vor_vx_b, 1) +GEN_VEXT_VX(vor_vx_h, 2) +GEN_VEXT_VX(vor_vx_w, 4) +GEN_VEXT_VX(vor_vx_d, 8) +GEN_VEXT_VX(vxor_vx_b, 1) +GEN_VEXT_VX(vxor_vx_h, 2) +GEN_VEXT_VX(vxor_vx_w, 4) +GEN_VEXT_VX(vxor_vx_d, 8) /* Vector Single-Width Bit Shift Instructions */ #define DO_SLL(N, M) (N << (M)) @@ -1474,22 +1512,22 @@ RVVCALL(OPIVX2, vmax_vx_b, OP_SSS_B, H1, H1, DO_MAX) RVVCALL(OPIVX2, vmax_vx_h, OP_SSS_H, H2, H2, DO_MAX) RVVCALL(OPIVX2, vmax_vx_w, OP_SSS_W, H4, H4, DO_MAX) RVVCALL(OPIVX2, vmax_vx_d, OP_SSS_D, H8, H8, DO_MAX) -GEN_VEXT_VX(vminu_vx_b) -GEN_VEXT_VX(vminu_vx_h) -GEN_VEXT_VX(vminu_vx_w) -GEN_VEXT_VX(vminu_vx_d) -GEN_VEXT_VX(vmin_vx_b) -GEN_VEXT_VX(vmin_vx_h) -GEN_VEXT_VX(vmin_vx_w) -GEN_VEXT_VX(vmin_vx_d) -GEN_VEXT_VX(vmaxu_vx_b) -GEN_VEXT_VX(vmaxu_vx_h) -GEN_VEXT_VX(vmaxu_vx_w) -GEN_VEXT_VX(vmaxu_vx_d) -GEN_VEXT_VX(vmax_vx_b) -GEN_VEXT_VX(vmax_vx_h) -GEN_VEXT_VX(vmax_vx_w) -GEN_VEXT_VX(vmax_vx_d) +GEN_VEXT_VX(vminu_vx_b, 1) +GEN_VEXT_VX(vminu_vx_h, 2) +GEN_VEXT_VX(vminu_vx_w, 4) +GEN_VEXT_VX(vminu_vx_d, 8) +GEN_VEXT_VX(vmin_vx_b, 1) +GEN_VEXT_VX(vmin_vx_h, 2) +GEN_VEXT_VX(vmin_vx_w, 4) +GEN_VEXT_VX(vmin_vx_d, 8) +GEN_VEXT_VX(vmaxu_vx_b, 1) +GEN_VEXT_VX(vmaxu_vx_h, 2) +GEN_VEXT_VX(vmaxu_vx_w, 4) +GEN_VEXT_VX(vmaxu_vx_d, 8) +GEN_VEXT_VX(vmax_vx_b, 1) +GEN_VEXT_VX(vmax_vx_h, 2) +GEN_VEXT_VX(vmax_vx_w, 4) +GEN_VEXT_VX(vmax_vx_d, 8) /* Vector Single-Width Integer Multiply Instructions */ #define DO_MUL(N, M) (N * M) @@ -1633,22 +1671,22 @@ RVVCALL(OPIVX2, vmulhsu_vx_b, OP_SUS_B, H1, H1, do_mulhsu_b) RVVCALL(OPIVX2, vmulhsu_vx_h, OP_SUS_H, H2, H2, do_mulhsu_h) RVVCALL(OPIVX2, vmulhsu_vx_w, OP_SUS_W, H4, H4, do_mulhsu_w) RVVCALL(OPIVX2, vmulhsu_vx_d, OP_SUS_D, H8, H8, do_mulhsu_d) -GEN_VEXT_VX(vmul_vx_b) -GEN_VEXT_VX(vmul_vx_h) -GEN_VEXT_VX(vmul_vx_w) -GEN_VEXT_VX(vmul_vx_d) -GEN_VEXT_VX(vmulh_vx_b) -GEN_VEXT_VX(vmulh_vx_h) -GEN_VEXT_VX(vmulh_vx_w) -GEN_VEXT_VX(vmulh_vx_d) -GEN_VEXT_VX(vmulhu_vx_b) -GEN_VEXT_VX(vmulhu_vx_h) -GEN_VEXT_VX(vmulhu_vx_w) -GEN_VEXT_VX(vmulhu_vx_d) -GEN_VEXT_VX(vmulhsu_vx_b) -GEN_VEXT_VX(vmulhsu_vx_h) -GEN_VEXT_VX(vmulhsu_vx_w) -GEN_VEXT_VX(vmulhsu_vx_d) +GEN_VEXT_VX(vmul_vx_b, 1) +GEN_VEXT_VX(vmul_vx_h, 2) +GEN_VEXT_VX(vmul_vx_w, 4) +GEN_VEXT_VX(vmul_vx_d, 8) +GEN_VEXT_VX(vmulh_vx_b, 1) +GEN_VEXT_VX(vmulh_vx_h, 2) +GEN_VEXT_VX(vmulh_vx_w, 4) +GEN_VEXT_VX(vmulh_vx_d, 8) +GEN_VEXT_VX(vmulhu_vx_b, 1) +GEN_VEXT_VX(vmulhu_vx_h, 2) +GEN_VEXT_VX(vmulhu_vx_w, 4) +GEN_VEXT_VX(vmulhu_vx_d, 8) +GEN_VEXT_VX(vmulhsu_vx_b, 1) +GEN_VEXT_VX(vmulhsu_vx_h, 2) +GEN_VEXT_VX(vmulhsu_vx_w, 4) +GEN_VEXT_VX(vmulhsu_vx_d, 8) /* Vector Integer Divide Instructions */ #define DO_DIVU(N, M) (unlikely(M == 0) ? (__typeof(N))(-1) : N / M) @@ -1707,22 +1745,22 @@ RVVCALL(OPIVX2, vrem_vx_b, OP_SSS_B, H1, H1, DO_REM) RVVCALL(OPIVX2, vrem_vx_h, OP_SSS_H, H2, H2, DO_REM) RVVCALL(OPIVX2, vrem_vx_w, OP_SSS_W, H4, H4, DO_REM) RVVCALL(OPIVX2, vrem_vx_d, OP_SSS_D, H8, H8, DO_REM) -GEN_VEXT_VX(vdivu_vx_b) -GEN_VEXT_VX(vdivu_vx_h) -GEN_VEXT_VX(vdivu_vx_w) -GEN_VEXT_VX(vdivu_vx_d) -GEN_VEXT_VX(vdiv_vx_b) -GEN_VEXT_VX(vdiv_vx_h) -GEN_VEXT_VX(vdiv_vx_w) -GEN_VEXT_VX(vdiv_vx_d) -GEN_VEXT_VX(vremu_vx_b) -GEN_VEXT_VX(vremu_vx_h) -GEN_VEXT_VX(vremu_vx_w) -GEN_VEXT_VX(vremu_vx_d) -GEN_VEXT_VX(vrem_vx_b) -GEN_VEXT_VX(vrem_vx_h) -GEN_VEXT_VX(vrem_vx_w) -GEN_VEXT_VX(vrem_vx_d) +GEN_VEXT_VX(vdivu_vx_b, 1) +GEN_VEXT_VX(vdivu_vx_h, 2) +GEN_VEXT_VX(vdivu_vx_w, 4) +GEN_VEXT_VX(vdivu_vx_d, 8) +GEN_VEXT_VX(vdiv_vx_b, 1) +GEN_VEXT_VX(vdiv_vx_h, 2) +GEN_VEXT_VX(vdiv_vx_w, 4) +GEN_VEXT_VX(vdiv_vx_d, 8) +GEN_VEXT_VX(vremu_vx_b, 1) +GEN_VEXT_VX(vremu_vx_h, 2) +GEN_VEXT_VX(vremu_vx_w, 4) +GEN_VEXT_VX(vremu_vx_d, 8) +GEN_VEXT_VX(vrem_vx_b, 1) +GEN_VEXT_VX(vrem_vx_h, 2) +GEN_VEXT_VX(vrem_vx_w, 4) +GEN_VEXT_VX(vrem_vx_d, 8) /* Vector Widening Integer Multiply Instructions */ RVVCALL(OPIVV2, vwmul_vv_b, WOP_SSS_B, H2, H1, H1, DO_MUL) @@ -1753,15 +1791,15 @@ RVVCALL(OPIVX2, vwmulu_vx_w, WOP_UUU_W, H8, H4, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_b, WOP_SUS_B, H2, H1, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_h, WOP_SUS_H, H4, H2, DO_MUL) RVVCALL(OPIVX2, vwmulsu_vx_w, WOP_SUS_W, H8, H4, DO_MUL) -GEN_VEXT_VX(vwmul_vx_b) -GEN_VEXT_VX(vwmul_vx_h) -GEN_VEXT_VX(vwmul_vx_w) -GEN_VEXT_VX(vwmulu_vx_b) -GEN_VEXT_VX(vwmulu_vx_h) -GEN_VEXT_VX(vwmulu_vx_w) -GEN_VEXT_VX(vwmulsu_vx_b) -GEN_VEXT_VX(vwmulsu_vx_h) -GEN_VEXT_VX(vwmulsu_vx_w) +GEN_VEXT_VX(vwmul_vx_b, 2) +GEN_VEXT_VX(vwmul_vx_h, 4) +GEN_VEXT_VX(vwmul_vx_w, 8) +GEN_VEXT_VX(vwmulu_vx_b, 2) +GEN_VEXT_VX(vwmulu_vx_h, 4) +GEN_VEXT_VX(vwmulu_vx_w, 8) +GEN_VEXT_VX(vwmulsu_vx_b, 2) +GEN_VEXT_VX(vwmulsu_vx_h, 4) +GEN_VEXT_VX(vwmulsu_vx_w, 8) /* Vector Single-Width Integer Multiply-Add Instructions */ #define OPIVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \ @@ -1834,22 +1872,22 @@ RVVCALL(OPIVX3, vnmsub_vx_b, OP_SSS_B, H1, H1, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_h, OP_SSS_H, H2, H2, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_w, OP_SSS_W, H4, H4, DO_NMSUB) RVVCALL(OPIVX3, vnmsub_vx_d, OP_SSS_D, H8, H8, DO_NMSUB) -GEN_VEXT_VX(vmacc_vx_b) -GEN_VEXT_VX(vmacc_vx_h) -GEN_VEXT_VX(vmacc_vx_w) -GEN_VEXT_VX(vmacc_vx_d) -GEN_VEXT_VX(vnmsac_vx_b) -GEN_VEXT_VX(vnmsac_vx_h) -GEN_VEXT_VX(vnmsac_vx_w) -GEN_VEXT_VX(vnmsac_vx_d) -GEN_VEXT_VX(vmadd_vx_b) -GEN_VEXT_VX(vmadd_vx_h) -GEN_VEXT_VX(vmadd_vx_w) -GEN_VEXT_VX(vmadd_vx_d) -GEN_VEXT_VX(vnmsub_vx_b) -GEN_VEXT_VX(vnmsub_vx_h) -GEN_VEXT_VX(vnmsub_vx_w) -GEN_VEXT_VX(vnmsub_vx_d) +GEN_VEXT_VX(vmacc_vx_b, 1) +GEN_VEXT_VX(vmacc_vx_h, 2) +GEN_VEXT_VX(vmacc_vx_w, 4) +GEN_VEXT_VX(vmacc_vx_d, 8) +GEN_VEXT_VX(vnmsac_vx_b, 1) +GEN_VEXT_VX(vnmsac_vx_h, 2) +GEN_VEXT_VX(vnmsac_vx_w, 4) +GEN_VEXT_VX(vnmsac_vx_d, 8) +GEN_VEXT_VX(vmadd_vx_b, 1) +GEN_VEXT_VX(vmadd_vx_h, 2) +GEN_VEXT_VX(vmadd_vx_w, 4) +GEN_VEXT_VX(vmadd_vx_d, 8) +GEN_VEXT_VX(vnmsub_vx_b, 1) +GEN_VEXT_VX(vnmsub_vx_h, 2) +GEN_VEXT_VX(vnmsub_vx_w, 4) +GEN_VEXT_VX(vnmsub_vx_d, 8) /* Vector Widening Integer Multiply-Add Instructions */ RVVCALL(OPIVV3, vwmaccu_vv_b, WOP_UUU_B, H2, H1, H1, DO_MACC) @@ -1883,18 +1921,18 @@ RVVCALL(OPIVX3, vwmaccsu_vx_w, WOP_SSU_W, H8, H4, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_b, WOP_SUS_B, H2, H1, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_h, WOP_SUS_H, H4, H2, DO_MACC) RVVCALL(OPIVX3, vwmaccus_vx_w, WOP_SUS_W, H8, H4, DO_MACC) -GEN_VEXT_VX(vwmaccu_vx_b) -GEN_VEXT_VX(vwmaccu_vx_h) -GEN_VEXT_VX(vwmaccu_vx_w) -GEN_VEXT_VX(vwmacc_vx_b) -GEN_VEXT_VX(vwmacc_vx_h) -GEN_VEXT_VX(vwmacc_vx_w) -GEN_VEXT_VX(vwmaccsu_vx_b) -GEN_VEXT_VX(vwmaccsu_vx_h) -GEN_VEXT_VX(vwmaccsu_vx_w) -GEN_VEXT_VX(vwmaccus_vx_b) -GEN_VEXT_VX(vwmaccus_vx_h) -GEN_VEXT_VX(vwmaccus_vx_w) +GEN_VEXT_VX(vwmaccu_vx_b, 2) +GEN_VEXT_VX(vwmaccu_vx_h, 4) +GEN_VEXT_VX(vwmaccu_vx_w, 8) +GEN_VEXT_VX(vwmacc_vx_b, 2) +GEN_VEXT_VX(vwmacc_vx_h, 4) +GEN_VEXT_VX(vwmacc_vx_w, 8) +GEN_VEXT_VX(vwmaccsu_vx_b, 2) +GEN_VEXT_VX(vwmaccsu_vx_h, 4) +GEN_VEXT_VX(vwmaccsu_vx_w, 8) +GEN_VEXT_VX(vwmaccus_vx_b, 2) +GEN_VEXT_VX(vwmaccus_vx_h, 4) +GEN_VEXT_VX(vwmaccus_vx_w, 8) /* Vector Integer Merge and Move Instructions */ #define GEN_VEXT_VMV_VV(NAME, ETYPE, H) \ From 7b1bff41c1524e5dc1c2815f68b0454656539993 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:33 +0000 Subject: [PATCH 76/90] target/riscv: rvv: Add tail agnostic for vector integer shift instructions Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-8@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 3 ++- target/riscv/vector_helper.c | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 22edf6228dfb..dbe687fb7397 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -1831,7 +1831,7 @@ do_opivx_gvec_shift(DisasContext *s, arg_rmrr *a, GVecGen2sFn32 *gvec_fn, return false; } - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { TCGv_i32 src1 = tcg_temp_new_i32(); tcg_gen_trunc_tl_i32(src1, get_gpr(s, a->rs1, EXT_NONE)); @@ -1890,6 +1890,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 7cdb5d12af29..056c2a1c7e2a 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -1271,6 +1271,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(TS1); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1282,6 +1285,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ *((TS1 *)vd + HS1(i)) = OP(s2, s1 & MASK); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_SHIFT_VV(vsll_vv_b, uint8_t, uint8_t, H1, H1, DO_SLL, 0x7) @@ -1306,6 +1311,10 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(TD); \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1316,6 +1325,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ *((TD *)vd + HD(i)) = OP(s2, s1 & MASK); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz);\ } GEN_VEXT_SHIFT_VX(vsll_vx_b, uint8_t, int8_t, H1, H1, DO_SLL, 0x7) From 38581e5c9a99d2ff9244709cb3577b06a1310006 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:51 +0000 Subject: [PATCH 77/90] target/riscv: rvv: Add tail agnostic for vector integer comparison instructions Compares write mask registers, and so always operate under a tail- agnostic policy. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-9@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 056c2a1c7e2a..a64506bf02f8 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -1371,6 +1371,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1382,6 +1384,13 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ vext_set_elem_mask(vd, i, DO_OP(s2, s1)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_CMP_VV(vmseq_vv_b, uint8_t, H1, DO_MSEQ) @@ -1420,6 +1429,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1431,6 +1442,13 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ DO_OP(s2, (ETYPE)(target_long)s1)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_CMP_VX(vmseq_vx_b, uint8_t, H1, DO_MSEQ) From 89a32de2d57591006d4a5d76e664b0b01c6998dc Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:51 +0000 Subject: [PATCH 78/90] target/riscv: rvv: Add tail agnostic for vector integer merge and move instructions Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-10@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 12 ++++++++---- target/riscv/vector_helper.c | 20 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index dbe687fb7397..e75a2fd1966f 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -2086,12 +2086,13 @@ static bool trans_vmv_v_v(DisasContext *s, arg_vmv_v_v *a) vext_check_isa_ill(s) && /* vmv.v.v has rs2 = 0 and vm = 1 */ vext_check_sss(s, a->rd, a->rs1, 0, 1)) { - if (s->vl_eq_vlmax) { + if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { tcg_gen_gvec_mov(s->sew, vreg_ofs(s, a->rd), vreg_ofs(s, a->rs1), MAXSZ(s), MAXSZ(s)); } else { uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_gvec_2_ptr * const fns[4] = { gen_helper_vmv_v_v_b, gen_helper_vmv_v_v_h, gen_helper_vmv_v_v_w, gen_helper_vmv_v_v_d, @@ -2126,7 +2127,7 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) s1 = get_gpr(s, a->rs1, EXT_SIGN); - if (s->vl_eq_vlmax) { + if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { tcg_gen_gvec_dup_tl(s->sew, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), s1); } else { @@ -2134,6 +2135,7 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) TCGv_i64 s1_i64 = tcg_temp_new_i64(); TCGv_ptr dest = tcg_temp_new_ptr(); uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_vmv_vx * const fns[4] = { gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h, gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d, @@ -2163,7 +2165,7 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) /* vmv.v.i has rs2 = 0 and vm = 1 */ vext_check_ss(s, a->rd, 0, 1)) { int64_t simm = sextract64(a->rs1, 0, 5); - if (s->vl_eq_vlmax) { + if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { tcg_gen_gvec_dup_imm(s->sew, vreg_ofs(s, a->rd), MAXSZ(s), MAXSZ(s), simm); mark_vs_dirty(s); @@ -2172,6 +2174,7 @@ static bool trans_vmv_v_i(DisasContext *s, arg_vmv_v_i *a) TCGv_i64 s1; TCGv_ptr dest; uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_vmv_vx * const fns[4] = { gen_helper_vmv_v_x_b, gen_helper_vmv_v_x_h, gen_helper_vmv_v_x_w, gen_helper_vmv_v_x_d, @@ -2743,7 +2746,7 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) TCGv_i64 t1; - if (s->vl_eq_vlmax) { + if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { t1 = tcg_temp_new_i64(); /* NaN-box f[rs1] */ do_nanbox(s, t1, cpu_fpr[a->rs1]); @@ -2755,6 +2758,7 @@ static bool trans_vfmv_v_f(DisasContext *s, arg_vfmv_v_f *a) TCGv_ptr dest; TCGv_i32 desc; uint32_t data = FIELD_DP32(0, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_vmv_vx * const fns[3] = { gen_helper_vmv_v_x_h, gen_helper_vmv_v_x_w, diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index a64506bf02f8..128238786e5f 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -1969,6 +1969,9 @@ void HELPER(NAME)(void *vd, void *vs1, CPURISCVState *env, \ uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -1976,6 +1979,8 @@ void HELPER(NAME)(void *vd, void *vs1, CPURISCVState *env, \ *((ETYPE *)vd + H(i)) = s1; \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VMV_VV(vmv_v_v_b, int8_t, H1) @@ -1988,12 +1993,17 @@ void HELPER(NAME)(void *vd, uint64_t s1, CPURISCVState *env, \ uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ *((ETYPE *)vd + H(i)) = (ETYPE)s1; \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VMV_VX(vmv_v_x_b, int8_t, H1) @@ -2006,6 +2016,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -2013,6 +2026,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ *((ETYPE *)vd + H(i)) = *(vt + H(i)); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VMERGE_VV(vmerge_vvm_b, int8_t, H1) @@ -2025,6 +2040,9 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ void *vs2, CPURISCVState *env, uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -2034,6 +2052,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ *((ETYPE *)vd + H(i)) = d; \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VMERGE_VX(vmerge_vxm_b, int8_t, H1) From 09106eed3041d5eb57dd768332146abe6d86e0e4 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:38 +0000 Subject: [PATCH 79/90] target/riscv: rvv: Add tail agnostic for vector fix-point arithmetic instructions Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-11@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 220 ++++++++++++++++++----------------- 1 file changed, 114 insertions(+), 106 deletions(-) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 128238786e5f..1fa93cf1f0be 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -2103,10 +2103,12 @@ static inline void vext_vv_rm_2(void *vd, void *v0, void *vs1, void *vs2, CPURISCVState *env, uint32_t desc, - opivv2_rm_fn *fn) + opivv2_rm_fn *fn, uint32_t esz) { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); switch (env->vxrm) { case 0: /* rnu */ @@ -2126,15 +2128,17 @@ vext_vv_rm_2(void *vd, void *v0, void *vs1, void *vs2, env, vl, vm, 3, fn); break; } + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); } /* generate helpers for fixed point instructions with OPIVV format */ -#define GEN_VEXT_VV_RM(NAME) \ +#define GEN_VEXT_VV_RM(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ vext_vv_rm_2(vd, v0, vs1, vs2, env, desc, \ - do_##NAME); \ + do_##NAME, ESZ); \ } static inline uint8_t saddu8(CPURISCVState *env, int vxrm, uint8_t a, uint8_t b) @@ -2184,10 +2188,10 @@ RVVCALL(OPIVV2_RM, vsaddu_vv_b, OP_UUU_B, H1, H1, H1, saddu8) RVVCALL(OPIVV2_RM, vsaddu_vv_h, OP_UUU_H, H2, H2, H2, saddu16) RVVCALL(OPIVV2_RM, vsaddu_vv_w, OP_UUU_W, H4, H4, H4, saddu32) RVVCALL(OPIVV2_RM, vsaddu_vv_d, OP_UUU_D, H8, H8, H8, saddu64) -GEN_VEXT_VV_RM(vsaddu_vv_b) -GEN_VEXT_VV_RM(vsaddu_vv_h) -GEN_VEXT_VV_RM(vsaddu_vv_w) -GEN_VEXT_VV_RM(vsaddu_vv_d) +GEN_VEXT_VV_RM(vsaddu_vv_b, 1) +GEN_VEXT_VV_RM(vsaddu_vv_h, 2) +GEN_VEXT_VV_RM(vsaddu_vv_w, 4) +GEN_VEXT_VV_RM(vsaddu_vv_d, 8) typedef void opivx2_rm_fn(void *vd, target_long s1, void *vs2, int i, CPURISCVState *env, int vxrm); @@ -2220,10 +2224,12 @@ static inline void vext_vx_rm_2(void *vd, void *v0, target_long s1, void *vs2, CPURISCVState *env, uint32_t desc, - opivx2_rm_fn *fn) + opivx2_rm_fn *fn, uint32_t esz) { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t total_elems = vext_get_total_elems(env, desc, esz); + uint32_t vta = vext_vta(desc); switch (env->vxrm) { case 0: /* rnu */ @@ -2243,25 +2249,27 @@ vext_vx_rm_2(void *vd, void *v0, target_long s1, void *vs2, env, vl, vm, 3, fn); break; } + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); } /* generate helpers for fixed point instructions with OPIVX format */ -#define GEN_VEXT_VX_RM(NAME) \ +#define GEN_VEXT_VX_RM(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, \ void *vs2, CPURISCVState *env, uint32_t desc) \ { \ vext_vx_rm_2(vd, v0, s1, vs2, env, desc, \ - do_##NAME); \ + do_##NAME, ESZ); \ } RVVCALL(OPIVX2_RM, vsaddu_vx_b, OP_UUU_B, H1, H1, saddu8) RVVCALL(OPIVX2_RM, vsaddu_vx_h, OP_UUU_H, H2, H2, saddu16) RVVCALL(OPIVX2_RM, vsaddu_vx_w, OP_UUU_W, H4, H4, saddu32) RVVCALL(OPIVX2_RM, vsaddu_vx_d, OP_UUU_D, H8, H8, saddu64) -GEN_VEXT_VX_RM(vsaddu_vx_b) -GEN_VEXT_VX_RM(vsaddu_vx_h) -GEN_VEXT_VX_RM(vsaddu_vx_w) -GEN_VEXT_VX_RM(vsaddu_vx_d) +GEN_VEXT_VX_RM(vsaddu_vx_b, 1) +GEN_VEXT_VX_RM(vsaddu_vx_h, 2) +GEN_VEXT_VX_RM(vsaddu_vx_w, 4) +GEN_VEXT_VX_RM(vsaddu_vx_d, 8) static inline int8_t sadd8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) { @@ -2307,19 +2315,19 @@ RVVCALL(OPIVV2_RM, vsadd_vv_b, OP_SSS_B, H1, H1, H1, sadd8) RVVCALL(OPIVV2_RM, vsadd_vv_h, OP_SSS_H, H2, H2, H2, sadd16) RVVCALL(OPIVV2_RM, vsadd_vv_w, OP_SSS_W, H4, H4, H4, sadd32) RVVCALL(OPIVV2_RM, vsadd_vv_d, OP_SSS_D, H8, H8, H8, sadd64) -GEN_VEXT_VV_RM(vsadd_vv_b) -GEN_VEXT_VV_RM(vsadd_vv_h) -GEN_VEXT_VV_RM(vsadd_vv_w) -GEN_VEXT_VV_RM(vsadd_vv_d) +GEN_VEXT_VV_RM(vsadd_vv_b, 1) +GEN_VEXT_VV_RM(vsadd_vv_h, 2) +GEN_VEXT_VV_RM(vsadd_vv_w, 4) +GEN_VEXT_VV_RM(vsadd_vv_d, 8) RVVCALL(OPIVX2_RM, vsadd_vx_b, OP_SSS_B, H1, H1, sadd8) RVVCALL(OPIVX2_RM, vsadd_vx_h, OP_SSS_H, H2, H2, sadd16) RVVCALL(OPIVX2_RM, vsadd_vx_w, OP_SSS_W, H4, H4, sadd32) RVVCALL(OPIVX2_RM, vsadd_vx_d, OP_SSS_D, H8, H8, sadd64) -GEN_VEXT_VX_RM(vsadd_vx_b) -GEN_VEXT_VX_RM(vsadd_vx_h) -GEN_VEXT_VX_RM(vsadd_vx_w) -GEN_VEXT_VX_RM(vsadd_vx_d) +GEN_VEXT_VX_RM(vsadd_vx_b, 1) +GEN_VEXT_VX_RM(vsadd_vx_h, 2) +GEN_VEXT_VX_RM(vsadd_vx_w, 4) +GEN_VEXT_VX_RM(vsadd_vx_d, 8) static inline uint8_t ssubu8(CPURISCVState *env, int vxrm, uint8_t a, uint8_t b) { @@ -2368,19 +2376,19 @@ RVVCALL(OPIVV2_RM, vssubu_vv_b, OP_UUU_B, H1, H1, H1, ssubu8) RVVCALL(OPIVV2_RM, vssubu_vv_h, OP_UUU_H, H2, H2, H2, ssubu16) RVVCALL(OPIVV2_RM, vssubu_vv_w, OP_UUU_W, H4, H4, H4, ssubu32) RVVCALL(OPIVV2_RM, vssubu_vv_d, OP_UUU_D, H8, H8, H8, ssubu64) -GEN_VEXT_VV_RM(vssubu_vv_b) -GEN_VEXT_VV_RM(vssubu_vv_h) -GEN_VEXT_VV_RM(vssubu_vv_w) -GEN_VEXT_VV_RM(vssubu_vv_d) +GEN_VEXT_VV_RM(vssubu_vv_b, 1) +GEN_VEXT_VV_RM(vssubu_vv_h, 2) +GEN_VEXT_VV_RM(vssubu_vv_w, 4) +GEN_VEXT_VV_RM(vssubu_vv_d, 8) RVVCALL(OPIVX2_RM, vssubu_vx_b, OP_UUU_B, H1, H1, ssubu8) RVVCALL(OPIVX2_RM, vssubu_vx_h, OP_UUU_H, H2, H2, ssubu16) RVVCALL(OPIVX2_RM, vssubu_vx_w, OP_UUU_W, H4, H4, ssubu32) RVVCALL(OPIVX2_RM, vssubu_vx_d, OP_UUU_D, H8, H8, ssubu64) -GEN_VEXT_VX_RM(vssubu_vx_b) -GEN_VEXT_VX_RM(vssubu_vx_h) -GEN_VEXT_VX_RM(vssubu_vx_w) -GEN_VEXT_VX_RM(vssubu_vx_d) +GEN_VEXT_VX_RM(vssubu_vx_b, 1) +GEN_VEXT_VX_RM(vssubu_vx_h, 2) +GEN_VEXT_VX_RM(vssubu_vx_w, 4) +GEN_VEXT_VX_RM(vssubu_vx_d, 8) static inline int8_t ssub8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) { @@ -2426,19 +2434,19 @@ RVVCALL(OPIVV2_RM, vssub_vv_b, OP_SSS_B, H1, H1, H1, ssub8) RVVCALL(OPIVV2_RM, vssub_vv_h, OP_SSS_H, H2, H2, H2, ssub16) RVVCALL(OPIVV2_RM, vssub_vv_w, OP_SSS_W, H4, H4, H4, ssub32) RVVCALL(OPIVV2_RM, vssub_vv_d, OP_SSS_D, H8, H8, H8, ssub64) -GEN_VEXT_VV_RM(vssub_vv_b) -GEN_VEXT_VV_RM(vssub_vv_h) -GEN_VEXT_VV_RM(vssub_vv_w) -GEN_VEXT_VV_RM(vssub_vv_d) +GEN_VEXT_VV_RM(vssub_vv_b, 1) +GEN_VEXT_VV_RM(vssub_vv_h, 2) +GEN_VEXT_VV_RM(vssub_vv_w, 4) +GEN_VEXT_VV_RM(vssub_vv_d, 8) RVVCALL(OPIVX2_RM, vssub_vx_b, OP_SSS_B, H1, H1, ssub8) RVVCALL(OPIVX2_RM, vssub_vx_h, OP_SSS_H, H2, H2, ssub16) RVVCALL(OPIVX2_RM, vssub_vx_w, OP_SSS_W, H4, H4, ssub32) RVVCALL(OPIVX2_RM, vssub_vx_d, OP_SSS_D, H8, H8, ssub64) -GEN_VEXT_VX_RM(vssub_vx_b) -GEN_VEXT_VX_RM(vssub_vx_h) -GEN_VEXT_VX_RM(vssub_vx_w) -GEN_VEXT_VX_RM(vssub_vx_d) +GEN_VEXT_VX_RM(vssub_vx_b, 1) +GEN_VEXT_VX_RM(vssub_vx_h, 2) +GEN_VEXT_VX_RM(vssub_vx_w, 4) +GEN_VEXT_VX_RM(vssub_vx_d, 8) /* Vector Single-Width Averaging Add and Subtract */ static inline uint8_t get_round(int vxrm, uint64_t v, uint8_t shift) @@ -2490,19 +2498,19 @@ RVVCALL(OPIVV2_RM, vaadd_vv_b, OP_SSS_B, H1, H1, H1, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_h, OP_SSS_H, H2, H2, H2, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_w, OP_SSS_W, H4, H4, H4, aadd32) RVVCALL(OPIVV2_RM, vaadd_vv_d, OP_SSS_D, H8, H8, H8, aadd64) -GEN_VEXT_VV_RM(vaadd_vv_b) -GEN_VEXT_VV_RM(vaadd_vv_h) -GEN_VEXT_VV_RM(vaadd_vv_w) -GEN_VEXT_VV_RM(vaadd_vv_d) +GEN_VEXT_VV_RM(vaadd_vv_b, 1) +GEN_VEXT_VV_RM(vaadd_vv_h, 2) +GEN_VEXT_VV_RM(vaadd_vv_w, 4) +GEN_VEXT_VV_RM(vaadd_vv_d, 8) RVVCALL(OPIVX2_RM, vaadd_vx_b, OP_SSS_B, H1, H1, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_h, OP_SSS_H, H2, H2, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_w, OP_SSS_W, H4, H4, aadd32) RVVCALL(OPIVX2_RM, vaadd_vx_d, OP_SSS_D, H8, H8, aadd64) -GEN_VEXT_VX_RM(vaadd_vx_b) -GEN_VEXT_VX_RM(vaadd_vx_h) -GEN_VEXT_VX_RM(vaadd_vx_w) -GEN_VEXT_VX_RM(vaadd_vx_d) +GEN_VEXT_VX_RM(vaadd_vx_b, 1) +GEN_VEXT_VX_RM(vaadd_vx_h, 2) +GEN_VEXT_VX_RM(vaadd_vx_w, 4) +GEN_VEXT_VX_RM(vaadd_vx_d, 8) static inline uint32_t aaddu32(CPURISCVState *env, int vxrm, uint32_t a, uint32_t b) @@ -2527,19 +2535,19 @@ RVVCALL(OPIVV2_RM, vaaddu_vv_b, OP_UUU_B, H1, H1, H1, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_h, OP_UUU_H, H2, H2, H2, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_w, OP_UUU_W, H4, H4, H4, aaddu32) RVVCALL(OPIVV2_RM, vaaddu_vv_d, OP_UUU_D, H8, H8, H8, aaddu64) -GEN_VEXT_VV_RM(vaaddu_vv_b) -GEN_VEXT_VV_RM(vaaddu_vv_h) -GEN_VEXT_VV_RM(vaaddu_vv_w) -GEN_VEXT_VV_RM(vaaddu_vv_d) +GEN_VEXT_VV_RM(vaaddu_vv_b, 1) +GEN_VEXT_VV_RM(vaaddu_vv_h, 2) +GEN_VEXT_VV_RM(vaaddu_vv_w, 4) +GEN_VEXT_VV_RM(vaaddu_vv_d, 8) RVVCALL(OPIVX2_RM, vaaddu_vx_b, OP_UUU_B, H1, H1, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_h, OP_UUU_H, H2, H2, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_w, OP_UUU_W, H4, H4, aaddu32) RVVCALL(OPIVX2_RM, vaaddu_vx_d, OP_UUU_D, H8, H8, aaddu64) -GEN_VEXT_VX_RM(vaaddu_vx_b) -GEN_VEXT_VX_RM(vaaddu_vx_h) -GEN_VEXT_VX_RM(vaaddu_vx_w) -GEN_VEXT_VX_RM(vaaddu_vx_d) +GEN_VEXT_VX_RM(vaaddu_vx_b, 1) +GEN_VEXT_VX_RM(vaaddu_vx_h, 2) +GEN_VEXT_VX_RM(vaaddu_vx_w, 4) +GEN_VEXT_VX_RM(vaaddu_vx_d, 8) static inline int32_t asub32(CPURISCVState *env, int vxrm, int32_t a, int32_t b) { @@ -2563,19 +2571,19 @@ RVVCALL(OPIVV2_RM, vasub_vv_b, OP_SSS_B, H1, H1, H1, asub32) RVVCALL(OPIVV2_RM, vasub_vv_h, OP_SSS_H, H2, H2, H2, asub32) RVVCALL(OPIVV2_RM, vasub_vv_w, OP_SSS_W, H4, H4, H4, asub32) RVVCALL(OPIVV2_RM, vasub_vv_d, OP_SSS_D, H8, H8, H8, asub64) -GEN_VEXT_VV_RM(vasub_vv_b) -GEN_VEXT_VV_RM(vasub_vv_h) -GEN_VEXT_VV_RM(vasub_vv_w) -GEN_VEXT_VV_RM(vasub_vv_d) +GEN_VEXT_VV_RM(vasub_vv_b, 1) +GEN_VEXT_VV_RM(vasub_vv_h, 2) +GEN_VEXT_VV_RM(vasub_vv_w, 4) +GEN_VEXT_VV_RM(vasub_vv_d, 8) RVVCALL(OPIVX2_RM, vasub_vx_b, OP_SSS_B, H1, H1, asub32) RVVCALL(OPIVX2_RM, vasub_vx_h, OP_SSS_H, H2, H2, asub32) RVVCALL(OPIVX2_RM, vasub_vx_w, OP_SSS_W, H4, H4, asub32) RVVCALL(OPIVX2_RM, vasub_vx_d, OP_SSS_D, H8, H8, asub64) -GEN_VEXT_VX_RM(vasub_vx_b) -GEN_VEXT_VX_RM(vasub_vx_h) -GEN_VEXT_VX_RM(vasub_vx_w) -GEN_VEXT_VX_RM(vasub_vx_d) +GEN_VEXT_VX_RM(vasub_vx_b, 1) +GEN_VEXT_VX_RM(vasub_vx_h, 2) +GEN_VEXT_VX_RM(vasub_vx_w, 4) +GEN_VEXT_VX_RM(vasub_vx_d, 8) static inline uint32_t asubu32(CPURISCVState *env, int vxrm, uint32_t a, uint32_t b) @@ -2600,19 +2608,19 @@ RVVCALL(OPIVV2_RM, vasubu_vv_b, OP_UUU_B, H1, H1, H1, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_h, OP_UUU_H, H2, H2, H2, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_w, OP_UUU_W, H4, H4, H4, asubu32) RVVCALL(OPIVV2_RM, vasubu_vv_d, OP_UUU_D, H8, H8, H8, asubu64) -GEN_VEXT_VV_RM(vasubu_vv_b) -GEN_VEXT_VV_RM(vasubu_vv_h) -GEN_VEXT_VV_RM(vasubu_vv_w) -GEN_VEXT_VV_RM(vasubu_vv_d) +GEN_VEXT_VV_RM(vasubu_vv_b, 1) +GEN_VEXT_VV_RM(vasubu_vv_h, 2) +GEN_VEXT_VV_RM(vasubu_vv_w, 4) +GEN_VEXT_VV_RM(vasubu_vv_d, 8) RVVCALL(OPIVX2_RM, vasubu_vx_b, OP_UUU_B, H1, H1, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_h, OP_UUU_H, H2, H2, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_w, OP_UUU_W, H4, H4, asubu32) RVVCALL(OPIVX2_RM, vasubu_vx_d, OP_UUU_D, H8, H8, asubu64) -GEN_VEXT_VX_RM(vasubu_vx_b) -GEN_VEXT_VX_RM(vasubu_vx_h) -GEN_VEXT_VX_RM(vasubu_vx_w) -GEN_VEXT_VX_RM(vasubu_vx_d) +GEN_VEXT_VX_RM(vasubu_vx_b, 1) +GEN_VEXT_VX_RM(vasubu_vx_h, 2) +GEN_VEXT_VX_RM(vasubu_vx_w, 4) +GEN_VEXT_VX_RM(vasubu_vx_d, 8) /* Vector Single-Width Fractional Multiply with Rounding and Saturation */ static inline int8_t vsmul8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) @@ -2707,19 +2715,19 @@ RVVCALL(OPIVV2_RM, vsmul_vv_b, OP_SSS_B, H1, H1, H1, vsmul8) RVVCALL(OPIVV2_RM, vsmul_vv_h, OP_SSS_H, H2, H2, H2, vsmul16) RVVCALL(OPIVV2_RM, vsmul_vv_w, OP_SSS_W, H4, H4, H4, vsmul32) RVVCALL(OPIVV2_RM, vsmul_vv_d, OP_SSS_D, H8, H8, H8, vsmul64) -GEN_VEXT_VV_RM(vsmul_vv_b) -GEN_VEXT_VV_RM(vsmul_vv_h) -GEN_VEXT_VV_RM(vsmul_vv_w) -GEN_VEXT_VV_RM(vsmul_vv_d) +GEN_VEXT_VV_RM(vsmul_vv_b, 1) +GEN_VEXT_VV_RM(vsmul_vv_h, 2) +GEN_VEXT_VV_RM(vsmul_vv_w, 4) +GEN_VEXT_VV_RM(vsmul_vv_d, 8) RVVCALL(OPIVX2_RM, vsmul_vx_b, OP_SSS_B, H1, H1, vsmul8) RVVCALL(OPIVX2_RM, vsmul_vx_h, OP_SSS_H, H2, H2, vsmul16) RVVCALL(OPIVX2_RM, vsmul_vx_w, OP_SSS_W, H4, H4, vsmul32) RVVCALL(OPIVX2_RM, vsmul_vx_d, OP_SSS_D, H8, H8, vsmul64) -GEN_VEXT_VX_RM(vsmul_vx_b) -GEN_VEXT_VX_RM(vsmul_vx_h) -GEN_VEXT_VX_RM(vsmul_vx_w) -GEN_VEXT_VX_RM(vsmul_vx_d) +GEN_VEXT_VX_RM(vsmul_vx_b, 1) +GEN_VEXT_VX_RM(vsmul_vx_h, 2) +GEN_VEXT_VX_RM(vsmul_vx_w, 4) +GEN_VEXT_VX_RM(vsmul_vx_d, 8) /* Vector Single-Width Scaling Shift Instructions */ static inline uint8_t @@ -2766,19 +2774,19 @@ RVVCALL(OPIVV2_RM, vssrl_vv_b, OP_UUU_B, H1, H1, H1, vssrl8) RVVCALL(OPIVV2_RM, vssrl_vv_h, OP_UUU_H, H2, H2, H2, vssrl16) RVVCALL(OPIVV2_RM, vssrl_vv_w, OP_UUU_W, H4, H4, H4, vssrl32) RVVCALL(OPIVV2_RM, vssrl_vv_d, OP_UUU_D, H8, H8, H8, vssrl64) -GEN_VEXT_VV_RM(vssrl_vv_b) -GEN_VEXT_VV_RM(vssrl_vv_h) -GEN_VEXT_VV_RM(vssrl_vv_w) -GEN_VEXT_VV_RM(vssrl_vv_d) +GEN_VEXT_VV_RM(vssrl_vv_b, 1) +GEN_VEXT_VV_RM(vssrl_vv_h, 2) +GEN_VEXT_VV_RM(vssrl_vv_w, 4) +GEN_VEXT_VV_RM(vssrl_vv_d, 8) RVVCALL(OPIVX2_RM, vssrl_vx_b, OP_UUU_B, H1, H1, vssrl8) RVVCALL(OPIVX2_RM, vssrl_vx_h, OP_UUU_H, H2, H2, vssrl16) RVVCALL(OPIVX2_RM, vssrl_vx_w, OP_UUU_W, H4, H4, vssrl32) RVVCALL(OPIVX2_RM, vssrl_vx_d, OP_UUU_D, H8, H8, vssrl64) -GEN_VEXT_VX_RM(vssrl_vx_b) -GEN_VEXT_VX_RM(vssrl_vx_h) -GEN_VEXT_VX_RM(vssrl_vx_w) -GEN_VEXT_VX_RM(vssrl_vx_d) +GEN_VEXT_VX_RM(vssrl_vx_b, 1) +GEN_VEXT_VX_RM(vssrl_vx_h, 2) +GEN_VEXT_VX_RM(vssrl_vx_w, 4) +GEN_VEXT_VX_RM(vssrl_vx_d, 8) static inline int8_t vssra8(CPURISCVState *env, int vxrm, int8_t a, int8_t b) @@ -2825,19 +2833,19 @@ RVVCALL(OPIVV2_RM, vssra_vv_b, OP_SSS_B, H1, H1, H1, vssra8) RVVCALL(OPIVV2_RM, vssra_vv_h, OP_SSS_H, H2, H2, H2, vssra16) RVVCALL(OPIVV2_RM, vssra_vv_w, OP_SSS_W, H4, H4, H4, vssra32) RVVCALL(OPIVV2_RM, vssra_vv_d, OP_SSS_D, H8, H8, H8, vssra64) -GEN_VEXT_VV_RM(vssra_vv_b) -GEN_VEXT_VV_RM(vssra_vv_h) -GEN_VEXT_VV_RM(vssra_vv_w) -GEN_VEXT_VV_RM(vssra_vv_d) +GEN_VEXT_VV_RM(vssra_vv_b, 1) +GEN_VEXT_VV_RM(vssra_vv_h, 2) +GEN_VEXT_VV_RM(vssra_vv_w, 4) +GEN_VEXT_VV_RM(vssra_vv_d, 8) RVVCALL(OPIVX2_RM, vssra_vx_b, OP_SSS_B, H1, H1, vssra8) RVVCALL(OPIVX2_RM, vssra_vx_h, OP_SSS_H, H2, H2, vssra16) RVVCALL(OPIVX2_RM, vssra_vx_w, OP_SSS_W, H4, H4, vssra32) RVVCALL(OPIVX2_RM, vssra_vx_d, OP_SSS_D, H8, H8, vssra64) -GEN_VEXT_VX_RM(vssra_vx_b) -GEN_VEXT_VX_RM(vssra_vx_h) -GEN_VEXT_VX_RM(vssra_vx_w) -GEN_VEXT_VX_RM(vssra_vx_d) +GEN_VEXT_VX_RM(vssra_vx_b, 1) +GEN_VEXT_VX_RM(vssra_vx_h, 2) +GEN_VEXT_VX_RM(vssra_vx_w, 4) +GEN_VEXT_VX_RM(vssra_vx_d, 8) /* Vector Narrowing Fixed-Point Clip Instructions */ static inline int8_t @@ -2900,16 +2908,16 @@ vnclip32(CPURISCVState *env, int vxrm, int64_t a, int32_t b) RVVCALL(OPIVV2_RM, vnclip_wv_b, NOP_SSS_B, H1, H2, H1, vnclip8) RVVCALL(OPIVV2_RM, vnclip_wv_h, NOP_SSS_H, H2, H4, H2, vnclip16) RVVCALL(OPIVV2_RM, vnclip_wv_w, NOP_SSS_W, H4, H8, H4, vnclip32) -GEN_VEXT_VV_RM(vnclip_wv_b) -GEN_VEXT_VV_RM(vnclip_wv_h) -GEN_VEXT_VV_RM(vnclip_wv_w) +GEN_VEXT_VV_RM(vnclip_wv_b, 1) +GEN_VEXT_VV_RM(vnclip_wv_h, 2) +GEN_VEXT_VV_RM(vnclip_wv_w, 4) RVVCALL(OPIVX2_RM, vnclip_wx_b, NOP_SSS_B, H1, H2, vnclip8) RVVCALL(OPIVX2_RM, vnclip_wx_h, NOP_SSS_H, H2, H4, vnclip16) RVVCALL(OPIVX2_RM, vnclip_wx_w, NOP_SSS_W, H4, H8, vnclip32) -GEN_VEXT_VX_RM(vnclip_wx_b) -GEN_VEXT_VX_RM(vnclip_wx_h) -GEN_VEXT_VX_RM(vnclip_wx_w) +GEN_VEXT_VX_RM(vnclip_wx_b, 1) +GEN_VEXT_VX_RM(vnclip_wx_h, 2) +GEN_VEXT_VX_RM(vnclip_wx_w, 4) static inline uint8_t vnclipu8(CPURISCVState *env, int vxrm, uint16_t a, uint8_t b) @@ -2962,16 +2970,16 @@ vnclipu32(CPURISCVState *env, int vxrm, uint64_t a, uint32_t b) RVVCALL(OPIVV2_RM, vnclipu_wv_b, NOP_UUU_B, H1, H2, H1, vnclipu8) RVVCALL(OPIVV2_RM, vnclipu_wv_h, NOP_UUU_H, H2, H4, H2, vnclipu16) RVVCALL(OPIVV2_RM, vnclipu_wv_w, NOP_UUU_W, H4, H8, H4, vnclipu32) -GEN_VEXT_VV_RM(vnclipu_wv_b) -GEN_VEXT_VV_RM(vnclipu_wv_h) -GEN_VEXT_VV_RM(vnclipu_wv_w) +GEN_VEXT_VV_RM(vnclipu_wv_b, 1) +GEN_VEXT_VV_RM(vnclipu_wv_h, 2) +GEN_VEXT_VV_RM(vnclipu_wv_w, 4) RVVCALL(OPIVX2_RM, vnclipu_wx_b, NOP_UUU_B, H1, H2, vnclipu8) RVVCALL(OPIVX2_RM, vnclipu_wx_h, NOP_UUU_H, H2, H4, vnclipu16) RVVCALL(OPIVX2_RM, vnclipu_wx_w, NOP_UUU_W, H4, H8, vnclipu32) -GEN_VEXT_VX_RM(vnclipu_wx_b) -GEN_VEXT_VX_RM(vnclipu_wx_h) -GEN_VEXT_VX_RM(vnclipu_wx_w) +GEN_VEXT_VX_RM(vnclipu_wx_b, 1) +GEN_VEXT_VX_RM(vnclipu_wx_h, 2) +GEN_VEXT_VX_RM(vnclipu_wx_w, 4) /* *** Vector Float Point Arithmetic Instructions From 5eacf7d8a0fb19ea4d87eb678462fdb9a29b9190 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:56 +0000 Subject: [PATCH 80/90] target/riscv: rvv: Add tail agnostic for vector floating-point instructions Compares write mask registers, and so always operate under a tail- agnostic policy. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-12@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 17 + target/riscv/vector_helper.c | 440 +++++++++++++----------- 2 files changed, 261 insertions(+), 196 deletions(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index e75a2fd1966f..1add4cb6553a 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -2338,6 +2338,9 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ + data = \ + FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ @@ -2420,6 +2423,9 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ gen_set_rm(s, RISCV_FRM_DYN); \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ + data = FIELD_DP32(data, VDATA, VTA_ALL_1S, \ + s->cfg_vta_all_1s); \ return opfvf_trans(a->rd, a->rs1, a->rs2, data, \ fns[s->sew - 1], s); \ } \ @@ -2458,6 +2464,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ @@ -2497,6 +2504,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ gen_set_rm(s, RISCV_FRM_DYN); \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ return opfvf_trans(a->rd, a->rs1, a->rs2, data, \ fns[s->sew - 1], s); \ } \ @@ -2533,6 +2541,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ @@ -2572,6 +2581,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmrr *a) \ gen_set_rm(s, RISCV_FRM_DYN); \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ return opfvf_trans(a->rd, a->rs1, a->rs2, data, \ fns[s->sew - 1], s); \ } \ @@ -2655,6 +2665,7 @@ static bool do_opfv(DisasContext *s, arg_rmr *a, data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs2), cpu_env, s->cfg_ptr->vlen / 8, @@ -2859,6 +2870,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), cpu_env, \ s->cfg_ptr->vlen / 8, \ @@ -2910,6 +2922,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ + data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), cpu_env, \ s->cfg_ptr->vlen / 8, \ @@ -2977,6 +2991,7 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), cpu_env, \ s->cfg_ptr->vlen / 8, \ @@ -3030,6 +3045,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ + data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = FIELD_DP32(data, VDATA, VTA, s->vta); \ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs2), cpu_env, \ s->cfg_ptr->vlen / 8, \ diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 1fa93cf1f0be..17390d8d06bc 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -2994,13 +2994,16 @@ static void do_##NAME(void *vd, void *vs1, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, s1, &env->fp_status); \ } -#define GEN_VEXT_VV_ENV(NAME) \ +#define GEN_VEXT_VV_ENV(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, ESZ); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -3010,14 +3013,17 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ do_##NAME(vd, vs1, vs2, i, env); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * ESZ, \ + total_elems * ESZ); \ } RVVCALL(OPFVV2, vfadd_vv_h, OP_UUU_H, H2, H2, H2, float16_add) RVVCALL(OPFVV2, vfadd_vv_w, OP_UUU_W, H4, H4, H4, float32_add) RVVCALL(OPFVV2, vfadd_vv_d, OP_UUU_D, H8, H8, H8, float64_add) -GEN_VEXT_VV_ENV(vfadd_vv_h) -GEN_VEXT_VV_ENV(vfadd_vv_w) -GEN_VEXT_VV_ENV(vfadd_vv_d) +GEN_VEXT_VV_ENV(vfadd_vv_h, 2) +GEN_VEXT_VV_ENV(vfadd_vv_w, 4) +GEN_VEXT_VV_ENV(vfadd_vv_d, 8) #define OPFVF2(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ @@ -3027,13 +3033,16 @@ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, (TX1)(T1)s1, &env->fp_status);\ } -#define GEN_VEXT_VF(NAME) \ +#define GEN_VEXT_VF(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, \ void *vs2, CPURISCVState *env, \ uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, ESZ); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -3043,27 +3052,30 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, \ do_##NAME(vd, s1, vs2, i, env); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * ESZ, \ + total_elems * ESZ); \ } RVVCALL(OPFVF2, vfadd_vf_h, OP_UUU_H, H2, H2, float16_add) RVVCALL(OPFVF2, vfadd_vf_w, OP_UUU_W, H4, H4, float32_add) RVVCALL(OPFVF2, vfadd_vf_d, OP_UUU_D, H8, H8, float64_add) -GEN_VEXT_VF(vfadd_vf_h) -GEN_VEXT_VF(vfadd_vf_w) -GEN_VEXT_VF(vfadd_vf_d) +GEN_VEXT_VF(vfadd_vf_h, 2) +GEN_VEXT_VF(vfadd_vf_w, 4) +GEN_VEXT_VF(vfadd_vf_d, 8) RVVCALL(OPFVV2, vfsub_vv_h, OP_UUU_H, H2, H2, H2, float16_sub) RVVCALL(OPFVV2, vfsub_vv_w, OP_UUU_W, H4, H4, H4, float32_sub) RVVCALL(OPFVV2, vfsub_vv_d, OP_UUU_D, H8, H8, H8, float64_sub) -GEN_VEXT_VV_ENV(vfsub_vv_h) -GEN_VEXT_VV_ENV(vfsub_vv_w) -GEN_VEXT_VV_ENV(vfsub_vv_d) +GEN_VEXT_VV_ENV(vfsub_vv_h, 2) +GEN_VEXT_VV_ENV(vfsub_vv_w, 4) +GEN_VEXT_VV_ENV(vfsub_vv_d, 8) RVVCALL(OPFVF2, vfsub_vf_h, OP_UUU_H, H2, H2, float16_sub) RVVCALL(OPFVF2, vfsub_vf_w, OP_UUU_W, H4, H4, float32_sub) RVVCALL(OPFVF2, vfsub_vf_d, OP_UUU_D, H8, H8, float64_sub) -GEN_VEXT_VF(vfsub_vf_h) -GEN_VEXT_VF(vfsub_vf_w) -GEN_VEXT_VF(vfsub_vf_d) +GEN_VEXT_VF(vfsub_vf_h, 2) +GEN_VEXT_VF(vfsub_vf_w, 4) +GEN_VEXT_VF(vfsub_vf_d, 8) static uint16_t float16_rsub(uint16_t a, uint16_t b, float_status *s) { @@ -3083,9 +3095,9 @@ static uint64_t float64_rsub(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVF2, vfrsub_vf_h, OP_UUU_H, H2, H2, float16_rsub) RVVCALL(OPFVF2, vfrsub_vf_w, OP_UUU_W, H4, H4, float32_rsub) RVVCALL(OPFVF2, vfrsub_vf_d, OP_UUU_D, H8, H8, float64_rsub) -GEN_VEXT_VF(vfrsub_vf_h) -GEN_VEXT_VF(vfrsub_vf_w) -GEN_VEXT_VF(vfrsub_vf_d) +GEN_VEXT_VF(vfrsub_vf_h, 2) +GEN_VEXT_VF(vfrsub_vf_w, 4) +GEN_VEXT_VF(vfrsub_vf_d, 8) /* Vector Widening Floating-Point Add/Subtract Instructions */ static uint32_t vfwadd16(uint16_t a, uint16_t b, float_status *s) @@ -3103,12 +3115,12 @@ static uint64_t vfwadd32(uint32_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwadd_vv_h, WOP_UUU_H, H4, H2, H2, vfwadd16) RVVCALL(OPFVV2, vfwadd_vv_w, WOP_UUU_W, H8, H4, H4, vfwadd32) -GEN_VEXT_VV_ENV(vfwadd_vv_h) -GEN_VEXT_VV_ENV(vfwadd_vv_w) +GEN_VEXT_VV_ENV(vfwadd_vv_h, 4) +GEN_VEXT_VV_ENV(vfwadd_vv_w, 8) RVVCALL(OPFVF2, vfwadd_vf_h, WOP_UUU_H, H4, H2, vfwadd16) RVVCALL(OPFVF2, vfwadd_vf_w, WOP_UUU_W, H8, H4, vfwadd32) -GEN_VEXT_VF(vfwadd_vf_h) -GEN_VEXT_VF(vfwadd_vf_w) +GEN_VEXT_VF(vfwadd_vf_h, 4) +GEN_VEXT_VF(vfwadd_vf_w, 8) static uint32_t vfwsub16(uint16_t a, uint16_t b, float_status *s) { @@ -3125,12 +3137,12 @@ static uint64_t vfwsub32(uint32_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwsub_vv_h, WOP_UUU_H, H4, H2, H2, vfwsub16) RVVCALL(OPFVV2, vfwsub_vv_w, WOP_UUU_W, H8, H4, H4, vfwsub32) -GEN_VEXT_VV_ENV(vfwsub_vv_h) -GEN_VEXT_VV_ENV(vfwsub_vv_w) +GEN_VEXT_VV_ENV(vfwsub_vv_h, 4) +GEN_VEXT_VV_ENV(vfwsub_vv_w, 8) RVVCALL(OPFVF2, vfwsub_vf_h, WOP_UUU_H, H4, H2, vfwsub16) RVVCALL(OPFVF2, vfwsub_vf_w, WOP_UUU_W, H8, H4, vfwsub32) -GEN_VEXT_VF(vfwsub_vf_h) -GEN_VEXT_VF(vfwsub_vf_w) +GEN_VEXT_VF(vfwsub_vf_h, 4) +GEN_VEXT_VF(vfwsub_vf_w, 8) static uint32_t vfwaddw16(uint32_t a, uint16_t b, float_status *s) { @@ -3144,12 +3156,12 @@ static uint64_t vfwaddw32(uint64_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwadd_wv_h, WOP_WUUU_H, H4, H2, H2, vfwaddw16) RVVCALL(OPFVV2, vfwadd_wv_w, WOP_WUUU_W, H8, H4, H4, vfwaddw32) -GEN_VEXT_VV_ENV(vfwadd_wv_h) -GEN_VEXT_VV_ENV(vfwadd_wv_w) +GEN_VEXT_VV_ENV(vfwadd_wv_h, 4) +GEN_VEXT_VV_ENV(vfwadd_wv_w, 8) RVVCALL(OPFVF2, vfwadd_wf_h, WOP_WUUU_H, H4, H2, vfwaddw16) RVVCALL(OPFVF2, vfwadd_wf_w, WOP_WUUU_W, H8, H4, vfwaddw32) -GEN_VEXT_VF(vfwadd_wf_h) -GEN_VEXT_VF(vfwadd_wf_w) +GEN_VEXT_VF(vfwadd_wf_h, 4) +GEN_VEXT_VF(vfwadd_wf_w, 8) static uint32_t vfwsubw16(uint32_t a, uint16_t b, float_status *s) { @@ -3163,39 +3175,39 @@ static uint64_t vfwsubw32(uint64_t a, uint32_t b, float_status *s) RVVCALL(OPFVV2, vfwsub_wv_h, WOP_WUUU_H, H4, H2, H2, vfwsubw16) RVVCALL(OPFVV2, vfwsub_wv_w, WOP_WUUU_W, H8, H4, H4, vfwsubw32) -GEN_VEXT_VV_ENV(vfwsub_wv_h) -GEN_VEXT_VV_ENV(vfwsub_wv_w) +GEN_VEXT_VV_ENV(vfwsub_wv_h, 4) +GEN_VEXT_VV_ENV(vfwsub_wv_w, 8) RVVCALL(OPFVF2, vfwsub_wf_h, WOP_WUUU_H, H4, H2, vfwsubw16) RVVCALL(OPFVF2, vfwsub_wf_w, WOP_WUUU_W, H8, H4, vfwsubw32) -GEN_VEXT_VF(vfwsub_wf_h) -GEN_VEXT_VF(vfwsub_wf_w) +GEN_VEXT_VF(vfwsub_wf_h, 4) +GEN_VEXT_VF(vfwsub_wf_w, 8) /* Vector Single-Width Floating-Point Multiply/Divide Instructions */ RVVCALL(OPFVV2, vfmul_vv_h, OP_UUU_H, H2, H2, H2, float16_mul) RVVCALL(OPFVV2, vfmul_vv_w, OP_UUU_W, H4, H4, H4, float32_mul) RVVCALL(OPFVV2, vfmul_vv_d, OP_UUU_D, H8, H8, H8, float64_mul) -GEN_VEXT_VV_ENV(vfmul_vv_h) -GEN_VEXT_VV_ENV(vfmul_vv_w) -GEN_VEXT_VV_ENV(vfmul_vv_d) +GEN_VEXT_VV_ENV(vfmul_vv_h, 2) +GEN_VEXT_VV_ENV(vfmul_vv_w, 4) +GEN_VEXT_VV_ENV(vfmul_vv_d, 8) RVVCALL(OPFVF2, vfmul_vf_h, OP_UUU_H, H2, H2, float16_mul) RVVCALL(OPFVF2, vfmul_vf_w, OP_UUU_W, H4, H4, float32_mul) RVVCALL(OPFVF2, vfmul_vf_d, OP_UUU_D, H8, H8, float64_mul) -GEN_VEXT_VF(vfmul_vf_h) -GEN_VEXT_VF(vfmul_vf_w) -GEN_VEXT_VF(vfmul_vf_d) +GEN_VEXT_VF(vfmul_vf_h, 2) +GEN_VEXT_VF(vfmul_vf_w, 4) +GEN_VEXT_VF(vfmul_vf_d, 8) RVVCALL(OPFVV2, vfdiv_vv_h, OP_UUU_H, H2, H2, H2, float16_div) RVVCALL(OPFVV2, vfdiv_vv_w, OP_UUU_W, H4, H4, H4, float32_div) RVVCALL(OPFVV2, vfdiv_vv_d, OP_UUU_D, H8, H8, H8, float64_div) -GEN_VEXT_VV_ENV(vfdiv_vv_h) -GEN_VEXT_VV_ENV(vfdiv_vv_w) -GEN_VEXT_VV_ENV(vfdiv_vv_d) +GEN_VEXT_VV_ENV(vfdiv_vv_h, 2) +GEN_VEXT_VV_ENV(vfdiv_vv_w, 4) +GEN_VEXT_VV_ENV(vfdiv_vv_d, 8) RVVCALL(OPFVF2, vfdiv_vf_h, OP_UUU_H, H2, H2, float16_div) RVVCALL(OPFVF2, vfdiv_vf_w, OP_UUU_W, H4, H4, float32_div) RVVCALL(OPFVF2, vfdiv_vf_d, OP_UUU_D, H8, H8, float64_div) -GEN_VEXT_VF(vfdiv_vf_h) -GEN_VEXT_VF(vfdiv_vf_w) -GEN_VEXT_VF(vfdiv_vf_d) +GEN_VEXT_VF(vfdiv_vf_h, 2) +GEN_VEXT_VF(vfdiv_vf_w, 4) +GEN_VEXT_VF(vfdiv_vf_d, 8) static uint16_t float16_rdiv(uint16_t a, uint16_t b, float_status *s) { @@ -3215,9 +3227,9 @@ static uint64_t float64_rdiv(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVF2, vfrdiv_vf_h, OP_UUU_H, H2, H2, float16_rdiv) RVVCALL(OPFVF2, vfrdiv_vf_w, OP_UUU_W, H4, H4, float32_rdiv) RVVCALL(OPFVF2, vfrdiv_vf_d, OP_UUU_D, H8, H8, float64_rdiv) -GEN_VEXT_VF(vfrdiv_vf_h) -GEN_VEXT_VF(vfrdiv_vf_w) -GEN_VEXT_VF(vfrdiv_vf_d) +GEN_VEXT_VF(vfrdiv_vf_h, 2) +GEN_VEXT_VF(vfrdiv_vf_w, 4) +GEN_VEXT_VF(vfrdiv_vf_d, 8) /* Vector Widening Floating-Point Multiply */ static uint32_t vfwmul16(uint16_t a, uint16_t b, float_status *s) @@ -3234,12 +3246,12 @@ static uint64_t vfwmul32(uint32_t a, uint32_t b, float_status *s) } RVVCALL(OPFVV2, vfwmul_vv_h, WOP_UUU_H, H4, H2, H2, vfwmul16) RVVCALL(OPFVV2, vfwmul_vv_w, WOP_UUU_W, H8, H4, H4, vfwmul32) -GEN_VEXT_VV_ENV(vfwmul_vv_h) -GEN_VEXT_VV_ENV(vfwmul_vv_w) +GEN_VEXT_VV_ENV(vfwmul_vv_h, 4) +GEN_VEXT_VV_ENV(vfwmul_vv_w, 8) RVVCALL(OPFVF2, vfwmul_vf_h, WOP_UUU_H, H4, H2, vfwmul16) RVVCALL(OPFVF2, vfwmul_vf_w, WOP_UUU_W, H8, H4, vfwmul32) -GEN_VEXT_VF(vfwmul_vf_h) -GEN_VEXT_VF(vfwmul_vf_w) +GEN_VEXT_VF(vfwmul_vf_h, 4) +GEN_VEXT_VF(vfwmul_vf_w, 8) /* Vector Single-Width Floating-Point Fused Multiply-Add Instructions */ #define OPFVV3(NAME, TD, T1, T2, TX1, TX2, HD, HS1, HS2, OP) \ @@ -3270,9 +3282,9 @@ static uint64_t fmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmacc_vv_h, OP_UUU_H, H2, H2, H2, fmacc16) RVVCALL(OPFVV3, vfmacc_vv_w, OP_UUU_W, H4, H4, H4, fmacc32) RVVCALL(OPFVV3, vfmacc_vv_d, OP_UUU_D, H8, H8, H8, fmacc64) -GEN_VEXT_VV_ENV(vfmacc_vv_h) -GEN_VEXT_VV_ENV(vfmacc_vv_w) -GEN_VEXT_VV_ENV(vfmacc_vv_d) +GEN_VEXT_VV_ENV(vfmacc_vv_h, 2) +GEN_VEXT_VV_ENV(vfmacc_vv_w, 4) +GEN_VEXT_VV_ENV(vfmacc_vv_d, 8) #define OPFVF3(NAME, TD, T1, T2, TX1, TX2, HD, HS2, OP) \ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ @@ -3286,9 +3298,9 @@ static void do_##NAME(void *vd, uint64_t s1, void *vs2, int i, \ RVVCALL(OPFVF3, vfmacc_vf_h, OP_UUU_H, H2, H2, fmacc16) RVVCALL(OPFVF3, vfmacc_vf_w, OP_UUU_W, H4, H4, fmacc32) RVVCALL(OPFVF3, vfmacc_vf_d, OP_UUU_D, H8, H8, fmacc64) -GEN_VEXT_VF(vfmacc_vf_h) -GEN_VEXT_VF(vfmacc_vf_w) -GEN_VEXT_VF(vfmacc_vf_d) +GEN_VEXT_VF(vfmacc_vf_h, 2) +GEN_VEXT_VF(vfmacc_vf_w, 4) +GEN_VEXT_VF(vfmacc_vf_d, 8) static uint16_t fnmacc16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3311,15 +3323,15 @@ static uint64_t fnmacc64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmacc_vv_h, OP_UUU_H, H2, H2, H2, fnmacc16) RVVCALL(OPFVV3, vfnmacc_vv_w, OP_UUU_W, H4, H4, H4, fnmacc32) RVVCALL(OPFVV3, vfnmacc_vv_d, OP_UUU_D, H8, H8, H8, fnmacc64) -GEN_VEXT_VV_ENV(vfnmacc_vv_h) -GEN_VEXT_VV_ENV(vfnmacc_vv_w) -GEN_VEXT_VV_ENV(vfnmacc_vv_d) +GEN_VEXT_VV_ENV(vfnmacc_vv_h, 2) +GEN_VEXT_VV_ENV(vfnmacc_vv_w, 4) +GEN_VEXT_VV_ENV(vfnmacc_vv_d, 8) RVVCALL(OPFVF3, vfnmacc_vf_h, OP_UUU_H, H2, H2, fnmacc16) RVVCALL(OPFVF3, vfnmacc_vf_w, OP_UUU_W, H4, H4, fnmacc32) RVVCALL(OPFVF3, vfnmacc_vf_d, OP_UUU_D, H8, H8, fnmacc64) -GEN_VEXT_VF(vfnmacc_vf_h) -GEN_VEXT_VF(vfnmacc_vf_w) -GEN_VEXT_VF(vfnmacc_vf_d) +GEN_VEXT_VF(vfnmacc_vf_h, 2) +GEN_VEXT_VF(vfnmacc_vf_w, 4) +GEN_VEXT_VF(vfnmacc_vf_d, 8) static uint16_t fmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3339,15 +3351,15 @@ static uint64_t fmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmsac_vv_h, OP_UUU_H, H2, H2, H2, fmsac16) RVVCALL(OPFVV3, vfmsac_vv_w, OP_UUU_W, H4, H4, H4, fmsac32) RVVCALL(OPFVV3, vfmsac_vv_d, OP_UUU_D, H8, H8, H8, fmsac64) -GEN_VEXT_VV_ENV(vfmsac_vv_h) -GEN_VEXT_VV_ENV(vfmsac_vv_w) -GEN_VEXT_VV_ENV(vfmsac_vv_d) +GEN_VEXT_VV_ENV(vfmsac_vv_h, 2) +GEN_VEXT_VV_ENV(vfmsac_vv_w, 4) +GEN_VEXT_VV_ENV(vfmsac_vv_d, 8) RVVCALL(OPFVF3, vfmsac_vf_h, OP_UUU_H, H2, H2, fmsac16) RVVCALL(OPFVF3, vfmsac_vf_w, OP_UUU_W, H4, H4, fmsac32) RVVCALL(OPFVF3, vfmsac_vf_d, OP_UUU_D, H8, H8, fmsac64) -GEN_VEXT_VF(vfmsac_vf_h) -GEN_VEXT_VF(vfmsac_vf_w) -GEN_VEXT_VF(vfmsac_vf_d) +GEN_VEXT_VF(vfmsac_vf_h, 2) +GEN_VEXT_VF(vfmsac_vf_w, 4) +GEN_VEXT_VF(vfmsac_vf_d, 8) static uint16_t fnmsac16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3367,15 +3379,15 @@ static uint64_t fnmsac64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmsac_vv_h, OP_UUU_H, H2, H2, H2, fnmsac16) RVVCALL(OPFVV3, vfnmsac_vv_w, OP_UUU_W, H4, H4, H4, fnmsac32) RVVCALL(OPFVV3, vfnmsac_vv_d, OP_UUU_D, H8, H8, H8, fnmsac64) -GEN_VEXT_VV_ENV(vfnmsac_vv_h) -GEN_VEXT_VV_ENV(vfnmsac_vv_w) -GEN_VEXT_VV_ENV(vfnmsac_vv_d) +GEN_VEXT_VV_ENV(vfnmsac_vv_h, 2) +GEN_VEXT_VV_ENV(vfnmsac_vv_w, 4) +GEN_VEXT_VV_ENV(vfnmsac_vv_d, 8) RVVCALL(OPFVF3, vfnmsac_vf_h, OP_UUU_H, H2, H2, fnmsac16) RVVCALL(OPFVF3, vfnmsac_vf_w, OP_UUU_W, H4, H4, fnmsac32) RVVCALL(OPFVF3, vfnmsac_vf_d, OP_UUU_D, H8, H8, fnmsac64) -GEN_VEXT_VF(vfnmsac_vf_h) -GEN_VEXT_VF(vfnmsac_vf_w) -GEN_VEXT_VF(vfnmsac_vf_d) +GEN_VEXT_VF(vfnmsac_vf_h, 2) +GEN_VEXT_VF(vfnmsac_vf_w, 4) +GEN_VEXT_VF(vfnmsac_vf_d, 8) static uint16_t fmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3395,15 +3407,15 @@ static uint64_t fmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmadd_vv_h, OP_UUU_H, H2, H2, H2, fmadd16) RVVCALL(OPFVV3, vfmadd_vv_w, OP_UUU_W, H4, H4, H4, fmadd32) RVVCALL(OPFVV3, vfmadd_vv_d, OP_UUU_D, H8, H8, H8, fmadd64) -GEN_VEXT_VV_ENV(vfmadd_vv_h) -GEN_VEXT_VV_ENV(vfmadd_vv_w) -GEN_VEXT_VV_ENV(vfmadd_vv_d) +GEN_VEXT_VV_ENV(vfmadd_vv_h, 2) +GEN_VEXT_VV_ENV(vfmadd_vv_w, 4) +GEN_VEXT_VV_ENV(vfmadd_vv_d, 8) RVVCALL(OPFVF3, vfmadd_vf_h, OP_UUU_H, H2, H2, fmadd16) RVVCALL(OPFVF3, vfmadd_vf_w, OP_UUU_W, H4, H4, fmadd32) RVVCALL(OPFVF3, vfmadd_vf_d, OP_UUU_D, H8, H8, fmadd64) -GEN_VEXT_VF(vfmadd_vf_h) -GEN_VEXT_VF(vfmadd_vf_w) -GEN_VEXT_VF(vfmadd_vf_d) +GEN_VEXT_VF(vfmadd_vf_h, 2) +GEN_VEXT_VF(vfmadd_vf_w, 4) +GEN_VEXT_VF(vfmadd_vf_d, 8) static uint16_t fnmadd16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3426,15 +3438,15 @@ static uint64_t fnmadd64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmadd_vv_h, OP_UUU_H, H2, H2, H2, fnmadd16) RVVCALL(OPFVV3, vfnmadd_vv_w, OP_UUU_W, H4, H4, H4, fnmadd32) RVVCALL(OPFVV3, vfnmadd_vv_d, OP_UUU_D, H8, H8, H8, fnmadd64) -GEN_VEXT_VV_ENV(vfnmadd_vv_h) -GEN_VEXT_VV_ENV(vfnmadd_vv_w) -GEN_VEXT_VV_ENV(vfnmadd_vv_d) +GEN_VEXT_VV_ENV(vfnmadd_vv_h, 2) +GEN_VEXT_VV_ENV(vfnmadd_vv_w, 4) +GEN_VEXT_VV_ENV(vfnmadd_vv_d, 8) RVVCALL(OPFVF3, vfnmadd_vf_h, OP_UUU_H, H2, H2, fnmadd16) RVVCALL(OPFVF3, vfnmadd_vf_w, OP_UUU_W, H4, H4, fnmadd32) RVVCALL(OPFVF3, vfnmadd_vf_d, OP_UUU_D, H8, H8, fnmadd64) -GEN_VEXT_VF(vfnmadd_vf_h) -GEN_VEXT_VF(vfnmadd_vf_w) -GEN_VEXT_VF(vfnmadd_vf_d) +GEN_VEXT_VF(vfnmadd_vf_h, 2) +GEN_VEXT_VF(vfnmadd_vf_w, 4) +GEN_VEXT_VF(vfnmadd_vf_d, 8) static uint16_t fmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3454,15 +3466,15 @@ static uint64_t fmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfmsub_vv_h, OP_UUU_H, H2, H2, H2, fmsub16) RVVCALL(OPFVV3, vfmsub_vv_w, OP_UUU_W, H4, H4, H4, fmsub32) RVVCALL(OPFVV3, vfmsub_vv_d, OP_UUU_D, H8, H8, H8, fmsub64) -GEN_VEXT_VV_ENV(vfmsub_vv_h) -GEN_VEXT_VV_ENV(vfmsub_vv_w) -GEN_VEXT_VV_ENV(vfmsub_vv_d) +GEN_VEXT_VV_ENV(vfmsub_vv_h, 2) +GEN_VEXT_VV_ENV(vfmsub_vv_w, 4) +GEN_VEXT_VV_ENV(vfmsub_vv_d, 8) RVVCALL(OPFVF3, vfmsub_vf_h, OP_UUU_H, H2, H2, fmsub16) RVVCALL(OPFVF3, vfmsub_vf_w, OP_UUU_W, H4, H4, fmsub32) RVVCALL(OPFVF3, vfmsub_vf_d, OP_UUU_D, H8, H8, fmsub64) -GEN_VEXT_VF(vfmsub_vf_h) -GEN_VEXT_VF(vfmsub_vf_w) -GEN_VEXT_VF(vfmsub_vf_d) +GEN_VEXT_VF(vfmsub_vf_h, 2) +GEN_VEXT_VF(vfmsub_vf_w, 4) +GEN_VEXT_VF(vfmsub_vf_d, 8) static uint16_t fnmsub16(uint16_t a, uint16_t b, uint16_t d, float_status *s) { @@ -3482,15 +3494,15 @@ static uint64_t fnmsub64(uint64_t a, uint64_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfnmsub_vv_h, OP_UUU_H, H2, H2, H2, fnmsub16) RVVCALL(OPFVV3, vfnmsub_vv_w, OP_UUU_W, H4, H4, H4, fnmsub32) RVVCALL(OPFVV3, vfnmsub_vv_d, OP_UUU_D, H8, H8, H8, fnmsub64) -GEN_VEXT_VV_ENV(vfnmsub_vv_h) -GEN_VEXT_VV_ENV(vfnmsub_vv_w) -GEN_VEXT_VV_ENV(vfnmsub_vv_d) +GEN_VEXT_VV_ENV(vfnmsub_vv_h, 2) +GEN_VEXT_VV_ENV(vfnmsub_vv_w, 4) +GEN_VEXT_VV_ENV(vfnmsub_vv_d, 8) RVVCALL(OPFVF3, vfnmsub_vf_h, OP_UUU_H, H2, H2, fnmsub16) RVVCALL(OPFVF3, vfnmsub_vf_w, OP_UUU_W, H4, H4, fnmsub32) RVVCALL(OPFVF3, vfnmsub_vf_d, OP_UUU_D, H8, H8, fnmsub64) -GEN_VEXT_VF(vfnmsub_vf_h) -GEN_VEXT_VF(vfnmsub_vf_w) -GEN_VEXT_VF(vfnmsub_vf_d) +GEN_VEXT_VF(vfnmsub_vf_h, 2) +GEN_VEXT_VF(vfnmsub_vf_w, 4) +GEN_VEXT_VF(vfnmsub_vf_d, 8) /* Vector Widening Floating-Point Fused Multiply-Add Instructions */ static uint32_t fwmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s) @@ -3507,12 +3519,12 @@ static uint64_t fwmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwmacc16) RVVCALL(OPFVV3, vfwmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwmacc32) -GEN_VEXT_VV_ENV(vfwmacc_vv_h) -GEN_VEXT_VV_ENV(vfwmacc_vv_w) +GEN_VEXT_VV_ENV(vfwmacc_vv_h, 4) +GEN_VEXT_VV_ENV(vfwmacc_vv_w, 8) RVVCALL(OPFVF3, vfwmacc_vf_h, WOP_UUU_H, H4, H2, fwmacc16) RVVCALL(OPFVF3, vfwmacc_vf_w, WOP_UUU_W, H8, H4, fwmacc32) -GEN_VEXT_VF(vfwmacc_vf_h) -GEN_VEXT_VF(vfwmacc_vf_w) +GEN_VEXT_VF(vfwmacc_vf_h, 4) +GEN_VEXT_VF(vfwmacc_vf_w, 8) static uint32_t fwnmacc16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3530,12 +3542,12 @@ static uint64_t fwnmacc32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwnmacc_vv_h, WOP_UUU_H, H4, H2, H2, fwnmacc16) RVVCALL(OPFVV3, vfwnmacc_vv_w, WOP_UUU_W, H8, H4, H4, fwnmacc32) -GEN_VEXT_VV_ENV(vfwnmacc_vv_h) -GEN_VEXT_VV_ENV(vfwnmacc_vv_w) +GEN_VEXT_VV_ENV(vfwnmacc_vv_h, 4) +GEN_VEXT_VV_ENV(vfwnmacc_vv_w, 8) RVVCALL(OPFVF3, vfwnmacc_vf_h, WOP_UUU_H, H4, H2, fwnmacc16) RVVCALL(OPFVF3, vfwnmacc_vf_w, WOP_UUU_W, H8, H4, fwnmacc32) -GEN_VEXT_VF(vfwnmacc_vf_h) -GEN_VEXT_VF(vfwnmacc_vf_w) +GEN_VEXT_VF(vfwnmacc_vf_h, 4) +GEN_VEXT_VF(vfwnmacc_vf_w, 8) static uint32_t fwmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3553,12 +3565,12 @@ static uint64_t fwmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwmsac16) RVVCALL(OPFVV3, vfwmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwmsac32) -GEN_VEXT_VV_ENV(vfwmsac_vv_h) -GEN_VEXT_VV_ENV(vfwmsac_vv_w) +GEN_VEXT_VV_ENV(vfwmsac_vv_h, 4) +GEN_VEXT_VV_ENV(vfwmsac_vv_w, 8) RVVCALL(OPFVF3, vfwmsac_vf_h, WOP_UUU_H, H4, H2, fwmsac16) RVVCALL(OPFVF3, vfwmsac_vf_w, WOP_UUU_W, H8, H4, fwmsac32) -GEN_VEXT_VF(vfwmsac_vf_h) -GEN_VEXT_VF(vfwmsac_vf_w) +GEN_VEXT_VF(vfwmsac_vf_h, 4) +GEN_VEXT_VF(vfwmsac_vf_w, 8) static uint32_t fwnmsac16(uint16_t a, uint16_t b, uint32_t d, float_status *s) { @@ -3576,12 +3588,12 @@ static uint64_t fwnmsac32(uint32_t a, uint32_t b, uint64_t d, float_status *s) RVVCALL(OPFVV3, vfwnmsac_vv_h, WOP_UUU_H, H4, H2, H2, fwnmsac16) RVVCALL(OPFVV3, vfwnmsac_vv_w, WOP_UUU_W, H8, H4, H4, fwnmsac32) -GEN_VEXT_VV_ENV(vfwnmsac_vv_h) -GEN_VEXT_VV_ENV(vfwnmsac_vv_w) +GEN_VEXT_VV_ENV(vfwnmsac_vv_h, 4) +GEN_VEXT_VV_ENV(vfwnmsac_vv_w, 8) RVVCALL(OPFVF3, vfwnmsac_vf_h, WOP_UUU_H, H4, H2, fwnmsac16) RVVCALL(OPFVF3, vfwnmsac_vf_w, WOP_UUU_W, H8, H4, fwnmsac32) -GEN_VEXT_VF(vfwnmsac_vf_h) -GEN_VEXT_VF(vfwnmsac_vf_w) +GEN_VEXT_VF(vfwnmsac_vf_h, 4) +GEN_VEXT_VF(vfwnmsac_vf_w, 8) /* Vector Floating-Point Square-Root Instruction */ /* (TD, T2, TX2) */ @@ -3597,12 +3609,15 @@ static void do_##NAME(void *vd, void *vs2, int i, \ *((TD *)vd + HD(i)) = OP(s2, &env->fp_status); \ } -#define GEN_VEXT_V_ENV(NAME) \ +#define GEN_VEXT_V_ENV(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, ESZ); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ if (vl == 0) { \ @@ -3615,14 +3630,16 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ do_##NAME(vd, vs2, i, env); \ } \ env->vstart = 0; \ + vext_set_elems_1s(vd, vta, vl * ESZ, \ + total_elems * ESZ); \ } RVVCALL(OPFVV1, vfsqrt_v_h, OP_UU_H, H2, H2, float16_sqrt) RVVCALL(OPFVV1, vfsqrt_v_w, OP_UU_W, H4, H4, float32_sqrt) RVVCALL(OPFVV1, vfsqrt_v_d, OP_UU_D, H8, H8, float64_sqrt) -GEN_VEXT_V_ENV(vfsqrt_v_h) -GEN_VEXT_V_ENV(vfsqrt_v_w) -GEN_VEXT_V_ENV(vfsqrt_v_d) +GEN_VEXT_V_ENV(vfsqrt_v_h, 2) +GEN_VEXT_V_ENV(vfsqrt_v_w, 4) +GEN_VEXT_V_ENV(vfsqrt_v_d, 8) /* * Vector Floating-Point Reciprocal Square-Root Estimate Instruction @@ -3802,9 +3819,9 @@ static float64 frsqrt7_d(float64 f, float_status *s) RVVCALL(OPFVV1, vfrsqrt7_v_h, OP_UU_H, H2, H2, frsqrt7_h) RVVCALL(OPFVV1, vfrsqrt7_v_w, OP_UU_W, H4, H4, frsqrt7_s) RVVCALL(OPFVV1, vfrsqrt7_v_d, OP_UU_D, H8, H8, frsqrt7_d) -GEN_VEXT_V_ENV(vfrsqrt7_v_h) -GEN_VEXT_V_ENV(vfrsqrt7_v_w) -GEN_VEXT_V_ENV(vfrsqrt7_v_d) +GEN_VEXT_V_ENV(vfrsqrt7_v_h, 2) +GEN_VEXT_V_ENV(vfrsqrt7_v_w, 4) +GEN_VEXT_V_ENV(vfrsqrt7_v_d, 8) /* * Vector Floating-Point Reciprocal Estimate Instruction @@ -3993,36 +4010,36 @@ static float64 frec7_d(float64 f, float_status *s) RVVCALL(OPFVV1, vfrec7_v_h, OP_UU_H, H2, H2, frec7_h) RVVCALL(OPFVV1, vfrec7_v_w, OP_UU_W, H4, H4, frec7_s) RVVCALL(OPFVV1, vfrec7_v_d, OP_UU_D, H8, H8, frec7_d) -GEN_VEXT_V_ENV(vfrec7_v_h) -GEN_VEXT_V_ENV(vfrec7_v_w) -GEN_VEXT_V_ENV(vfrec7_v_d) +GEN_VEXT_V_ENV(vfrec7_v_h, 2) +GEN_VEXT_V_ENV(vfrec7_v_w, 4) +GEN_VEXT_V_ENV(vfrec7_v_d, 8) /* Vector Floating-Point MIN/MAX Instructions */ RVVCALL(OPFVV2, vfmin_vv_h, OP_UUU_H, H2, H2, H2, float16_minimum_number) RVVCALL(OPFVV2, vfmin_vv_w, OP_UUU_W, H4, H4, H4, float32_minimum_number) RVVCALL(OPFVV2, vfmin_vv_d, OP_UUU_D, H8, H8, H8, float64_minimum_number) -GEN_VEXT_VV_ENV(vfmin_vv_h) -GEN_VEXT_VV_ENV(vfmin_vv_w) -GEN_VEXT_VV_ENV(vfmin_vv_d) +GEN_VEXT_VV_ENV(vfmin_vv_h, 2) +GEN_VEXT_VV_ENV(vfmin_vv_w, 4) +GEN_VEXT_VV_ENV(vfmin_vv_d, 8) RVVCALL(OPFVF2, vfmin_vf_h, OP_UUU_H, H2, H2, float16_minimum_number) RVVCALL(OPFVF2, vfmin_vf_w, OP_UUU_W, H4, H4, float32_minimum_number) RVVCALL(OPFVF2, vfmin_vf_d, OP_UUU_D, H8, H8, float64_minimum_number) -GEN_VEXT_VF(vfmin_vf_h) -GEN_VEXT_VF(vfmin_vf_w) -GEN_VEXT_VF(vfmin_vf_d) +GEN_VEXT_VF(vfmin_vf_h, 2) +GEN_VEXT_VF(vfmin_vf_w, 4) +GEN_VEXT_VF(vfmin_vf_d, 8) RVVCALL(OPFVV2, vfmax_vv_h, OP_UUU_H, H2, H2, H2, float16_maximum_number) RVVCALL(OPFVV2, vfmax_vv_w, OP_UUU_W, H4, H4, H4, float32_maximum_number) RVVCALL(OPFVV2, vfmax_vv_d, OP_UUU_D, H8, H8, H8, float64_maximum_number) -GEN_VEXT_VV_ENV(vfmax_vv_h) -GEN_VEXT_VV_ENV(vfmax_vv_w) -GEN_VEXT_VV_ENV(vfmax_vv_d) +GEN_VEXT_VV_ENV(vfmax_vv_h, 2) +GEN_VEXT_VV_ENV(vfmax_vv_w, 4) +GEN_VEXT_VV_ENV(vfmax_vv_d, 8) RVVCALL(OPFVF2, vfmax_vf_h, OP_UUU_H, H2, H2, float16_maximum_number) RVVCALL(OPFVF2, vfmax_vf_w, OP_UUU_W, H4, H4, float32_maximum_number) RVVCALL(OPFVF2, vfmax_vf_d, OP_UUU_D, H8, H8, float64_maximum_number) -GEN_VEXT_VF(vfmax_vf_h) -GEN_VEXT_VF(vfmax_vf_w) -GEN_VEXT_VF(vfmax_vf_d) +GEN_VEXT_VF(vfmax_vf_h, 2) +GEN_VEXT_VF(vfmax_vf_w, 4) +GEN_VEXT_VF(vfmax_vf_d, 8) /* Vector Floating-Point Sign-Injection Instructions */ static uint16_t fsgnj16(uint16_t a, uint16_t b, float_status *s) @@ -4043,15 +4060,15 @@ static uint64_t fsgnj64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnj_vv_h, OP_UUU_H, H2, H2, H2, fsgnj16) RVVCALL(OPFVV2, vfsgnj_vv_w, OP_UUU_W, H4, H4, H4, fsgnj32) RVVCALL(OPFVV2, vfsgnj_vv_d, OP_UUU_D, H8, H8, H8, fsgnj64) -GEN_VEXT_VV_ENV(vfsgnj_vv_h) -GEN_VEXT_VV_ENV(vfsgnj_vv_w) -GEN_VEXT_VV_ENV(vfsgnj_vv_d) +GEN_VEXT_VV_ENV(vfsgnj_vv_h, 2) +GEN_VEXT_VV_ENV(vfsgnj_vv_w, 4) +GEN_VEXT_VV_ENV(vfsgnj_vv_d, 8) RVVCALL(OPFVF2, vfsgnj_vf_h, OP_UUU_H, H2, H2, fsgnj16) RVVCALL(OPFVF2, vfsgnj_vf_w, OP_UUU_W, H4, H4, fsgnj32) RVVCALL(OPFVF2, vfsgnj_vf_d, OP_UUU_D, H8, H8, fsgnj64) -GEN_VEXT_VF(vfsgnj_vf_h) -GEN_VEXT_VF(vfsgnj_vf_w) -GEN_VEXT_VF(vfsgnj_vf_d) +GEN_VEXT_VF(vfsgnj_vf_h, 2) +GEN_VEXT_VF(vfsgnj_vf_w, 4) +GEN_VEXT_VF(vfsgnj_vf_d, 8) static uint16_t fsgnjn16(uint16_t a, uint16_t b, float_status *s) { @@ -4071,15 +4088,15 @@ static uint64_t fsgnjn64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnjn_vv_h, OP_UUU_H, H2, H2, H2, fsgnjn16) RVVCALL(OPFVV2, vfsgnjn_vv_w, OP_UUU_W, H4, H4, H4, fsgnjn32) RVVCALL(OPFVV2, vfsgnjn_vv_d, OP_UUU_D, H8, H8, H8, fsgnjn64) -GEN_VEXT_VV_ENV(vfsgnjn_vv_h) -GEN_VEXT_VV_ENV(vfsgnjn_vv_w) -GEN_VEXT_VV_ENV(vfsgnjn_vv_d) +GEN_VEXT_VV_ENV(vfsgnjn_vv_h, 2) +GEN_VEXT_VV_ENV(vfsgnjn_vv_w, 4) +GEN_VEXT_VV_ENV(vfsgnjn_vv_d, 8) RVVCALL(OPFVF2, vfsgnjn_vf_h, OP_UUU_H, H2, H2, fsgnjn16) RVVCALL(OPFVF2, vfsgnjn_vf_w, OP_UUU_W, H4, H4, fsgnjn32) RVVCALL(OPFVF2, vfsgnjn_vf_d, OP_UUU_D, H8, H8, fsgnjn64) -GEN_VEXT_VF(vfsgnjn_vf_h) -GEN_VEXT_VF(vfsgnjn_vf_w) -GEN_VEXT_VF(vfsgnjn_vf_d) +GEN_VEXT_VF(vfsgnjn_vf_h, 2) +GEN_VEXT_VF(vfsgnjn_vf_w, 4) +GEN_VEXT_VF(vfsgnjn_vf_d, 8) static uint16_t fsgnjx16(uint16_t a, uint16_t b, float_status *s) { @@ -4099,15 +4116,15 @@ static uint64_t fsgnjx64(uint64_t a, uint64_t b, float_status *s) RVVCALL(OPFVV2, vfsgnjx_vv_h, OP_UUU_H, H2, H2, H2, fsgnjx16) RVVCALL(OPFVV2, vfsgnjx_vv_w, OP_UUU_W, H4, H4, H4, fsgnjx32) RVVCALL(OPFVV2, vfsgnjx_vv_d, OP_UUU_D, H8, H8, H8, fsgnjx64) -GEN_VEXT_VV_ENV(vfsgnjx_vv_h) -GEN_VEXT_VV_ENV(vfsgnjx_vv_w) -GEN_VEXT_VV_ENV(vfsgnjx_vv_d) +GEN_VEXT_VV_ENV(vfsgnjx_vv_h, 2) +GEN_VEXT_VV_ENV(vfsgnjx_vv_w, 4) +GEN_VEXT_VV_ENV(vfsgnjx_vv_d, 8) RVVCALL(OPFVF2, vfsgnjx_vf_h, OP_UUU_H, H2, H2, fsgnjx16) RVVCALL(OPFVF2, vfsgnjx_vf_w, OP_UUU_W, H4, H4, fsgnjx32) RVVCALL(OPFVF2, vfsgnjx_vf_d, OP_UUU_D, H8, H8, fsgnjx64) -GEN_VEXT_VF(vfsgnjx_vf_h) -GEN_VEXT_VF(vfsgnjx_vf_w) -GEN_VEXT_VF(vfsgnjx_vf_d) +GEN_VEXT_VF(vfsgnjx_vf_h, 2) +GEN_VEXT_VF(vfsgnjx_vf_w, 4) +GEN_VEXT_VF(vfsgnjx_vf_d, 8) /* Vector Floating-Point Compare Instructions */ #define GEN_VEXT_CMP_VV_ENV(NAME, ETYPE, H, DO_OP) \ @@ -4116,6 +4133,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4128,6 +4147,13 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ DO_OP(s2, s1, &env->fp_status)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_CMP_VV_ENV(vmfeq_vv_h, uint16_t, H2, float16_eq_quiet) @@ -4140,6 +4166,8 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4151,6 +4179,13 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ DO_OP(s2, (ETYPE)s1, &env->fp_status)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail-agnostic */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } GEN_VEXT_CMP_VF(vmfeq_vf_h, uint16_t, H2, float16_eq_quiet) @@ -4251,12 +4286,15 @@ static void do_##NAME(void *vd, void *vs2, int i) \ *((TD *)vd + HD(i)) = OP(s2); \ } -#define GEN_VEXT_V(NAME) \ +#define GEN_VEXT_V(NAME, ESZ) \ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, ESZ); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4266,6 +4304,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ do_##NAME(vd, vs2, i); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * ESZ, \ + total_elems * ESZ); \ } target_ulong fclass_h(uint64_t frs1) @@ -4328,17 +4369,22 @@ target_ulong fclass_d(uint64_t frs1) RVVCALL(OPIVV1, vfclass_v_h, OP_UU_H, H2, H2, fclass_h) RVVCALL(OPIVV1, vfclass_v_w, OP_UU_W, H4, H4, fclass_s) RVVCALL(OPIVV1, vfclass_v_d, OP_UU_D, H8, H8, fclass_d) -GEN_VEXT_V(vfclass_v_h) -GEN_VEXT_V(vfclass_v_w) -GEN_VEXT_V(vfclass_v_d) +GEN_VEXT_V(vfclass_v_h, 2) +GEN_VEXT_V(vfclass_v_w, 4) +GEN_VEXT_V(vfclass_v_d, 8) /* Vector Floating-Point Merge Instruction */ + #define GEN_VFMERGE_VF(NAME, ETYPE, H) \ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = \ + vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4347,6 +4393,8 @@ void HELPER(NAME)(void *vd, void *v0, uint64_t s1, void *vs2, \ = (!vm && !vext_elem_mask(v0, i) ? s2 : s1); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VFMERGE_VF(vfmerge_vfm_h, int16_t, H2) @@ -4358,33 +4406,33 @@ GEN_VFMERGE_VF(vfmerge_vfm_d, int64_t, H8) RVVCALL(OPFVV1, vfcvt_xu_f_v_h, OP_UU_H, H2, H2, float16_to_uint16) RVVCALL(OPFVV1, vfcvt_xu_f_v_w, OP_UU_W, H4, H4, float32_to_uint32) RVVCALL(OPFVV1, vfcvt_xu_f_v_d, OP_UU_D, H8, H8, float64_to_uint64) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_h) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_w) -GEN_VEXT_V_ENV(vfcvt_xu_f_v_d) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_h, 2) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_w, 4) +GEN_VEXT_V_ENV(vfcvt_xu_f_v_d, 8) /* vfcvt.x.f.v vd, vs2, vm # Convert float to signed integer. */ RVVCALL(OPFVV1, vfcvt_x_f_v_h, OP_UU_H, H2, H2, float16_to_int16) RVVCALL(OPFVV1, vfcvt_x_f_v_w, OP_UU_W, H4, H4, float32_to_int32) RVVCALL(OPFVV1, vfcvt_x_f_v_d, OP_UU_D, H8, H8, float64_to_int64) -GEN_VEXT_V_ENV(vfcvt_x_f_v_h) -GEN_VEXT_V_ENV(vfcvt_x_f_v_w) -GEN_VEXT_V_ENV(vfcvt_x_f_v_d) +GEN_VEXT_V_ENV(vfcvt_x_f_v_h, 2) +GEN_VEXT_V_ENV(vfcvt_x_f_v_w, 4) +GEN_VEXT_V_ENV(vfcvt_x_f_v_d, 8) /* vfcvt.f.xu.v vd, vs2, vm # Convert unsigned integer to float. */ RVVCALL(OPFVV1, vfcvt_f_xu_v_h, OP_UU_H, H2, H2, uint16_to_float16) RVVCALL(OPFVV1, vfcvt_f_xu_v_w, OP_UU_W, H4, H4, uint32_to_float32) RVVCALL(OPFVV1, vfcvt_f_xu_v_d, OP_UU_D, H8, H8, uint64_to_float64) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_h) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_w) -GEN_VEXT_V_ENV(vfcvt_f_xu_v_d) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_h, 2) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_w, 4) +GEN_VEXT_V_ENV(vfcvt_f_xu_v_d, 8) /* vfcvt.f.x.v vd, vs2, vm # Convert integer to float. */ RVVCALL(OPFVV1, vfcvt_f_x_v_h, OP_UU_H, H2, H2, int16_to_float16) RVVCALL(OPFVV1, vfcvt_f_x_v_w, OP_UU_W, H4, H4, int32_to_float32) RVVCALL(OPFVV1, vfcvt_f_x_v_d, OP_UU_D, H8, H8, int64_to_float64) -GEN_VEXT_V_ENV(vfcvt_f_x_v_h) -GEN_VEXT_V_ENV(vfcvt_f_x_v_w) -GEN_VEXT_V_ENV(vfcvt_f_x_v_d) +GEN_VEXT_V_ENV(vfcvt_f_x_v_h, 2) +GEN_VEXT_V_ENV(vfcvt_f_x_v_w, 4) +GEN_VEXT_V_ENV(vfcvt_f_x_v_d, 8) /* Widening Floating-Point/Integer Type-Convert Instructions */ /* (TD, T2, TX2) */ @@ -4394,30 +4442,30 @@ GEN_VEXT_V_ENV(vfcvt_f_x_v_d) /* vfwcvt.xu.f.v vd, vs2, vm # Convert float to double-width unsigned integer.*/ RVVCALL(OPFVV1, vfwcvt_xu_f_v_h, WOP_UU_H, H4, H2, float16_to_uint32) RVVCALL(OPFVV1, vfwcvt_xu_f_v_w, WOP_UU_W, H8, H4, float32_to_uint64) -GEN_VEXT_V_ENV(vfwcvt_xu_f_v_h) -GEN_VEXT_V_ENV(vfwcvt_xu_f_v_w) +GEN_VEXT_V_ENV(vfwcvt_xu_f_v_h, 4) +GEN_VEXT_V_ENV(vfwcvt_xu_f_v_w, 8) /* vfwcvt.x.f.v vd, vs2, vm # Convert float to double-width signed integer. */ RVVCALL(OPFVV1, vfwcvt_x_f_v_h, WOP_UU_H, H4, H2, float16_to_int32) RVVCALL(OPFVV1, vfwcvt_x_f_v_w, WOP_UU_W, H8, H4, float32_to_int64) -GEN_VEXT_V_ENV(vfwcvt_x_f_v_h) -GEN_VEXT_V_ENV(vfwcvt_x_f_v_w) +GEN_VEXT_V_ENV(vfwcvt_x_f_v_h, 4) +GEN_VEXT_V_ENV(vfwcvt_x_f_v_w, 8) /* vfwcvt.f.xu.v vd, vs2, vm # Convert unsigned integer to double-width float */ RVVCALL(OPFVV1, vfwcvt_f_xu_v_b, WOP_UU_B, H2, H1, uint8_to_float16) RVVCALL(OPFVV1, vfwcvt_f_xu_v_h, WOP_UU_H, H4, H2, uint16_to_float32) RVVCALL(OPFVV1, vfwcvt_f_xu_v_w, WOP_UU_W, H8, H4, uint32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_b) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_h) -GEN_VEXT_V_ENV(vfwcvt_f_xu_v_w) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_b, 2) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_h, 4) +GEN_VEXT_V_ENV(vfwcvt_f_xu_v_w, 8) /* vfwcvt.f.x.v vd, vs2, vm # Convert integer to double-width float. */ RVVCALL(OPFVV1, vfwcvt_f_x_v_b, WOP_UU_B, H2, H1, int8_to_float16) RVVCALL(OPFVV1, vfwcvt_f_x_v_h, WOP_UU_H, H4, H2, int16_to_float32) RVVCALL(OPFVV1, vfwcvt_f_x_v_w, WOP_UU_W, H8, H4, int32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_b) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_h) -GEN_VEXT_V_ENV(vfwcvt_f_x_v_w) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_b, 2) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_h, 4) +GEN_VEXT_V_ENV(vfwcvt_f_x_v_w, 8) /* * vfwcvt.f.f.v vd, vs2, vm @@ -4430,8 +4478,8 @@ static uint32_t vfwcvtffv16(uint16_t a, float_status *s) RVVCALL(OPFVV1, vfwcvt_f_f_v_h, WOP_UU_H, H4, H2, vfwcvtffv16) RVVCALL(OPFVV1, vfwcvt_f_f_v_w, WOP_UU_W, H8, H4, float32_to_float64) -GEN_VEXT_V_ENV(vfwcvt_f_f_v_h) -GEN_VEXT_V_ENV(vfwcvt_f_f_v_w) +GEN_VEXT_V_ENV(vfwcvt_f_f_v_h, 4) +GEN_VEXT_V_ENV(vfwcvt_f_f_v_w, 8) /* Narrowing Floating-Point/Integer Type-Convert Instructions */ /* (TD, T2, TX2) */ @@ -4442,29 +4490,29 @@ GEN_VEXT_V_ENV(vfwcvt_f_f_v_w) RVVCALL(OPFVV1, vfncvt_xu_f_w_b, NOP_UU_B, H1, H2, float16_to_uint8) RVVCALL(OPFVV1, vfncvt_xu_f_w_h, NOP_UU_H, H2, H4, float32_to_uint16) RVVCALL(OPFVV1, vfncvt_xu_f_w_w, NOP_UU_W, H4, H8, float64_to_uint32) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_b) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_h) -GEN_VEXT_V_ENV(vfncvt_xu_f_w_w) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_b, 1) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_h, 2) +GEN_VEXT_V_ENV(vfncvt_xu_f_w_w, 4) /* vfncvt.x.f.v vd, vs2, vm # Convert double-width float to signed integer. */ RVVCALL(OPFVV1, vfncvt_x_f_w_b, NOP_UU_B, H1, H2, float16_to_int8) RVVCALL(OPFVV1, vfncvt_x_f_w_h, NOP_UU_H, H2, H4, float32_to_int16) RVVCALL(OPFVV1, vfncvt_x_f_w_w, NOP_UU_W, H4, H8, float64_to_int32) -GEN_VEXT_V_ENV(vfncvt_x_f_w_b) -GEN_VEXT_V_ENV(vfncvt_x_f_w_h) -GEN_VEXT_V_ENV(vfncvt_x_f_w_w) +GEN_VEXT_V_ENV(vfncvt_x_f_w_b, 1) +GEN_VEXT_V_ENV(vfncvt_x_f_w_h, 2) +GEN_VEXT_V_ENV(vfncvt_x_f_w_w, 4) /* vfncvt.f.xu.v vd, vs2, vm # Convert double-width unsigned integer to float */ RVVCALL(OPFVV1, vfncvt_f_xu_w_h, NOP_UU_H, H2, H4, uint32_to_float16) RVVCALL(OPFVV1, vfncvt_f_xu_w_w, NOP_UU_W, H4, H8, uint64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_xu_w_h) -GEN_VEXT_V_ENV(vfncvt_f_xu_w_w) +GEN_VEXT_V_ENV(vfncvt_f_xu_w_h, 2) +GEN_VEXT_V_ENV(vfncvt_f_xu_w_w, 4) /* vfncvt.f.x.v vd, vs2, vm # Convert double-width integer to float. */ RVVCALL(OPFVV1, vfncvt_f_x_w_h, NOP_UU_H, H2, H4, int32_to_float16) RVVCALL(OPFVV1, vfncvt_f_x_w_w, NOP_UU_W, H4, H8, int64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_x_w_h) -GEN_VEXT_V_ENV(vfncvt_f_x_w_w) +GEN_VEXT_V_ENV(vfncvt_f_x_w_h, 2) +GEN_VEXT_V_ENV(vfncvt_f_x_w_w, 4) /* vfncvt.f.f.v vd, vs2, vm # Convert double float to single-width float. */ static uint16_t vfncvtffv16(uint32_t a, float_status *s) @@ -4474,8 +4522,8 @@ static uint16_t vfncvtffv16(uint32_t a, float_status *s) RVVCALL(OPFVV1, vfncvt_f_f_w_h, NOP_UU_H, H2, H4, vfncvtffv16) RVVCALL(OPFVV1, vfncvt_f_f_w_w, NOP_UU_W, H4, H8, float64_to_float32) -GEN_VEXT_V_ENV(vfncvt_f_f_w_h) -GEN_VEXT_V_ENV(vfncvt_f_f_w_w) +GEN_VEXT_V_ENV(vfncvt_f_f_w_h, 2) +GEN_VEXT_V_ENV(vfncvt_f_f_w_w, 4) /* *** Vector Reduction Operations From df4f52a7582c5cd8d2d2f73684996ebc18ac72e3 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:16 +0000 Subject: [PATCH 81/90] target/riscv: rvv: Add tail agnostic for vector reduction instructions Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-13@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/vector_helper.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 17390d8d06bc..174a548ac23b 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -4535,6 +4535,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(TD); \ + uint32_t vlenb = simd_maxsz(desc); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ TD s1 = *((TD *)vs1 + HD(0)); \ \ @@ -4547,6 +4550,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ } \ *((TD *)vd + HD(0)) = s1; \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, esz, vlenb); \ } /* vd[0] = sum(vs1[0], vs2[*]) */ @@ -4616,6 +4621,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(TD); \ + uint32_t vlenb = simd_maxsz(desc); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ TD s1 = *((TD *)vs1 + HD(0)); \ \ @@ -4628,6 +4636,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ } \ *((TD *)vd + HD(0)) = s1; \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, esz, vlenb); \ } /* Unordered sum */ @@ -4652,6 +4662,9 @@ void HELPER(vfwredsum_vs_h)(void *vd, void *v0, void *vs1, { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t esz = sizeof(uint32_t); + uint32_t vlenb = simd_maxsz(desc); + uint32_t vta = vext_vta(desc); uint32_t i; uint32_t s1 = *((uint32_t *)vs1 + H4(0)); @@ -4665,6 +4678,8 @@ void HELPER(vfwredsum_vs_h)(void *vd, void *v0, void *vs1, } *((uint32_t *)vd + H4(0)) = s1; env->vstart = 0; + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, esz, vlenb); } void HELPER(vfwredsum_vs_w)(void *vd, void *v0, void *vs1, @@ -4672,6 +4687,9 @@ void HELPER(vfwredsum_vs_w)(void *vd, void *v0, void *vs1, { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t esz = sizeof(uint64_t); + uint32_t vlenb = simd_maxsz(desc); + uint32_t vta = vext_vta(desc); uint32_t i; uint64_t s1 = *((uint64_t *)vs1); @@ -4685,6 +4703,8 @@ void HELPER(vfwredsum_vs_w)(void *vd, void *v0, void *vs1, } *((uint64_t *)vd) = s1; env->vstart = 0; + /* set tail elements to 1s */ + vext_set_elems_1s(vd, vta, esz, vlenb); } /* From acc6ffd482f6b0435bea7434eb5b13441c6d4a33 Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:35 +0000 Subject: [PATCH 82/90] target/riscv: rvv: Add tail agnostic for vector mask instructions The tail elements in the destination mask register are updated under a tail-agnostic policy. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-14@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 6 +++++ target/riscv/vector_helper.c | 30 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 1add4cb6553a..a94e634a6bf1 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -3135,6 +3135,8 @@ static bool trans_##NAME(DisasContext *s, arg_r *a) \ tcg_gen_brcond_tl(TCG_COND_GEU, cpu_vstart, cpu_vl, over); \ \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = \ + FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\ tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), \ vreg_ofs(s, a->rs1), \ vreg_ofs(s, a->rs2), cpu_env, \ @@ -3239,6 +3241,8 @@ static bool trans_##NAME(DisasContext *s, arg_rmr *a) \ \ data = FIELD_DP32(data, VDATA, VM, a->vm); \ data = FIELD_DP32(data, VDATA, LMUL, s->lmul); \ + data = \ + FIELD_DP32(data, VDATA, VTA_ALL_1S, s->cfg_vta_all_1s);\ tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), \ vreg_ofs(s, 0), vreg_ofs(s, a->rs2), \ cpu_env, s->cfg_ptr->vlen / 8, \ @@ -3276,6 +3280,7 @@ static bool trans_viota_m(DisasContext *s, arg_viota_m *a) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_gvec_3_ptr * const fns[4] = { gen_helper_viota_m_b, gen_helper_viota_m_h, gen_helper_viota_m_w, gen_helper_viota_m_d, @@ -3305,6 +3310,7 @@ static bool trans_vid_v(DisasContext *s, arg_vid_v *a) data = FIELD_DP32(data, VDATA, VM, a->vm); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); static gen_helper_gvec_2_ptr * const fns[4] = { gen_helper_vid_v_b, gen_helper_vid_v_h, gen_helper_vid_v_w, gen_helper_vid_v_d, diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 174a548ac23b..75b59cf91773 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -4717,6 +4717,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t total_elems = env_archcpu(env)->cfg.vlen; \ + uint32_t vta_all_1s = vext_vta_all_1s(desc); \ uint32_t i; \ int a, b; \ \ @@ -4726,6 +4728,15 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, \ vext_set_elem_mask(vd, i, OP(b, a)); \ } \ env->vstart = 0; \ + /* mask destination register are always tail- \ + * agnostic \ + */ \ + /* set tail elements to 1s */ \ + if (vta_all_1s) { \ + for (; i < total_elems; i++) { \ + vext_set_elem_mask(vd, i, 1); \ + } \ + } \ } #define DO_NAND(N, M) (!(N & M)) @@ -4793,6 +4804,8 @@ static void vmsetm(void *vd, void *v0, void *vs2, CPURISCVState *env, { uint32_t vm = vext_vm(desc); uint32_t vl = env->vl; + uint32_t total_elems = env_archcpu(env)->cfg.vlen; + uint32_t vta_all_1s = vext_vta_all_1s(desc); int i; bool first_mask_bit = false; @@ -4821,6 +4834,13 @@ static void vmsetm(void *vd, void *v0, void *vs2, CPURISCVState *env, } } env->vstart = 0; + /* mask destination register are always tail-agnostic */ + /* set tail elements to 1s */ + if (vta_all_1s) { + for (; i < total_elems; i++) { + vext_set_elem_mask(vd, i, 1); + } + } } void HELPER(vmsbf_m)(void *vd, void *v0, void *vs2, CPURISCVState *env, @@ -4848,6 +4868,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, CPURISCVState *env, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t sum = 0; \ int i; \ \ @@ -4861,6 +4884,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, CPURISCVState *env, \ } \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VIOTA_M(viota_m_b, uint8_t, H1) @@ -4874,6 +4899,9 @@ void HELPER(NAME)(void *vd, void *v0, CPURISCVState *env, uint32_t desc) \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ int i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4883,6 +4911,8 @@ void HELPER(NAME)(void *vd, void *v0, CPURISCVState *env, uint32_t desc) \ *((ETYPE *)vd + H(i)) = i; \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VID_V(vid_v_b, uint8_t, H1) From 803963f7cb7220be0c80d9acd87a0ebea167f35e Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:56 +0000 Subject: [PATCH 83/90] target/riscv: rvv: Add tail agnostic for vector permutation instructions Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Acked-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-15@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 7 +++-- target/riscv/vector_helper.c | 40 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index a94e634a6bf1..4f84d4878a2c 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -3669,7 +3669,7 @@ static bool trans_vrgather_vx(DisasContext *s, arg_rmrr *a) return false; } - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { int scale = s->lmul - (s->sew + 3); int vlmax = s->cfg_ptr->vlen >> -scale; TCGv_i64 dest = tcg_temp_new_i64(); @@ -3701,7 +3701,7 @@ static bool trans_vrgather_vi(DisasContext *s, arg_rmrr *a) return false; } - if (a->vm && s->vl_eq_vlmax) { + if (a->vm && s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { int scale = s->lmul - (s->sew + 3); int vlmax = s->cfg_ptr->vlen >> -scale; if (a->rs1 >= vlmax) { @@ -3753,6 +3753,7 @@ static bool trans_vcompress_vm(DisasContext *s, arg_r *a) tcg_gen_brcondi_tl(TCG_COND_EQ, cpu_vl, 0, over); data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_4_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs1), vreg_ofs(s, a->rs2), cpu_env, s->cfg_ptr->vlen / 8, @@ -3853,6 +3854,8 @@ static bool int_ext_op(DisasContext *s, arg_rmr *a, uint8_t seq) } data = FIELD_DP32(data, VDATA, VM, a->vm); + data = FIELD_DP32(data, VDATA, LMUL, s->lmul); + data = FIELD_DP32(data, VDATA, VTA, s->vta); tcg_gen_gvec_3_ptr(vreg_ofs(s, a->rd), vreg_ofs(s, 0), vreg_ofs(s, a->rs2), cpu_env, diff --git a/target/riscv/vector_helper.c b/target/riscv/vector_helper.c index 75b59cf91773..a96fc49c71e3 100644 --- a/target/riscv/vector_helper.c +++ b/target/riscv/vector_helper.c @@ -4931,6 +4931,9 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ { \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ target_ulong offset = s1, i_min, i; \ \ i_min = MAX(env->vstart, offset); \ @@ -4940,6 +4943,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ } \ *((ETYPE *)vd + H(i)) = *((ETYPE *)vs2 + H(i - offset)); \ } \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } /* vslideup.vx vd, vs2, rs1, vm # vd[i+rs1] = vs2[i] */ @@ -4955,6 +4960,9 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(ETYPE))); \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ target_ulong i_max, i; \ \ i_max = MAX(MIN(s1 < vlmax ? vlmax - s1 : 0, vl), env->vstart); \ @@ -4971,6 +4979,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ } \ \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } /* vslidedown.vx vd, vs2, rs1, vm # vd[i] = vs2[i+rs1] */ @@ -4986,6 +4996,9 @@ static void vslide1up_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ typedef uint##BITWIDTH##_t ETYPE; \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -4999,6 +5012,8 @@ static void vslide1up_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ } \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VSLIE1UP(8, H1) @@ -5026,6 +5041,9 @@ static void vslide1down_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ typedef uint##BITWIDTH##_t ETYPE; \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -5039,6 +5057,8 @@ static void vslide1down_##BITWIDTH(void *vd, void *v0, target_ulong s1, \ } \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_VSLIDE1DOWN(8, H1) @@ -5092,6 +5112,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(TS2))); \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(TS2); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint64_t index; \ uint32_t i; \ \ @@ -5107,6 +5130,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ } \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } /* vd[i] = (vs1[i] >= VLMAX) ? 0 : vs2[vs1[i]]; */ @@ -5127,6 +5152,9 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ uint32_t vlmax = vext_max_elems(desc, ctzl(sizeof(ETYPE))); \ uint32_t vm = vext_vm(desc); \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint64_t index = s1; \ uint32_t i; \ \ @@ -5141,6 +5169,8 @@ void HELPER(NAME)(void *vd, void *v0, target_ulong s1, void *vs2, \ } \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } /* vd[i] = (x[rs1] >= VLMAX) ? 0 : vs2[rs1] */ @@ -5155,6 +5185,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ CPURISCVState *env, uint32_t desc) \ { \ uint32_t vl = env->vl; \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t num = 0, i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -5165,6 +5198,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs1, void *vs2, \ num++; \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } /* Compress into vd elements of vs2 where vs1 is enabled */ @@ -5196,6 +5231,9 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ { \ uint32_t vl = env->vl; \ uint32_t vm = vext_vm(desc); \ + uint32_t esz = sizeof(ETYPE); \ + uint32_t total_elems = vext_get_total_elems(env, desc, esz); \ + uint32_t vta = vext_vta(desc); \ uint32_t i; \ \ for (i = env->vstart; i < vl; i++) { \ @@ -5205,6 +5243,8 @@ void HELPER(NAME)(void *vd, void *v0, void *vs2, \ *((ETYPE *)vd + HD(i)) = *((DTYPE *)vs2 + HS1(i)); \ } \ env->vstart = 0; \ + /* set tail elements to 1s */ \ + vext_set_elems_1s(vd, vta, vl * esz, total_elems * esz); \ } GEN_VEXT_INT_EXT(vzext_vf2_h, uint16_t, uint8_t, H2, H1) From b8312675d62b878d6647065f01c2c1337a74d4ee Mon Sep 17 00:00:00 2001 From: eopXD Date: Mon, 6 Jun 2022 06:16:56 +0000 Subject: [PATCH 84/90] target/riscv: rvv: Add option 'rvv_ta_all_1s' to enable optional tail agnostic behavior According to v-spec, tail agnostic behavior can be either kept as undisturbed or set elements' bits to all 1s. To distinguish the difference of tail policies, QEMU should be able to simulate the tail agnostic behavior as "set tail elements' bits to all 1s". There are multiple possibility for agnostic elements according to v-spec. The main intent of this patch-set tries to add option that can distinguish between tail policies. Setting agnostic elements to all 1s allows QEMU to express this. This commit adds option 'rvv_ta_all_1s' is added to enable the behavior, it is default as disabled. Signed-off-by: eop Chen Reviewed-by: Frank Chang Reviewed-by: Weiwei Li Reviewed-by: Alistair Francis Message-Id: <165449614532.19704.7000832880482980398-16@git.sr.ht> Signed-off-by: Alistair Francis --- target/riscv/cpu.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index 0497af45cc95..e5aa1e9c1bf4 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -918,6 +918,8 @@ static Property riscv_cpu_properties[] = { DEFINE_PROP_UINT64("resetvec", RISCVCPU, cfg.resetvec, DEFAULT_RSTVEC), DEFINE_PROP_BOOL("short-isa-string", RISCVCPU, cfg.short_isa_string, false), + + DEFINE_PROP_BOOL("rvv_ta_all_1s", RISCVCPU, cfg.rvv_ta_all_1s, false), DEFINE_PROP_END_OF_LIST(), }; From 26b2bc58599c02b35e55afbd1bd050faa3d187c2 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Wed, 8 Jun 2022 16:14:37 +1000 Subject: [PATCH 85/90] target/riscv: Don't expose the CPU properties on names CPUs There are currently two types of RISC-V CPUs: - Generic CPUs (base or any) that allow complete custimisation - "Named" CPUs that match existing hardware Users can use the base CPUs to custimise the extensions that they want, for example -cpu rv64,v=true. We originally exposed these as part of the named CPUs as well, but that was by accident. Exposing the CPU properties to named CPUs means that we accidently enable extensions that don't exist on the CPUs by default. For example the SiFive E CPU currently support the zba extension, which is a bug. This patch instead only exposes the CPU extensions to the generic CPUs. Signed-off-by: Alistair Francis Reviewed-by: Bin Meng Message-Id: <20220608061437.314434-1-alistair.francis@opensource.wdc.com> Signed-off-by: Alistair Francis --- target/riscv/cpu.c | 57 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/target/riscv/cpu.c b/target/riscv/cpu.c index e5aa1e9c1bf4..05e652135171 100644 --- a/target/riscv/cpu.c +++ b/target/riscv/cpu.c @@ -118,6 +118,8 @@ static const char * const riscv_intr_names[] = { "reserved" }; +static void register_cpu_props(DeviceState *dev); + const char *riscv_cpu_get_trap_name(target_ulong cause, bool async) { if (async) { @@ -161,6 +163,7 @@ static void riscv_any_cpu_init(Object *obj) set_misa(env, MXL_RV64, RVI | RVM | RVA | RVF | RVD | RVC | RVU); #endif set_priv_version(env, PRIV_VERSION_1_12_0); + register_cpu_props(DEVICE(obj)); } #if defined(TARGET_RISCV64) @@ -169,6 +172,7 @@ static void rv64_base_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; /* We set this in the realise function */ set_misa(env, MXL_RV64, 0); + register_cpu_props(DEVICE(obj)); } static void rv64_sifive_u_cpu_init(Object *obj) @@ -181,9 +185,11 @@ static void rv64_sifive_u_cpu_init(Object *obj) static void rv64_sifive_e_cpu_init(Object *obj) { CPURISCVState *env = &RISCV_CPU(obj)->env; + RISCVCPU *cpu = RISCV_CPU(obj); + set_misa(env, MXL_RV64, RVI | RVM | RVA | RVC | RVU); set_priv_version(env, PRIV_VERSION_1_10_0); - qdev_prop_set_bit(DEVICE(obj), "mmu", false); + cpu->cfg.mmu = false; } static void rv128_base_cpu_init(Object *obj) @@ -197,6 +203,7 @@ static void rv128_base_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; /* We set this in the realise function */ set_misa(env, MXL_RV128, 0); + register_cpu_props(DEVICE(obj)); } #else static void rv32_base_cpu_init(Object *obj) @@ -204,6 +211,7 @@ static void rv32_base_cpu_init(Object *obj) CPURISCVState *env = &RISCV_CPU(obj)->env; /* We set this in the realise function */ set_misa(env, MXL_RV32, 0); + register_cpu_props(DEVICE(obj)); } static void rv32_sifive_u_cpu_init(Object *obj) @@ -216,27 +224,33 @@ static void rv32_sifive_u_cpu_init(Object *obj) static void rv32_sifive_e_cpu_init(Object *obj) { CPURISCVState *env = &RISCV_CPU(obj)->env; + RISCVCPU *cpu = RISCV_CPU(obj); + set_misa(env, MXL_RV32, RVI | RVM | RVA | RVC | RVU); set_priv_version(env, PRIV_VERSION_1_10_0); - qdev_prop_set_bit(DEVICE(obj), "mmu", false); + cpu->cfg.mmu = false; } static void rv32_ibex_cpu_init(Object *obj) { CPURISCVState *env = &RISCV_CPU(obj)->env; + RISCVCPU *cpu = RISCV_CPU(obj); + set_misa(env, MXL_RV32, RVI | RVM | RVC | RVU); set_priv_version(env, PRIV_VERSION_1_10_0); - qdev_prop_set_bit(DEVICE(obj), "mmu", false); - qdev_prop_set_bit(DEVICE(obj), "x-epmp", true); + cpu->cfg.mmu = false; + cpu->cfg.epmp = true; } static void rv32_imafcu_nommu_cpu_init(Object *obj) { CPURISCVState *env = &RISCV_CPU(obj)->env; + RISCVCPU *cpu = RISCV_CPU(obj); + set_misa(env, MXL_RV32, RVI | RVM | RVA | RVF | RVC | RVU); set_priv_version(env, PRIV_VERSION_1_10_0); set_resetvec(env, DEFAULT_RSTVEC); - qdev_prop_set_bit(DEVICE(obj), "mmu", false); + cpu->cfg.mmu = false; } #endif @@ -249,6 +263,7 @@ static void riscv_host_cpu_init(Object *obj) #elif defined(TARGET_RISCV64) set_misa(env, MXL_RV64, 0); #endif + register_cpu_props(DEVICE(obj)); } #endif @@ -836,6 +851,12 @@ static void riscv_cpu_init(Object *obj) { RISCVCPU *cpu = RISCV_CPU(obj); + cpu->cfg.ext_counters = true; + cpu->cfg.ext_ifencei = true; + cpu->cfg.ext_icsr = true; + cpu->cfg.mmu = true; + cpu->cfg.pmp = true; + cpu_set_cpustate_pointers(cpu); #ifndef CONFIG_USER_ONLY @@ -844,7 +865,7 @@ static void riscv_cpu_init(Object *obj) #endif /* CONFIG_USER_ONLY */ } -static Property riscv_cpu_properties[] = { +static Property riscv_cpu_extensions[] = { /* Defaults for standard extensions */ DEFINE_PROP_BOOL("i", RISCVCPU, cfg.ext_i, true), DEFINE_PROP_BOOL("e", RISCVCPU, cfg.ext_e, false), @@ -867,17 +888,12 @@ static Property riscv_cpu_properties[] = { DEFINE_PROP_BOOL("Zve64f", RISCVCPU, cfg.ext_zve64f, false), DEFINE_PROP_BOOL("mmu", RISCVCPU, cfg.mmu, true), DEFINE_PROP_BOOL("pmp", RISCVCPU, cfg.pmp, true), - DEFINE_PROP_BOOL("debug", RISCVCPU, cfg.debug, true), DEFINE_PROP_STRING("priv_spec", RISCVCPU, cfg.priv_spec), DEFINE_PROP_STRING("vext_spec", RISCVCPU, cfg.vext_spec), DEFINE_PROP_UINT16("vlen", RISCVCPU, cfg.vlen, 128), DEFINE_PROP_UINT16("elen", RISCVCPU, cfg.elen, 64), - DEFINE_PROP_UINT32("mvendorid", RISCVCPU, cfg.mvendorid, 0), - DEFINE_PROP_UINT64("marchid", RISCVCPU, cfg.marchid, RISCV_CPU_MARCHID), - DEFINE_PROP_UINT64("mimpid", RISCVCPU, cfg.mimpid, RISCV_CPU_MIMPID), - DEFINE_PROP_BOOL("svinval", RISCVCPU, cfg.ext_svinval, false), DEFINE_PROP_BOOL("svnapot", RISCVCPU, cfg.ext_svnapot, false), DEFINE_PROP_BOOL("svpbmt", RISCVCPU, cfg.ext_svpbmt, false), @@ -915,6 +931,25 @@ static Property riscv_cpu_properties[] = { DEFINE_PROP_BOOL("x-epmp", RISCVCPU, cfg.epmp, false), DEFINE_PROP_BOOL("x-aia", RISCVCPU, cfg.aia, false), + DEFINE_PROP_END_OF_LIST(), +}; + +static void register_cpu_props(DeviceState *dev) +{ + Property *prop; + + for (prop = riscv_cpu_extensions; prop && prop->name; prop++) { + qdev_property_add_static(dev, prop); + } +} + +static Property riscv_cpu_properties[] = { + DEFINE_PROP_BOOL("debug", RISCVCPU, cfg.debug, true), + + DEFINE_PROP_UINT32("mvendorid", RISCVCPU, cfg.mvendorid, 0), + DEFINE_PROP_UINT64("marchid", RISCVCPU, cfg.marchid, RISCV_CPU_MARCHID), + DEFINE_PROP_UINT64("mimpid", RISCVCPU, cfg.mimpid, RISCV_CPU_MIMPID), + DEFINE_PROP_UINT64("resetvec", RISCVCPU, cfg.resetvec, DEFAULT_RSTVEC), DEFINE_PROP_BOOL("short-isa-string", RISCVCPU, cfg.short_isa_string, false), From 07314158f6aa4d2589520c194a7531b9364a8d54 Mon Sep 17 00:00:00 2001 From: Alistair Francis Date: Thu, 9 Jun 2022 09:47:01 +1000 Subject: [PATCH 86/90] target/riscv: trans_rvv: Avoid assert for RV32 and e64 When running a 32-bit guest, with a e64 vmv.v.x and vl_eq_vlmax set to true the `tcg_debug_assert(vece <= MO_32)` will be triggered inside tcg_gen_gvec_dup_i32(). This patch checks that condition and instead uses tcg_gen_gvec_dup_i64() is required. Resolves: https://gitlab.com/qemu-project/qemu/-/issues/1028 Suggested-by: Robert Bu Signed-off-by: Alistair Francis Reviewed-by: Richard Henderson Message-Id: <20220608234701.369536-1-alistair.francis@opensource.wdc.com> Signed-off-by: Alistair Francis --- target/riscv/insn_trans/trans_rvv.c.inc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/target/riscv/insn_trans/trans_rvv.c.inc b/target/riscv/insn_trans/trans_rvv.c.inc index 4f84d4878a2c..6c091824b6bc 100644 --- a/target/riscv/insn_trans/trans_rvv.c.inc +++ b/target/riscv/insn_trans/trans_rvv.c.inc @@ -2128,8 +2128,16 @@ static bool trans_vmv_v_x(DisasContext *s, arg_vmv_v_x *a) s1 = get_gpr(s, a->rs1, EXT_SIGN); if (s->vl_eq_vlmax && !(s->vta && s->lmul < 0)) { - tcg_gen_gvec_dup_tl(s->sew, vreg_ofs(s, a->rd), - MAXSZ(s), MAXSZ(s), s1); + if (get_xl(s) == MXL_RV32 && s->sew == MO_64) { + TCGv_i64 s1_i64 = tcg_temp_new_i64(); + tcg_gen_ext_tl_i64(s1_i64, s1); + tcg_gen_gvec_dup_i64(s->sew, vreg_ofs(s, a->rd), + MAXSZ(s), MAXSZ(s), s1_i64); + tcg_temp_free_i64(s1_i64); + } else { + tcg_gen_gvec_dup_tl(s->sew, vreg_ofs(s, a->rd), + MAXSZ(s), MAXSZ(s), s1); + } } else { TCGv_i32 desc; TCGv_i64 s1_i64 = tcg_temp_new_i64(); From 893920616a6205ec8588f14b1df97e531fd675e5 Mon Sep 17 00:00:00 2001 From: Matheus Ferst Date: Mon, 4 Apr 2022 11:46:53 -0300 Subject: [PATCH 87/90] target/ppc: Add flag for ISA v2.06 BCDA instructions Adds an insns_flags2 for the BCD assist instructions introduced in Power ISA 2.06. These instructions are not listed in the manuals for e5500[1] and e6500[2], so the flag is only added for POWER7/8/9/10 models. [1] https://www.nxp.com/files-static/32bit/doc/ref_manual/EREF_RM.pdf [2] https://www.nxp.com/docs/en/reference-manual/E6500RM.pdf Signed-off-by: Matheus Ferst --- target/ppc/cpu.h | 5 ++++- target/ppc/cpu_init.c | 10 ++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/target/ppc/cpu.h b/target/ppc/cpu.h index 6d78078f379d..642bae311f91 100644 --- a/target/ppc/cpu.h +++ b/target/ppc/cpu.h @@ -2277,6 +2277,8 @@ enum { PPC2_ISA310 = 0x0000000000100000ULL, /* lwsync instruction */ PPC2_MEM_LWSYNC = 0x0000000000200000ULL, + /* ISA 2.06 BCD assist instructions */ + PPC2_BCDA_ISA206 = 0x0000000000400000ULL, #define PPC_TCG_INSNS2 (PPC2_BOOKE206 | PPC2_VSX | PPC2_PRCNTL | PPC2_DBRX | \ PPC2_ISA205 | PPC2_VSX207 | PPC2_PERM_ISA206 | \ @@ -2285,7 +2287,8 @@ enum { PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | \ PPC2_ALTIVEC_207 | PPC2_ISA207S | PPC2_DFP | \ PPC2_FP_CVT_S64 | PPC2_TM | PPC2_PM_ISA206 | \ - PPC2_ISA300 | PPC2_ISA310 | PPC2_MEM_LWSYNC) + PPC2_ISA300 | PPC2_ISA310 | PPC2_MEM_LWSYNC | \ + PPC2_BCDA_ISA206) }; /*****************************************************************************/ diff --git a/target/ppc/cpu_init.c b/target/ppc/cpu_init.c index 0f891afa0484..a5d2fe4057bd 100644 --- a/target/ppc/cpu_init.c +++ b/target/ppc/cpu_init.c @@ -5985,7 +5985,7 @@ POWERPC_FAMILY(POWER7)(ObjectClass *oc, void *data) PPC2_PERM_ISA206 | PPC2_DIVE_ISA206 | PPC2_ATOMIC_ISA206 | PPC2_FP_CVT_ISA206 | PPC2_FP_TST_ISA206 | PPC2_FP_CVT_S64 | - PPC2_PM_ISA206 | PPC2_MEM_LWSYNC; + PPC2_PM_ISA206 | PPC2_MEM_LWSYNC | PPC2_BCDA_ISA206; pcc->msr_mask = (1ull << MSR_SF) | (1ull << MSR_VR) | (1ull << MSR_VSX) | @@ -6159,7 +6159,8 @@ POWERPC_FAMILY(POWER8)(ObjectClass *oc, void *data) PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 | PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 | - PPC2_TM | PPC2_PM_ISA206 | PPC2_MEM_LWSYNC; + PPC2_TM | PPC2_PM_ISA206 | PPC2_MEM_LWSYNC | + PPC2_BCDA_ISA206; pcc->msr_mask = (1ull << MSR_SF) | (1ull << MSR_HV) | (1ull << MSR_TM) | @@ -6379,7 +6380,8 @@ POWERPC_FAMILY(POWER9)(ObjectClass *oc, void *data) PPC2_FP_TST_ISA206 | PPC2_BCTAR_ISA207 | PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 | PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 | - PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL | PPC2_MEM_LWSYNC; + PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL | PPC2_MEM_LWSYNC | + PPC2_BCDA_ISA206; pcc->msr_mask = (1ull << MSR_SF) | (1ull << MSR_HV) | (1ull << MSR_TM) | @@ -6597,7 +6599,7 @@ POWERPC_FAMILY(POWER10)(ObjectClass *oc, void *data) PPC2_LSQ_ISA207 | PPC2_ALTIVEC_207 | PPC2_ISA205 | PPC2_ISA207S | PPC2_FP_CVT_S64 | PPC2_TM | PPC2_ISA300 | PPC2_PRCNTL | PPC2_ISA310 | - PPC2_MEM_LWSYNC; + PPC2_MEM_LWSYNC | PPC2_BCDA_ISA206; pcc->msr_mask = (1ull << MSR_SF) | (1ull << MSR_HV) | (1ull << MSR_TM) | From 0e8b39c2e306a81a7f7588a2aa0d1e0a59e9537c Mon Sep 17 00:00:00 2001 From: Matheus Ferst Date: Mon, 4 Apr 2022 13:17:44 -0300 Subject: [PATCH 88/90] target/ppc: implement addg6s Implements the following Power ISA v2.06 instruction: addg6s: Add and Generate Sixes Signed-off-by: Matheus Ferst --- target/ppc/insn32.decode | 4 +++ target/ppc/translate/fixedpoint-impl.c.inc | 37 ++++++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/target/ppc/insn32.decode b/target/ppc/insn32.decode index 18a94fa3b52e..f32619f68f26 100644 --- a/target/ppc/insn32.decode +++ b/target/ppc/insn32.decode @@ -299,6 +299,10 @@ CNTTZDM 011111 ..... ..... ..... 1000111011 - @X PDEPD 011111 ..... ..... ..... 0010011100 - @X PEXTD 011111 ..... ..... ..... 0010111100 - @X +## BCD Assist + +ADDG6S 011111 ..... ..... ..... - 001001010 - @X + ### Float-Point Load Instructions LFS 110000 ..... ..... ................ @D diff --git a/target/ppc/translate/fixedpoint-impl.c.inc b/target/ppc/translate/fixedpoint-impl.c.inc index 1aab32be0364..490e49cfc7c5 100644 --- a/target/ppc/translate/fixedpoint-impl.c.inc +++ b/target/ppc/translate/fixedpoint-impl.c.inc @@ -492,3 +492,40 @@ static bool trans_PEXTD(DisasContext *ctx, arg_X *a) #endif return true; } + +static bool trans_ADDG6S(DisasContext *ctx, arg_X *a) +{ + const uint64_t carry_bits = 0x1111111111111111ULL; + TCGv t0, t1, carry, zero = tcg_constant_tl(0); + + REQUIRE_INSNS_FLAGS2(ctx, BCDA_ISA206); + + t0 = tcg_temp_new(); + t1 = tcg_const_tl(0); + carry = tcg_const_tl(0); + + for (int i = 0; i < 16; i++) { + tcg_gen_shri_tl(t0, cpu_gpr[a->ra], i * 4); + tcg_gen_andi_tl(t0, t0, 0xf); + tcg_gen_add_tl(t1, t1, t0); + + tcg_gen_shri_tl(t0, cpu_gpr[a->rb], i * 4); + tcg_gen_andi_tl(t0, t0, 0xf); + tcg_gen_add_tl(t1, t1, t0); + + tcg_gen_andi_tl(t1, t1, 0x10); + tcg_gen_setcond_tl(TCG_COND_NE, t1, t1, zero); + + tcg_gen_shli_tl(t0, t1, i * 4); + tcg_gen_or_tl(carry, carry, t0); + } + + tcg_gen_xori_tl(carry, carry, (target_long)carry_bits); + tcg_gen_muli_tl(cpu_gpr[a->rt], carry, 6); + + tcg_temp_free(t0); + tcg_temp_free(t1); + tcg_temp_free(carry); + + return true; +} From 972c87e9a272e9212b1715d782728269b3e280c4 Mon Sep 17 00:00:00 2001 From: Matheus Ferst Date: Mon, 4 Apr 2022 13:32:03 -0300 Subject: [PATCH 89/90] target/ppc: implement cbcdtd Implements the Convert Binary Coded Decimal To Declets instruction. Since libdecnumber doesn't expose the methods for direct conversion (decDigitsToDPD, BCD2DPD, etc.), the BCD values are converted to decimal32 format, from which the declets are extracted. Where the behavior is undefined, we try to match the result observed in a POWER9 DD2.3. Reviewed-by: Richard Henderson Signed-off-by: Matheus Ferst --- target/ppc/dfp_helper.c | 39 ++++++++++++++++++++++ target/ppc/helper.h | 1 + target/ppc/insn32.decode | 4 +++ target/ppc/translate/fixedpoint-impl.c.inc | 7 ++++ 4 files changed, 51 insertions(+) diff --git a/target/ppc/dfp_helper.c b/target/ppc/dfp_helper.c index 0d01ac3de0b7..db9e994c8c26 100644 --- a/target/ppc/dfp_helper.c +++ b/target/ppc/dfp_helper.c @@ -1391,3 +1391,42 @@ DFP_HELPER_SHIFT(DSCLI, 64, 1) DFP_HELPER_SHIFT(DSCLIQ, 128, 1) DFP_HELPER_SHIFT(DSCRI, 64, 0) DFP_HELPER_SHIFT(DSCRIQ, 128, 0) + +target_ulong helper_CBCDTD(target_ulong s) +{ + uint64_t res = 0; + uint32_t dec32; + uint8_t bcd[6]; + int w, i, offs; + decNumber a; + decContext context; + + decContextDefault(&context, DEC_INIT_DECIMAL32); + + for (w = 1; w >= 0; w--) { + res <<= 32; + decNumberZero(&a); + /* Extract each BCD field of word "w" */ + for (i = 5; i >= 0; i--) { + offs = 4 * (5 - i) + 32 * w; + bcd[i] = extract64(s, offs, 4); + if (bcd[i] > 9) { + /* + * If the field value is greater than 9, the results are + * undefined. We could use a fixed value like 0 or 9, but + * an and with 9 seems to better match the hardware behavior. + */ + bcd[i] &= 9; + } + } + + /* Create a decNumber with the BCD values and convert to decimal32 */ + decNumberSetBCD(&a, bcd, 6); + decimal32FromNumber((decimal32 *)&dec32, &a, &context); + + /* Extract the two declets from the decimal32 value */ + res |= dec32 & 0xfffff; + } + + return res; +} diff --git a/target/ppc/helper.h b/target/ppc/helper.h index 6233e28d853f..3a18d8da94db 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -54,6 +54,7 @@ DEF_HELPER_3(sraw, tl, env, tl, tl) DEF_HELPER_FLAGS_2(CFUGED, TCG_CALL_NO_RWG_SE, i64, i64, i64) DEF_HELPER_FLAGS_2(PDEPD, TCG_CALL_NO_RWG_SE, i64, i64, i64) DEF_HELPER_FLAGS_2(PEXTD, TCG_CALL_NO_RWG_SE, i64, i64, i64) +DEF_HELPER_FLAGS_1(CBCDTD, TCG_CALL_NO_RWG_SE, tl, tl) #if defined(TARGET_PPC64) DEF_HELPER_FLAGS_2(cmpeqb, TCG_CALL_NO_RWG_SE, i32, tl, tl) DEF_HELPER_FLAGS_1(popcntw, TCG_CALL_NO_RWG_SE, tl, tl) diff --git a/target/ppc/insn32.decode b/target/ppc/insn32.decode index f32619f68f26..1937351ea6f8 100644 --- a/target/ppc/insn32.decode +++ b/target/ppc/insn32.decode @@ -87,6 +87,9 @@ &X_rc rt ra rb rc:bool @X_rc ...... rt:5 ra:5 rb:5 .......... rc:1 &X_rc +&X_sa rs ra +@X_sa ...... rs:5 ra:5 ..... .......... . &X_sa + %x_frtp 22:4 !function=times_2 %x_frap 17:4 !function=times_2 %x_frbp 12:4 !function=times_2 @@ -302,6 +305,7 @@ PEXTD 011111 ..... ..... ..... 0010111100 - @X ## BCD Assist ADDG6S 011111 ..... ..... ..... - 001001010 - @X +CBCDTD 011111 ..... ..... ----- 0100111010 - @X_sa ### Float-Point Load Instructions diff --git a/target/ppc/translate/fixedpoint-impl.c.inc b/target/ppc/translate/fixedpoint-impl.c.inc index 490e49cfc7c5..892c9d256886 100644 --- a/target/ppc/translate/fixedpoint-impl.c.inc +++ b/target/ppc/translate/fixedpoint-impl.c.inc @@ -529,3 +529,10 @@ static bool trans_ADDG6S(DisasContext *ctx, arg_X *a) return true; } + +static bool trans_CBCDTD(DisasContext *ctx, arg_X_sa *a) +{ + REQUIRE_INSNS_FLAGS2(ctx, BCDA_ISA206); + gen_helper_CBCDTD(cpu_gpr[a->ra], cpu_gpr[a->rs]); + return true; +} From 7011e24e45270a96f0a9afbd181e1f2281577475 Mon Sep 17 00:00:00 2001 From: Matheus Ferst Date: Mon, 4 Apr 2022 13:55:30 -0300 Subject: [PATCH 90/90] target/ppc: implement cdtbcd Implements the Convert Declets To Binary Coded Decimal instruction. Since libdecnumber doesn't expose the methods for direct conversion (decDigitsFromDPD, DPD2BCD, etc), a positive decimal32 with zero exponent is used as an intermediate value to convert the declets. Reviewed-by: Richard Henderson Signed-off-by: Matheus Ferst --- target/ppc/dfp_helper.c | 26 ++++++++++++++++++++++ target/ppc/helper.h | 1 + target/ppc/insn32.decode | 1 + target/ppc/translate/fixedpoint-impl.c.inc | 7 ++++++ 4 files changed, 35 insertions(+) diff --git a/target/ppc/dfp_helper.c b/target/ppc/dfp_helper.c index db9e994c8c26..5ba74b21245e 100644 --- a/target/ppc/dfp_helper.c +++ b/target/ppc/dfp_helper.c @@ -1392,6 +1392,32 @@ DFP_HELPER_SHIFT(DSCLIQ, 128, 1) DFP_HELPER_SHIFT(DSCRI, 64, 0) DFP_HELPER_SHIFT(DSCRIQ, 128, 0) +target_ulong helper_CDTBCD(target_ulong s) +{ + uint64_t res = 0; + uint32_t dec32, declets; + uint8_t bcd[6]; + int i, w, sh; + decNumber a; + + for (w = 1; w >= 0; w--) { + res <<= 32; + declets = extract64(s, 32 * w, 20); + if (declets) { + /* decimal32 with zero exponent and word "w" declets */ + dec32 = (0x225ULL << 20) | declets; + decimal32ToNumber((decimal32 *)&dec32, &a); + decNumberGetBCD(&a, bcd); + for (i = 0; i < a.digits; i++) { + sh = 4 * (a.digits - 1 - i); + res |= (uint64_t)bcd[i] << sh; + } + } + } + + return res; +} + target_ulong helper_CBCDTD(target_ulong s) { uint64_t res = 0; diff --git a/target/ppc/helper.h b/target/ppc/helper.h index 3a18d8da94db..25a0fb41e11b 100644 --- a/target/ppc/helper.h +++ b/target/ppc/helper.h @@ -54,6 +54,7 @@ DEF_HELPER_3(sraw, tl, env, tl, tl) DEF_HELPER_FLAGS_2(CFUGED, TCG_CALL_NO_RWG_SE, i64, i64, i64) DEF_HELPER_FLAGS_2(PDEPD, TCG_CALL_NO_RWG_SE, i64, i64, i64) DEF_HELPER_FLAGS_2(PEXTD, TCG_CALL_NO_RWG_SE, i64, i64, i64) +DEF_HELPER_FLAGS_1(CDTBCD, TCG_CALL_NO_RWG_SE, tl, tl) DEF_HELPER_FLAGS_1(CBCDTD, TCG_CALL_NO_RWG_SE, tl, tl) #if defined(TARGET_PPC64) DEF_HELPER_FLAGS_2(cmpeqb, TCG_CALL_NO_RWG_SE, i32, tl, tl) diff --git a/target/ppc/insn32.decode b/target/ppc/insn32.decode index 1937351ea6f8..1f15f68e494c 100644 --- a/target/ppc/insn32.decode +++ b/target/ppc/insn32.decode @@ -305,6 +305,7 @@ PEXTD 011111 ..... ..... ..... 0010111100 - @X ## BCD Assist ADDG6S 011111 ..... ..... ..... - 001001010 - @X +CDTBCD 011111 ..... ..... ----- 0100011010 - @X_sa CBCDTD 011111 ..... ..... ----- 0100111010 - @X_sa ### Float-Point Load Instructions diff --git a/target/ppc/translate/fixedpoint-impl.c.inc b/target/ppc/translate/fixedpoint-impl.c.inc index 892c9d256886..cb0097bedb30 100644 --- a/target/ppc/translate/fixedpoint-impl.c.inc +++ b/target/ppc/translate/fixedpoint-impl.c.inc @@ -530,6 +530,13 @@ static bool trans_ADDG6S(DisasContext *ctx, arg_X *a) return true; } +static bool trans_CDTBCD(DisasContext *ctx, arg_X_sa *a) +{ + REQUIRE_INSNS_FLAGS2(ctx, BCDA_ISA206); + gen_helper_CDTBCD(cpu_gpr[a->ra], cpu_gpr[a->rs]); + return true; +} + static bool trans_CBCDTD(DisasContext *ctx, arg_X_sa *a) { REQUIRE_INSNS_FLAGS2(ctx, BCDA_ISA206);