diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef47948d..76581a71 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -189,6 +189,33 @@ jobs: name: meson test log path: build/meson-logs/testlog.txt + macos-build: + name: "macOS build" + needs: [sanity] + runs-on: macos-latest + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + repository: samsungds/macvfn + ref: libvfn4 # TODO: 'master' once merged + + - name: Configure PR branch as submodule + run: | + SUBMODULE_PATH="MacVFN/libvfn" + BRANCH_NAME=${GITHUB_HEAD_REF#refs/heads/} + git config -f .gitmodules submodule.$SUBMODULE_PATH.url ${GITHUB_SERVER_URL}/${{github.event.pull_request.head.repo.full_name}} + git config -f .gitmodules submodule.$SUBMODULE_PATH.branch "$BRANCH_NAME" + cat .gitmodules + git submodule update --remote --init $SUBMODULE_PATH + + - name: Install dependencies + run: brew install meson + + - name: Xcode build + run: make build + device-test: name: "run emulated nvme device tests" diff --git a/.gitignore b/.gitignore index 6e92f57d..f18f49d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ tags +crc64table.h +config.h diff --git a/docs/api/support/log.rst b/docs/api/support/log.rst index 3b0aebd5..e5ef12a0 100644 --- a/docs/api/support/log.rst +++ b/docs/api/support/log.rst @@ -3,4 +3,4 @@ Logging ======= -.. kernel-doc:: include/vfn/support/log.h +.. kernel-doc:: include/vfn/support/platform/linux/log.h diff --git a/docs/api/support/mem.rst b/docs/api/support/mem.rst index 9a629f06..66f9f662 100644 --- a/docs/api/support/mem.rst +++ b/docs/api/support/mem.rst @@ -3,4 +3,4 @@ Memory Allocation Helpers ========================= -.. kernel-doc:: include/vfn/support/mem.h +.. kernel-doc:: include/vfn/support/platform/linux/mem.h diff --git a/docs/api/support/mmio.rst b/docs/api/support/mmio.rst index 55f21eb1..ff1bdbf9 100644 --- a/docs/api/support/mmio.rst +++ b/docs/api/support/mmio.rst @@ -6,4 +6,4 @@ Memory-mapped I/O These helpers expect and return little endian types (i.e. ``leint32_t``) which may help with locating endianness related bugs using the sparse static analyzer. -.. kernel-doc:: include/vfn/support/mmio.h +.. kernel-doc:: include/vfn/support/platform/linux/mmio.h diff --git a/docs/api/support/mutex.rst b/docs/api/support/mutex.rst index b1bbc337..ef85dd4b 100644 --- a/docs/api/support/mutex.rst +++ b/docs/api/support/mutex.rst @@ -3,4 +3,4 @@ Autolockable Mutex ================== -.. kernel-doc:: include/vfn/support/mutex.h +.. kernel-doc:: include/vfn/support/platform/linux/mutex.h diff --git a/examples/cmb-p2p.c b/examples/cmb-p2p.c index bda13f03..b5ebf3e8 100644 --- a/examples/cmb-p2p.c +++ b/examples/cmb-p2p.c @@ -59,13 +59,13 @@ static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64 struct iommu_iova_range *iova_ranges; int num_iova_ranges; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); if (NVME_GET(cap, CAP_CMBS)) - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(0x1)); + mmio_hl_write64(ctrl->regs, NVME_REG_CMBMSC, cpu_to_le64(0x1)); - cmbsz = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBSZ)); - cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CMBSZ)); + cmbloc = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CMBLOC)); szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); len = szu * NVME_GET(cmbsz, CMBSZ_SZ); @@ -103,7 +103,7 @@ static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64 printf("assigned cmb base address is 0x%lx\n", *cba); /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*cba | 0x3)); + mmio_hl_write64(ctrl->regs, NVME_REG_CMBMSC, cpu_to_le64(*cba | 0x3)); return cmb; } diff --git a/examples/cmb.c b/examples/cmb.c index 682e990d..b475a9bb 100644 --- a/examples/cmb.c +++ b/examples/cmb.c @@ -68,19 +68,19 @@ int main(int argc, char **argv) if (nvme_init(&ctrl, bdf, NULL)) err(1, "failed to init nvme controller"); - vs = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_VS)); + vs = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_VS)); if (vs < NVME_VERSION(1, 4, 0)) errx(1, "controller must be compliant with at least nvme v1.4.0"); - cap = le64_to_cpu(mmio_read64(ctrl.regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl.regs, NVME_REG_CAP)); if (!NVME_GET(cap, CAP_CMBS)) errx(1, "controller memory buffer not supported (cap 0x%lx)", cap); /* enable the CMBSZ and CMBLOC registers */ - mmio_hl_write64(ctrl.regs + NVME_REG_CMBMSC, cpu_to_le64(0x1)); + mmio_hl_write64(ctrl.regs, NVME_REG_CMBMSC, cpu_to_le64(0x1)); - cmbsz = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_CMBSZ)); - cmbloc = le32_to_cpu(mmio_read32(ctrl.regs + NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_CMBSZ)); + cmbloc = le32_to_cpu(mmio_read32(ctrl.regs, NVME_REG_CMBLOC)); szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); len = szu * NVME_GET(cmbsz, CMBSZ_SZ); @@ -113,7 +113,7 @@ int main(int argc, char **argv) printf("assigned cmb base address is 0x%lx\n", cba); /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl.regs + NVME_REG_CMBMSC, cpu_to_le64(cba | 0x3)); + mmio_hl_write64(ctrl.regs, NVME_REG_CMBMSC, cpu_to_le64(cba | 0x3)); cmd = (union nvme_cmd) { .opcode = nvme_admin_identify, diff --git a/examples/regs.c b/examples/regs.c index 00244da8..38f87267 100644 --- a/examples/regs.c +++ b/examples/regs.c @@ -66,30 +66,30 @@ int main(int argc, char **argv) regs = ctrl.regs; - cap = le64_to_cpu(mmio_read64(regs + NVME_REG_CAP)); - vs = le32_to_cpu(mmio_read32(regs + NVME_REG_VS)); - intms = le32_to_cpu(mmio_read32(regs + NVME_REG_INTMS)); - intmc = le32_to_cpu(mmio_read32(regs + NVME_REG_INTMC)); - cc = le32_to_cpu(mmio_read32(regs + NVME_REG_CC)); - csts = le32_to_cpu(mmio_read32(regs + NVME_REG_CSTS)); - nssr = le32_to_cpu(mmio_read32(regs + NVME_REG_NSSR)); - aqa = le32_to_cpu(mmio_read32(regs + NVME_REG_AQA)); - asq = le64_to_cpu(mmio_read64(regs + NVME_REG_ASQ)); - acq = le64_to_cpu(mmio_read64(regs + NVME_REG_ACQ)); - cmbloc = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBLOC)); - cmbsz = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBSZ)); - bpinfo = le32_to_cpu(mmio_read32(regs + NVME_REG_BPINFO)); - bprsel = le32_to_cpu(mmio_read32(regs + NVME_REG_BPRSEL)); - bpmbl = le64_to_cpu(mmio_read64(regs + NVME_REG_BPMBL)); - cmbmsc = le64_to_cpu(mmio_read64(regs + NVME_REG_CMBMSC)); - cmbsts = le32_to_cpu(mmio_read32(regs + NVME_REG_CMBSTS)); - pmrcap = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRCAP)); - pmrctl = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRCTL)); - pmrsts = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRSTS)); - pmrebs = le32_to_cpu(mmio_read32(regs + NVME_REG_PMREBS)); - pmrswtp = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRSWTP)); - pmrmsc = le32_to_cpu(mmio_read32(regs + NVME_REG_PMRMSCL)) | - (uint64_t)le32_to_cpu(mmio_read32(regs + NVME_REG_PMRMSCU)) << 32; + cap = le64_to_cpu(mmio_read64(regs, NVME_REG_CAP)); + vs = le32_to_cpu(mmio_read32(regs, NVME_REG_VS)); + intms = le32_to_cpu(mmio_read32(regs, NVME_REG_INTMS)); + intmc = le32_to_cpu(mmio_read32(regs, NVME_REG_INTMC)); + cc = le32_to_cpu(mmio_read32(regs, NVME_REG_CC)); + csts = le32_to_cpu(mmio_read32(regs, NVME_REG_CSTS)); + nssr = le32_to_cpu(mmio_read32(regs, NVME_REG_NSSR)); + aqa = le32_to_cpu(mmio_read32(regs, NVME_REG_AQA)); + asq = le64_to_cpu(mmio_read64(regs, NVME_REG_ASQ)); + acq = le64_to_cpu(mmio_read64(regs, NVME_REG_ACQ)); + cmbloc = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBLOC)); + cmbsz = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBSZ)); + bpinfo = le32_to_cpu(mmio_read32(regs, NVME_REG_BPINFO)); + bprsel = le32_to_cpu(mmio_read32(regs, NVME_REG_BPRSEL)); + bpmbl = le64_to_cpu(mmio_read64(regs, NVME_REG_BPMBL)); + cmbmsc = le64_to_cpu(mmio_read64(regs, NVME_REG_CMBMSC)); + cmbsts = le32_to_cpu(mmio_read32(regs, NVME_REG_CMBSTS)); + pmrcap = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRCAP)); + pmrctl = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRCTL)); + pmrsts = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRSTS)); + pmrebs = le32_to_cpu(mmio_read32(regs, NVME_REG_PMREBS)); + pmrswtp = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRSWTP)); + pmrmsc = le32_to_cpu(mmio_read32(regs, NVME_REG_PMRMSCL)) | + (uint64_t)le32_to_cpu(mmio_read32(regs, NVME_REG_PMRMSCU)) << 32; printf("%-16s %lx\n", "CAP", cap); printf("%-16s %lx\n", "CAP.MQES", NVME_CAP_MQES(cap)); diff --git a/include/vfn/driverkit.h b/include/vfn/driverkit.h new file mode 100644 index 00000000..8d50ffc8 --- /dev/null +++ b/include/vfn/driverkit.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_VFIO_H +#define LIBVFN_VFIO_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include +#include + +#ifdef __cplusplus +} +#endif + +#endif /* LIBVFN_VFIO_H */ diff --git a/include/vfn/driverkit/device.h b/include/vfn/driverkit/device.h new file mode 100644 index 00000000..a783c77a --- /dev/null +++ b/include/vfn/driverkit/device.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_DRIVERKIT_DEVICE_H +#define LIBVFN_DRIVERKIT_DEVICE_H + +#include +#include +#include "iommu/context.h" + +struct driverkit_bar_info { + uint8_t type; + uint8_t memory_index; + uint64_t size; +}; + +struct driverkit_pci_device { + IOPCIDevice *dev; + struct iommu_ctx ctx; + const char *bdf; + + struct driverkit_bar_info bar_region_info[6]; +}; + +#define __iommu_ctx(x) (&((struct driverkit_pci_device *)(x))->ctx) + +#endif /* LIBVFN_DRIVERKIT_DEVICE_H */ diff --git a/include/vfn/driverkit/pci.h b/include/vfn/driverkit/pci.h new file mode 100644 index 00000000..01d54415 --- /dev/null +++ b/include/vfn/driverkit/pci.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +#ifndef LIBVFN_VFIO_PCI_H +#define LIBVFN_VFIO_PCI_H + +#define off_t int + +// NOTE: Dummy PROT to satisfy Linux code +enum { + PROT_READ = 0, + PROT_WRITE = 1, +}; + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf); + +/** + * vfio_pci_map_bar - faux mapping in DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to map + * @len: number of bytes to map + * @offset: offset at which to start mapping + * @prot: what accesses to permit to the mapped area (see ``man mmap``). + * + * Used in place of VFIO map device memory region identified by @idx into virtual memory. + * + * Return: On success, returns the virtual memory address mapped. On error, + * returns ``NULL`` and sets ``errno``. + */ +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, + int prot); + +/** + * vfio_pci_unmap_bar - faux unmap a region in virtual memory for DriverKit + * @pci: &struct driverkit_pci_device + * @idx: the vfio region index to unmap + * @mem: virtual memory address to unmap + * @len: number of bytes to unmap + * @offset: offset at which to start unmapping + * + * Used in place of VFIO to unmap the virtual memory address, previously mapped + * to the vfio device memory region identified by @idx. + */ +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, + uint64_t offset); + +#endif /* LIBVFN_VFIO_PCI_H */ diff --git a/include/vfn/iommu.h b/include/vfn/iommu.h index 98542b08..9f78bb41 100644 --- a/include/vfn/iommu.h +++ b/include/vfn/iommu.h @@ -13,21 +13,15 @@ #ifndef LIBVFN_IOMMU_H #define LIBVFN_IOMMU_H -#ifdef __cplusplus -extern "C" { -#endif - #include #include #include +#ifndef __APPLE__ #include +#endif #include #include -#ifdef __cplusplus -} -#endif - #endif /* LIBVFN_IOMMU_H */ diff --git a/include/vfn/iommu/dma.h b/include/vfn/iommu/dma.h index d6abc809..e21ab5a0 100644 --- a/include/vfn/iommu/dma.h +++ b/include/vfn/iommu/dma.h @@ -31,6 +31,7 @@ enum iommu_map_flags { IOMMU_MAP_NOREAD = 1 << 3, }; +#ifndef __APPLE__ /** * iommu_map_vaddr - Map a virtual memory address to an I/O virtual address * @ctx: &struct iommu_ctx @@ -53,6 +54,9 @@ enum iommu_map_flags { */ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, unsigned long flags); +#endif +int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags, void *opaque); /** * iommu_unmap_vaddr - Unmap a virtual memory address in the IOMMU @@ -78,12 +82,14 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len); */ int iommu_unmap_all(struct iommu_ctx *ctx); +#ifndef __APPLE__ #ifndef IOMMU_IOAS_IOVA_RANGES struct iommu_iova_range { __aligned_u64 start; __aligned_u64 last; }; #endif +#endif /** * iommu_translate_vaddr - Translate a virtual address into an iova @@ -98,6 +104,7 @@ struct iommu_iova_range { */ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); +#ifndef __APPLE__ /** * iommu_get_iova_ranges - Get iova ranges * @ctx: &struct iommu_ctx @@ -108,5 +115,6 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova); * Return: the number of elements in the array pointed to. */ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges); +#endif #endif /* LIBVFN_IOMMU_DMA_H */ diff --git a/include/vfn/nvme.h b/include/vfn/nvme.h index a1023c4b..0dcb720a 100644 --- a/include/vfn/nvme.h +++ b/include/vfn/nvme.h @@ -13,38 +13,21 @@ #ifndef LIBVFN_NVME_H #define LIBVFN_NVME_H -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include +#include -#include +#ifndef __APPLE__ +#include +#else +#define NVME_AQ_QSIZE 32 +#include +#endif -#include #include -#include #include -#include #include #include #include #include #include -#ifdef __cplusplus -} -#endif - #endif /* LIBVFN_NVME_H */ diff --git a/include/vfn/nvme/ctrl.h b/include/vfn/nvme/ctrl.h index b2a686bb..570baf7a 100644 --- a/include/vfn/nvme/ctrl.h +++ b/include/vfn/nvme/ctrl.h @@ -43,7 +43,16 @@ struct nvme_ctrl { /** * @pci: vfio pci device state */ + #ifdef __APPLE__ + struct driverkit_pci_device pci; + #else struct vfio_pci_device pci; + #endif + + /** + * @serial: Serial number from controller + */ + char serial[20]; /** * @regs: Controller Configuration (``MBAR.CC``) registers diff --git a/include/vfn/nvme/queue.h b/include/vfn/nvme/queue.h index b2316826..54cba926 100644 --- a/include/vfn/nvme/queue.h +++ b/include/vfn/nvme/queue.h @@ -27,6 +27,7 @@ struct nvme_dbbuf { */ struct nvme_cq { /* private: */ + void *vaddr_opaque; void *vaddr; uint64_t iova; @@ -51,10 +52,12 @@ struct nvme_sq { /* private: */ struct nvme_cq *cq; + void *vaddr_opaque; void *vaddr; uint64_t iova; struct { + void *vaddr_opaque; void *vaddr; uint64_t iova; } pages; @@ -84,7 +87,7 @@ struct nvme_sq { */ static inline void nvme_sq_post(struct nvme_sq *sq, const union nvme_cmd *sqe) { - memcpy(sq->vaddr + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); + memcpy(((uint8_t *) sq->vaddr) + (sq->tail << NVME_SQES), sqe, 1 << NVME_SQES); trace_guard(NVME_SQ_POST) { trace_emit("sqid %d tail %d\n", sq->id, sq->tail); @@ -147,7 +150,7 @@ static inline void nvme_sq_update_tail(struct nvme_sq *sq) /* do not reorder queue entry store with doorbell store */ wmb(); - mmio_write32(sq->doorbell, cpu_to_le32(sq->tail)); + mmio_write32(sq->doorbell, 0, cpu_to_le32(sq->tail)); } sq->ptail = sq->tail; @@ -174,7 +177,7 @@ static inline void nvme_sq_exec(struct nvme_sq *sq, const union nvme_cmd *sqe) */ static inline struct nvme_cqe *nvme_cq_head(struct nvme_cq *cq) { - return (struct nvme_cqe *)(cq->vaddr + (cq->head << NVME_CQES)); + return (struct nvme_cqe *)(((uint8_t *) cq->vaddr) + (cq->head << NVME_CQES)); } /** @@ -190,7 +193,7 @@ static inline void nvme_cq_update_head(struct nvme_cq *cq) } if (nvme_try_dbbuf(cq->head, &cq->dbbuf)) - mmio_write32(cq->doorbell, cpu_to_le32(cq->head)); + mmio_write32(cq->doorbell, 0, cpu_to_le32(cq->head)); } /** @@ -279,7 +282,7 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n); * @cq: Completion queue * @cqes: Pointer to array of struct cqe to place cqes into * @n: Number of cqes to reap - * @ts: Maximum time to wait for CQEs + * @timeout_ns: Maximum time to wait for CQEs in nanoseconds * * Continuously poll @cq and copy @n cqes into @cqes if not NULL. * @@ -288,6 +291,6 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n); * Return: ``n`` on success. On timeout, returns the number of cqes reaped * (i.e., less than ``n``) and sets ``errno``. */ -int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, struct timespec *ts); +int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, uint64_t timeout_ns); #endif /* LIBVFN_NVME_QUEUE_H */ diff --git a/include/vfn/nvme/rq.h b/include/vfn/nvme/rq.h index ee90cb51..722f68fb 100644 --- a/include/vfn/nvme/rq.h +++ b/include/vfn/nvme/rq.h @@ -217,6 +217,7 @@ static inline void nvme_rq_exec(struct nvme_rq *rq, union nvme_cmd *cmd) int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, uint64_t iova, size_t len); +#ifndef __APPLE__ /** * nvme_rq_mapv_prp - Set up the Physical Region Pages in the data pointer of * the command from an iovec. @@ -234,6 +235,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * */ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, struct iovec *iov, int niov); +#endif /** * nvme_rq_spin - Spin for completion of the command associated with the request diff --git a/include/vfn/support.h b/include/vfn/support.h index 050b3179..eaa86c1f 100644 --- a/include/vfn/support.h +++ b/include/vfn/support.h @@ -17,19 +17,32 @@ extern "C" { #endif +#ifndef __APPLE__ #include #include +#include +#include #include +#include +#include +#include + +#include + +#include +#else +#include +#include +#include +#endif + +#include // PRIx64 used for logging #include #include #include #include #include #include -#include -#include - -#include #include #include @@ -37,13 +50,15 @@ extern "C" { #include #include #include -#include #include #include #include #include -#include #include +#ifndef __APPLE__ +#include +#include +#endif #ifdef __cplusplus } diff --git a/include/vfn/support/log.h b/include/vfn/support/log.h index 32d3faf9..3f15878e 100644 --- a/include/vfn/support/log.h +++ b/include/vfn/support/log.h @@ -10,83 +10,16 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_LOG_H -#define LIBVFN_SUPPORT_LOG_H - -extern struct log_state { - int v; -} __log_state; - -enum __log_level { - LOG_ERROR, - LOG_INFO, - LOG_DEBUG, -}; - -/** - * logv - Determine if log verbosity is as given - * @v: target verbositry level - * - * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. - */ -static inline bool logv(int v) -{ - if (atomic_load_acquire(&__log_state.v) >= v) - return true; - - return false; -} - -/** - * logv_set - Set log verbosity level - * @v: verbosity level - */ -static inline void logv_set(int v) -{ - atomic_store_release(&__log_state.v, v); -} - -/** - * __log - Log a message at a given verbosity level - * @v: verbosity level - * @fmt: format string - * @...: format string arguments - * - * Log a formatted message to stderr if current verbosity level is at least @v. - */ -static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const *fmt, ...) -{ - va_list va; - - if (!logv(v)) - return; - - va_start(va, fmt); - vfprintf(stderr, fmt, va); - va_end(va); -} - -#ifndef log_fmt -#define log_fmt(fmt) fmt -#endif - -#ifdef DEBUG -# define log_error(fmt, ...) __log(LOG_ERROR, "E %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) -# define log_info(fmt, ...) __log(LOG_INFO, "I %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) -# define log_debug(fmt, ...) __log(LOG_DEBUG, "D %s (%s:%d): " log_fmt(fmt), \ - __func__, __FILE__, __LINE__, ##__VA_ARGS__) +#ifdef __APPLE__ +#include #else -# define log_error(fmt, ...) __log(LOG_ERROR, log_fmt(fmt), ##__VA_ARGS__) -# define log_info(fmt, ...) __log(LOG_INFO, log_fmt(fmt), ##__VA_ARGS__) -# define log_debug(fmt, ...) __log(LOG_DEBUG, log_fmt(fmt), ##__VA_ARGS__) -#endif /* DEBUG */ +#include +#endif #define log_fatal(fmt, ...) \ do { \ log_error(fmt, ##__VA_ARGS__); \ - exit(errno); \ + abort(); \ } while (0) #define log_fatal_if(expr, fmt, ...) \ @@ -94,5 +27,3 @@ static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const if (expr) \ log_fatal(fmt, ##__VA_ARGS__); \ } while (0) - -#endif /* LIBVFN_SUPPORT_LOG_H */ diff --git a/include/vfn/support/mem.h b/include/vfn/support/mem.h index 6773c4b1..1470fadc 100644 --- a/include/vfn/support/mem.h +++ b/include/vfn/support/mem.h @@ -13,101 +13,9 @@ #ifndef LIBVFN_SUPPORT_MEM_H #define LIBVFN_SUPPORT_MEM_H -extern size_t __VFN_PAGESIZE; -extern int __VFN_PAGESHIFT; - -void backtrace_abort(void); - -static inline void __do_autofree(void *mem) -{ - void **memp = (void **)mem; - - free(*memp); -} - -#define __autofree __attribute__((cleanup(__do_autofree))) - -static inline bool would_overflow(unsigned int n, size_t sz) -{ - return n > SIZE_MAX / sz; -} - -static inline size_t __abort_on_overflow(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) - backtrace_abort(); - - return n * sz; -} - -/** - * xmalloc - version of malloc that cannot fail - * @sz: number of bytes to allocate - * - * Call malloc, but only return NULL when @sz is zero. Otherwise, abort. - * - * Return: pointer to allocated memory - */ -static inline void *xmalloc(size_t sz) -{ - void *mem; - - if (unlikely(!sz)) - return NULL; - - mem = malloc(sz); - if (unlikely(!mem)) - backtrace_abort(); - - return mem; -} - -static inline void *zmalloc(size_t sz) -{ - void *mem; - - if (unlikely(!sz)) - return NULL; - - mem = calloc(1, sz); - if (unlikely(!mem)) - backtrace_abort(); - - return mem; -} - -static inline void *mallocn(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return xmalloc(n * sz); -} - -static inline void *zmallocn(unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return zmalloc(n * sz); -} - -static inline void *reallocn(void *mem, unsigned int n, size_t sz) -{ - if (would_overflow(n, sz)) { - fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); - - backtrace_abort(); - } - - return realloc(mem, n * sz); -} +ssize_t __pgmap(void **mem, size_t sz, void **opaque); +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque); +void __pgunmap(void *mem, size_t len, void *opaque); #define _new_t(t, n, f) \ ((t *) f(n, sizeof(t))) @@ -118,10 +26,12 @@ static inline void *reallocn(void *mem, unsigned int n, size_t sz) ssize_t pgmap(void **mem, size_t sz); ssize_t pgmapn(void **mem, unsigned int n, size_t sz); -static inline void pgunmap(void *mem, size_t len) -{ - if (munmap(mem, len)) - backtrace_abort(); -} +void pgunmap(void *mem, size_t len); + +#ifdef __APPLE__ +#include +#else +#include +#endif #endif /* LIBVFN_SUPPORT_MEM_H */ diff --git a/include/vfn/support/mmio.h b/include/vfn/support/mmio.h index 5929e7c8..21ba30b5 100644 --- a/include/vfn/support/mmio.h +++ b/include/vfn/support/mmio.h @@ -10,80 +10,8 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MMIO_H -#define LIBVFN_SUPPORT_MMIO_H - -/** - * mmio_read32 - read 4 bytes in memory-mapped register - * @addr: memory-mapped register - * - * Return: read value (native endian) - */ -static inline leint32_t mmio_read32(void *addr) -{ - /* memory-mapped register */ - return *(const volatile leint32_t __force *)addr; -} - -/** - * mmio_lh_read64 - read 8 bytes in memory-mapped register - * @addr: memory-mapped register - * - * Read the low 4 bytes first, then the high 4 bytes. - * - * Return: read value (native endian) - */ -static inline leint64_t mmio_lh_read64(void *addr) -{ - uint32_t lo, hi; - - /* memory-mapped register */ - lo = *(const volatile uint32_t __force *)addr; - /* memory-mapped register */ - hi = *(const volatile uint32_t __force *)(addr + 4); - - return (leint64_t __force)(((uint64_t)hi << 32) | lo); -} - -#define mmio_read64(addr) mmio_lh_read64(addr) - -/** - * mmio_write32 - write 4 bytes to memory-mapped register - * @addr: memory-mapped register - * @v: value to write (native endian) - */ -static inline void mmio_write32(void *addr, leint32_t v) -{ - /* memory-mapped register */ - *(volatile leint32_t __force *)addr = v; -} - -/** - * mmio_lh_write64 - write 8 bytes to memory-mapped register - * @addr: memory-mapped register - * @v: value to write (native endian) - * - * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes - * first, then high). - */ -static inline void mmio_lh_write64(void *addr, leint64_t v) -{ - mmio_write32(addr, (leint32_t __force)v); - mmio_write32(addr + 4, (leint32_t __force)(v >> 32)); -} - -/** - * mmio_hl_write64 - write 8 bytes to memory-mapped register - * @addr: memory-mapped register - * @v: value to write (native endian) - * - * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes - * first, then low). - */ -static inline void mmio_hl_write64(void *addr, leint64_t v) -{ - mmio_write32(addr + 4, (leint32_t __force)((uint64_t __force)v >> 32)); - mmio_write32(addr, (leint32_t __force)v); -} - -#endif /* LIBVFN_SUPPORT_MMIO_H */ +#ifdef __APPLE__ +#include +#else +#include +#endif diff --git a/include/vfn/support/mutex.h b/include/vfn/support/mutex.h index 3b84af26..26aa3546 100644 --- a/include/vfn/support/mutex.h +++ b/include/vfn/support/mutex.h @@ -10,40 +10,8 @@ * COPYING and LICENSE files for more information. */ -#ifndef LIBVFN_SUPPORT_MUTEX_H -#define LIBVFN_SUPPORT_MUTEX_H - -/* - * DOC: Autolockable mutex - * - * Define a __autolock() macro that will lock the given mutex as well as ensure - * that it is unlocked when going out of scope. This is inspired by the - * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this - * version only supports the pthread_mutex_t. - */ - -static inline pthread_mutex_t *__mutex_auto_lock(pthread_mutex_t *mutex) -{ - pthread_mutex_lock(mutex); - return mutex; -} - -static inline void __mutex_auto_unlock(pthread_mutex_t *mutex) -{ - if (mutex) - pthread_mutex_unlock(mutex); -} - -DEFINE_AUTOPTR(pthread_mutex_t, __mutex_auto_unlock) - -/** - * __autolock - autolock mutex - * @x: pointer to pthread mutex - * - * Lock the mutex and unlock it automatically when going out of scope. - */ -#define __autolock(x) \ - __autoptr(pthread_mutex_t) \ - glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(x) - -#endif /* LIBVFN_SUPPORT_MUTEX_H */ +#ifdef __APPLE__ +#include +#else +#include +#endif diff --git a/include/vfn/support/platform/linux/log.h b/include/vfn/support/platform/linux/log.h new file mode 100644 index 00000000..74c66e79 --- /dev/null +++ b/include/vfn/support/platform/linux/log.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_LOG_H +#define LIBVFN_SUPPORT_LOG_H + +extern struct log_state { + int v; +} __log_state; + +enum __log_level { + LOG_ERROR, + LOG_INFO, + LOG_DEBUG, +}; + +/** + * logv - Determine if log verbosity is as given + * @v: target verbositry level + * + * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. + */ +static inline bool logv(int v) +{ + if (atomic_load_acquire(&__log_state.v) >= v) + return true; + + return false; +} + +/** + * logv_set - Set log verbosity level + * @v: verbosity level + */ +static inline void logv_set(int v) +{ + atomic_store_release(&__log_state.v, v); +} + +/** + * __log - Log a message at a given verbosity level + * @v: verbosity level + * @fmt: format string + * @...: format string arguments + * + * Log a formatted message to stderr if current verbosity level is at least @v. + */ +static inline void __attribute__((format(printf, 2, 3))) __log(int v, char const *fmt, ...) +{ + va_list va; + + if (!logv(v)) + return; + + va_start(va, fmt); + vfprintf(stderr, fmt, va); + va_end(va); +} + +#ifndef log_fmt +#define log_fmt(fmt) fmt +#endif + +#ifdef DEBUG +# define log_error(fmt, ...) __log(LOG_ERROR, "E %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +# define log_info(fmt, ...) __log(LOG_INFO, "I %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +# define log_debug(fmt, ...) __log(LOG_DEBUG, "D %s (%s:%d): " log_fmt(fmt), \ + __func__, __FILE__, __LINE__, ##__VA_ARGS__) +#else +# define log_error(fmt, ...) __log(LOG_ERROR, log_fmt(fmt), ##__VA_ARGS__) +# define log_info(fmt, ...) __log(LOG_INFO, log_fmt(fmt), ##__VA_ARGS__) +# define log_debug(fmt, ...) __log(LOG_DEBUG, log_fmt(fmt), ##__VA_ARGS__) +#endif /* DEBUG */ + +#endif /* LIBVFN_SUPPORT_LOG_H */ diff --git a/include/vfn/support/platform/linux/mem.h b/include/vfn/support/platform/linux/mem.h new file mode 100644 index 00000000..67b4cf68 --- /dev/null +++ b/include/vfn/support/platform/linux/mem.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +extern size_t __VFN_PAGESIZE; +extern int __VFN_PAGESHIFT; + +void backtrace_abort(void); + +static inline void __do_autofree(void *mem) +{ + void **memp = (void **)mem; + + free(*memp); +} + +#define __autofree __attribute__((cleanup(__do_autofree))) + +static inline bool would_overflow(unsigned int n, size_t sz) +{ + return n > SIZE_MAX / sz; +} + +static inline size_t __abort_on_overflow(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) + backtrace_abort(); + + return n * sz; +} + +/** + * xmalloc - version of malloc that cannot fail + * @sz: number of bytes to allocate + * + * Call malloc, but only return NULL when @sz is zero. Otherwise, abort. + * + * Return: pointer to allocated memory + */ +static inline void *xmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + mem = malloc(sz); + if (unlikely(!mem)) + backtrace_abort(); + + return mem; +} + +static inline void *zmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + mem = calloc(1, sz); + if (unlikely(!mem)) + backtrace_abort(); + + return mem; +} + +static inline void *mallocn(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return xmalloc(n * sz); +} + +static inline void *zmallocn(unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return zmalloc(n * sz); +} + +static inline void *reallocn(void *mem, unsigned int n, size_t sz) +{ + if (would_overflow(n, sz)) { + fprintf(stderr, "allocation of %d * %zu bytes would overflow\n", n, sz); + + backtrace_abort(); + } + + return realloc(mem, n * sz); +} diff --git a/include/vfn/support/platform/linux/mmio.h b/include/vfn/support/platform/linux/mmio.h new file mode 100644 index 00000000..1b52ae23 --- /dev/null +++ b/include/vfn/support/platform/linux/mmio.h @@ -0,0 +1,94 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MMIO_H +#define LIBVFN_SUPPORT_MMIO_H + +/** + * mmio_read32 - read 4 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Return: read value (native endian) + */ +static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) +{ + /* memory-mapped register */ + return *(const volatile leint32_t __force *)(base_addr + offset); +} + +/** + * mmio_lh_read64 - read 8 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Read the low 4 bytes first, then the high 4 bytes. + * + * Return: read value (native endian) + */ +static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) +{ + uint32_t lo, hi; + + /* memory-mapped register */ + lo = *(const volatile uint32_t __force *)(base_addr + offset); + /* memory-mapped register */ + hi = *(const volatile uint32_t __force *)(base_addr + offset + 4); + + return (leint64_t __force)(((uint64_t)hi << 32) | lo); +} + +#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) + +/** + * mmio_write32 - write 4 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + */ +static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) +{ + /* memory-mapped register */ + *(volatile leint32_t __force *)(base_addr + offset) = v; +} + +/** + * mmio_lh_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes + * first, then high). + */ +static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset, (leint32_t __force)v); + mmio_write32(base_addr, offset + 4, (leint32_t __force)(v >> 32)); +} + +/** + * mmio_hl_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes + * first, then low). + */ +static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset + 4, (leint32_t __force)((uint64_t __force)v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); +} + +#endif /* LIBVFN_SUPPORT_MMIO_H */ diff --git a/include/vfn/support/platform/linux/mutex.h b/include/vfn/support/platform/linux/mutex.h new file mode 100644 index 00000000..3b84af26 --- /dev/null +++ b/include/vfn/support/platform/linux/mutex.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MUTEX_H +#define LIBVFN_SUPPORT_MUTEX_H + +/* + * DOC: Autolockable mutex + * + * Define a __autolock() macro that will lock the given mutex as well as ensure + * that it is unlocked when going out of scope. This is inspired by the + * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this + * version only supports the pthread_mutex_t. + */ + +static inline pthread_mutex_t *__mutex_auto_lock(pthread_mutex_t *mutex) +{ + pthread_mutex_lock(mutex); + return mutex; +} + +static inline void __mutex_auto_unlock(pthread_mutex_t *mutex) +{ + if (mutex) + pthread_mutex_unlock(mutex); +} + +DEFINE_AUTOPTR(pthread_mutex_t, __mutex_auto_unlock) + +/** + * __autolock - autolock mutex + * @x: pointer to pthread mutex + * + * Lock the mutex and unlock it automatically when going out of scope. + */ +#define __autolock(x) \ + __autoptr(pthread_mutex_t) \ + glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(x) + +#endif /* LIBVFN_SUPPORT_MUTEX_H */ diff --git a/include/vfn/support/platform/macos/byteswap.h b/include/vfn/support/platform/macos/byteswap.h new file mode 100644 index 00000000..123fc537 --- /dev/null +++ b/include/vfn/support/platform/macos/byteswap.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#define bswap_16 __builtin_bswap16 +#define bswap_32 __builtin_bswap32 +#define bswap_64 __builtin_bswap64 diff --git a/include/vfn/support/platform/macos/errno.h b/include/vfn/support/platform/macos/errno.h new file mode 100644 index 00000000..5d63071e --- /dev/null +++ b/include/vfn/support/platform/macos/errno.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_ERRNO_H +#define LIBVFN_SUPPORT_ERRNO_H + +extern "C" { +extern int errno; + +#define ENOENT 2 +#define EIO 5 +#define EBUSY 16 +#define EEXIST 17 +#define EINVAL 22 +#define EAGAIN 35 +#define ETIMEDOUT 60 +} + +#endif diff --git a/include/vfn/support/platform/macos/log.h b/include/vfn/support/platform/macos/log.h new file mode 100644 index 00000000..890037dc --- /dev/null +++ b/include/vfn/support/platform/macos/log.h @@ -0,0 +1,63 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_LOG_H +#define LIBVFN_SUPPORT_LOG_H + +#include +#include +#include + +extern struct log_state { + unsigned int v; +} __log_state; + +enum __log_level { + LOG_ERROR, + LOG_INFO, + LOG_DEBUG, +}; + +/** + * logv - Determine if log verbosity is as given + * @v: target verbositry level + * + * Return: ``true`` if verbosity is at least @v, ``false`` otherwise. + */ +static inline bool logv(unsigned int v) +{ + if (atomic_load_acquire(&__log_state.v) >= v) + return true; + + return false; +} + +/** + * logv_set - Set log verbosity level + * @v: verbosity level + */ +static inline void logv_set(unsigned int v) +{ + atomic_store_release(&__log_state.v, v); +} + +#ifdef DEBUG +#define log_error(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN ERROR: " fmt "\n", ##__VA_ARGS__) +#define log_info(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN INFO: " fmt "\n", ##__VA_ARGS__) +#define log_debug(fmt, ...) os_log(OS_LOG_DEFAULT, "MacVFN DEBUG: " fmt "\n", ##__VA_ARGS__) +#else +#define log_error(fmt, ...) +#define log_info(fmt, ...) +#define log_debug(fmt, ...) +#endif /* DEBUG */ + +#endif /* LIBVFN_SUPPORT_LOG_H */ diff --git a/include/vfn/support/platform/macos/mem.h b/include/vfn/support/platform/macos/mem.h new file mode 100644 index 00000000..ff6eefca --- /dev/null +++ b/include/vfn/support/platform/macos/mem.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + + +#include +#include + +extern size_t __VFN_PAGESIZE; +extern int __VFN_PAGESHIFT; + +void backtrace_abort(void); + +static inline void __do_autofree(void *mem) +{ + free(mem); +} + +#define __autofree __attribute__((cleanup(__do_autofree))) + +static inline bool would_overflow(unsigned int n, size_t sz) +{ + return n > SIZE_MAX / sz; +} + +static inline void *zmalloc(size_t sz) +{ + void *mem; + + if (unlikely(!sz)) + return NULL; + + sz += sizeof(uint64_t); + mem = IOMallocZero(sz); + if (unlikely(!mem)) { + log_debug("IOMallocZero failed!"); + backtrace_abort(); + } + + *(uint64_t *)mem = sz; // Save length of malloc -- used on free. + return (void *)(((uint8_t *)mem) + sizeof(uint64_t)); +} + +void *mallocn(unsigned int n, size_t sz); + +static inline void *zmallocn(unsigned int n, size_t sz) +{ + return zmalloc(n * sz); +} + +inline void free(void *ptr) +{ + void *base_ptr = (void *)(((uint8_t *)ptr)-sizeof(uint64_t)); + uint64_t length = *(uint64_t *)base_ptr; + + IOFree(base_ptr, length); +} diff --git a/include/vfn/support/platform/macos/mmio.h b/include/vfn/support/platform/macos/mmio.h new file mode 100644 index 00000000..48a60310 --- /dev/null +++ b/include/vfn/support/platform/macos/mmio.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MMIO_H +#define LIBVFN_SUPPORT_MMIO_H + +struct macvfn_pci_map_bar { + struct driverkit_pci_device *pci; + unsigned int idx; + size_t len; + uint64_t offset; +}; + +/** + * mmio_read32 - read 4 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Return: read value (native endian) + */ +static inline leint32_t mmio_read32(void *base_addr, uint64_t offset) +{ + /* memory-mapped register */ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + uint32_t val; + + mapping->pci->dev->MemoryRead32(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, &val); + + return (leint32_t __force)(val); +} + +/** + * mmio_lh_read64 - read 8 bytes in memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * + * Read the low 4 bytes first, then the high 4 bytes. + * + * Return: read value (native endian) + */ +static inline leint64_t mmio_lh_read64(void *base_addr, uint64_t offset) +{ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + uint64_t val; + + mapping->pci->dev->MemoryRead64(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, &val); + + return (leint64_t __force)(val); +} + +#define mmio_read64(base_addr, offset) mmio_lh_read64(base_addr, offset) + +/** + * mmio_write32 - write 4 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + */ +static inline void mmio_write32(void *base_addr, uint64_t offset, leint32_t v) +{ + /* memory-mapped register */ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) base_addr; + + mapping->pci->dev->MemoryWrite32(mapping->pci->bar_region_info[mapping->idx].memory_index, + mapping->offset + offset, v); +} + +/** + * mmio_lh_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (low bytes + * first, then high). + */ +static inline void mmio_lh_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32(base_addr, offset, (leint32_t __force)v); + mmio_write32((uint8_t *)base_addr, offset + 4, (leint32_t __force)(v >> 32)); +} + +/** + * mmio_hl_write64 - write 8 bytes to memory-mapped register + * @base_addr: base address of memory-mapped register + * @offset: offset into the base address + * @v: value to write (native endian) + * + * Write 8 bytes to memory-mapped register as two 4 byte writes (high bytes + * first, then low). + */ +static inline void mmio_hl_write64(void *base_addr, uint64_t offset, leint64_t v) +{ + mmio_write32((uint8_t *)base_addr, offset + 4, + (leint32_t __force)((uint64_t __force)v >> 32)); + mmio_write32(base_addr, offset, (leint32_t __force)v); +} + +#endif /* LIBVFN_SUPPORT_MMIO_H */ diff --git a/include/vfn/support/platform/macos/mutex.h b/include/vfn/support/platform/macos/mutex.h new file mode 100644 index 00000000..a3d2a666 --- /dev/null +++ b/include/vfn/support/platform/macos/mutex.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: LGPL-2.1-or-later or MIT */ + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifndef LIBVFN_SUPPORT_MUTEX_H +#define LIBVFN_SUPPORT_MUTEX_H + +/* + * Autolockable mutex + * + * Define a __autolock() macro that will lock the given mutex as well as ensure + * that it is unlocked when going out of scope. This is inspired by the + * polymorphic locking functions in QEMU (include/qemu/lockable.h), but this + * version only supports the IOLock. + */ + +static inline IOLock *__mutex_auto_lock(IOLock *mutex) +{ + IOLockLock(mutex); + return mutex; +} + +static inline void __mutex_auto_unlock(IOLock *mutex) +{ + if (mutex) + IOLockUnlock(mutex); +} + +DEFINE_AUTOPTR(IOLock, __mutex_auto_unlock) + +#define __autolock(x) \ + __autoptr(IOLock) \ + glue(autolock, __COUNTER__) __attribute__((__unused__)) = __mutex_auto_lock(*x) + +#endif /* LIBVFN_SUPPORT_MUTEX_H */ diff --git a/include/vfn/trace.h b/include/vfn/trace.h index 59dd981b..0fdf8bec 100644 --- a/include/vfn/trace.h +++ b/include/vfn/trace.h @@ -13,9 +13,11 @@ #ifndef LIBVFN_TRACE_H #define LIBVFN_TRACE_H +#ifndef __APPLE__ #include +#endif -#ifdef DEBUG +#if defined(DEBUG) && !defined(__APPLE__) extern __thread const char *__trace_event; # define __trace_prefix(fmt) "T %s (%s:%d) " fmt diff --git a/src/driverkit/pci.c b/src/driverkit/pci.c new file mode 100644 index 00000000..fe2cb74c --- /dev/null +++ b/src/driverkit/pci.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + + + +#define log_fmt(fmt) "driverkit/pci: " fmt + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// NOTE: Faux open for macOS +int vfio_pci_open(struct driverkit_pci_device *pci, const char *bdf) +{ + return 0; +} + +void *vfio_pci_map_bar(struct driverkit_pci_device *pci, int idx, size_t len, uint64_t offset, + int prot) +{ + struct macvfn_pci_map_bar *mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + mapping->pci = pci; + mapping->idx = idx; + mapping->len = len; + mapping->offset = offset; + + return mapping; +} + +void vfio_pci_unmap_bar(struct driverkit_pci_device *pci, int idx, void *mem, size_t len, + uint64_t offset) +{ + free(mem); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/iommu/context.c b/src/iommu/context.c index b8c331e7..79b0261c 100644 --- a/src/iommu/context.c +++ b/src/iommu/context.c @@ -12,12 +12,6 @@ #define log_fmt(fmt) "iommu/context: " fmt -#include -#include -#include - -#include - #include "vfn/iommu.h" #include "vfn/support.h" @@ -51,11 +45,16 @@ struct iommu_ctx *iommu_get_default_context(void) fallback: #endif +#ifdef __APPLE__ + return driverkit_get_default_iommu_context(); +#else return vfio_get_default_iommu_context(); +#endif } struct iommu_ctx *iommu_get_context(const char *name) { +#ifndef __APPLE__ #ifdef HAVE_VFIO_DEVICE_BIND_IOMMUFD if (__iommufd_broken) goto fallback; @@ -65,8 +64,12 @@ struct iommu_ctx *iommu_get_context(const char *name) fallback: #endif return vfio_get_iommu_context(name); +#else + return driverkit_get_iommu_context(name); +#endif } +#ifndef __APPLE__ void iommu_ctx_init(struct iommu_ctx *ctx) { ctx->nranges = 1; @@ -84,3 +87,4 @@ void iommu_ctx_init(struct iommu_ctx *ctx) skiplist_init(&ctx->map.list); pthread_mutex_init(&ctx->map.lock, NULL); } +#endif diff --git a/src/iommu/context.h b/src/iommu/context.h index d260b561..97d8f034 100644 --- a/src/iommu/context.h +++ b/src/iommu/context.h @@ -10,36 +10,48 @@ * COPYING and LICENSE files for more information. */ +#ifndef LIBVFN_SRC_IOMMU_CONTEXT_H +#define LIBVFN_SRC_IOMMU_CONTEXT_H + + +#ifdef __cplusplus +extern "C" { +#endif #include "util/skiplist.h" + struct iommu_ctx; +struct iova_mapping { + void *vaddr; + size_t len; + uint64_t iova; + void *opaque[2]; + + unsigned long flags; + + struct skiplist_node list; +}; + struct iommu_ctx_ops { /* container/ioas ops */ int (*iova_reserve)(struct iommu_ctx *ctx, size_t len, uint64_t *iova, unsigned long flags); void (*iova_put_ephemeral)(struct iommu_ctx *ctx); - int (*dma_map)(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags); - int (*dma_unmap)(struct iommu_ctx *ctx, uint64_t iova, size_t len); + int (*dma_map)(struct iommu_ctx *ctx, struct iova_mapping *m); + int (*dma_unmap)(struct iommu_ctx *ctx, struct iova_mapping *m); int (*dma_unmap_all)(struct iommu_ctx *ctx); /* device ops */ int (*get_device_fd)(struct iommu_ctx *ctx, const char *bdf); }; -struct iova_mapping { - void *vaddr; - size_t len; - uint64_t iova; - - unsigned long flags; - - struct skiplist_node list; -}; - struct iova_map { + #ifndef __APPLE__ pthread_mutex_t lock; + #else + IOLock * lock; + #endif struct skiplist list; }; @@ -47,13 +59,19 @@ struct iommu_ctx { struct iova_map map; struct iommu_ctx_ops ops; + #ifndef __APPLE__ pthread_mutex_t lock; int nranges; struct iommu_iova_range *iova_ranges; + #else + IOPCIDevice *pci; + IOLock *lock; + #endif }; struct iommu_ctx *iommu_get_default_context(void); +#ifndef __APPLE__ struct iommu_ctx *vfio_get_default_iommu_context(void); struct iommu_ctx *vfio_get_iommu_context(const char *name); @@ -61,6 +79,18 @@ struct iommu_ctx *vfio_get_iommu_context(const char *name); struct iommu_ctx *iommufd_get_default_iommu_context(void); struct iommu_ctx *iommufd_get_iommu_context(const char *name); #endif +#else +struct iommu_ctx *driverkit_get_default_iommu_context(void); +struct iommu_ctx *driverkit_get_iommu_context(const char *name); +#endif void iommu_ctx_init(struct iommu_ctx *ctx); +#ifndef __APPLE__ int iommu_iova_range_to_string(struct iommu_iova_range *range, char **str); +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* LIBVFN_SRC_IOMMU_CONTEXT_H */ diff --git a/src/iommu/dma.c b/src/iommu/dma.c index 3aa7b289..5a15c25b 100644 --- a/src/iommu/dma.c +++ b/src/iommu/dma.c @@ -12,11 +12,9 @@ #define log_fmt(fmt) "iommu/dma: " fmt -#include -#include -#include -#include -#include +#ifdef __cplusplus +extern "C" { +#endif #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" @@ -32,37 +30,28 @@ static int iova_cmp(const void *vaddr, const struct skiplist_node *n) if (vaddr < m->vaddr) return -1; - else if (vaddr >= m->vaddr + m->len) + else if ((uintptr_t)vaddr >= ((uintptr_t)m->vaddr) + m->len) return 1; return 0; } -static int iova_map_add(struct iova_map *map, void *vaddr, size_t len, uint64_t iova, - unsigned long flags) +static int iova_map_add(struct iova_map *map, struct iova_mapping *m) { __autolock(&map->lock); struct skiplist_node *update[SKIPLIST_LEVELS] = {}; - struct iova_mapping *m; - if (!len) { + if (!m->len) { errno = EINVAL; return -1; } - if (skiplist_find(&map->list, vaddr, iova_cmp, update)) { + if (skiplist_find(&map->list, m->vaddr, iova_cmp, update)) { errno = EEXIST; return -1; } - m = znew_t(struct iova_mapping, 1); - - m->vaddr = vaddr; - m->len = len; - m->iova = iova; - m->flags = flags; - skiplist_link(&map->list, &m->list, update); return 0; @@ -106,38 +95,49 @@ bool iommu_translate_vaddr(struct iommu_ctx *ctx, void *vaddr, uint64_t *iova) struct iova_mapping *m = iova_map_find(&ctx->map, vaddr); if (m) { - *iova = m->iova + (vaddr - m->vaddr); + *iova = m->iova + ((uintptr_t)vaddr - (uintptr_t)m->vaddr); return true; } return false; } -int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags) +int _iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags, void *opaque) { uint64_t _iova; - + struct iova_mapping *m; if (iommu_translate_vaddr(ctx, vaddr, &_iova)) goto out; if (flags & IOMMU_MAP_FIXED_IOVA) { _iova = *iova; } else if (ctx->ops.iova_reserve && ctx->ops.iova_reserve(ctx, len, &_iova, flags)) { - log_debug("failed to allocate iova\n"); + log_error("failed to allocate iova\n"); return -1; } - if (ctx->ops.dma_map(ctx, vaddr, len, &_iova, flags)) { - log_debug("failed to map dma\n"); + m = znew_t(struct iova_mapping, 1); + m->vaddr = vaddr; + m->len = len; + m->iova = 0; // Physical/IOMMU address + m->opaque[0] = NULL; // IODMACommand + m->opaque[1] = opaque; // IOMemoryDescriptor + m->flags = flags; + + if (ctx->ops.dma_map(ctx, m)) { + log_error("failed to map dma\n"); + free(m); return -1; } - if (iova_map_add(&ctx->map, vaddr, len, _iova, flags)) { - log_debug("failed to add mapping\n"); + if (iova_map_add(&ctx->map, m)) { + log_error("failed to add mapping\n"); + free(m); return -1; } + _iova = m->iova; out: if (iova) *iova = _iova; @@ -145,6 +145,14 @@ int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *io return 0; } +#ifndef __APPLE__ +int iommu_map_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, + unsigned long flags) +{ + return _iommu_map_vaddr(ctx, vaddr, len, iova, flags, NULL); +} +#endif + int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) { struct iova_mapping *m; @@ -158,8 +166,8 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) if (len) *len = m->len; - if (ctx->ops.dma_unmap(ctx, m->iova, m->len)) { - log_debug("failed to unmap dma\n"); + if (ctx->ops.dma_unmap(ctx, m)) { + log_error("failed to unmap dma\n"); return -1; } @@ -173,10 +181,10 @@ int iommu_unmap_vaddr(struct iommu_ctx *ctx, void *vaddr, size_t *len) static void __unmap_mapping(void *opaque, struct skiplist_node *n) { - struct iommu_ctx *ctx = opaque; + struct iommu_ctx *ctx = (struct iommu_ctx *)opaque; struct iova_mapping *m = container_of_var(n, m, list); - log_fatal_if(ctx->ops.dma_unmap(ctx, m->len, m->iova), + log_fatal_if(ctx->ops.dma_unmap(ctx, m), "failed to unmap dma (iova 0x%" PRIx64 " len %zu)\n", m->iova, m->len); free(m); @@ -186,7 +194,7 @@ int iommu_unmap_all(struct iommu_ctx *ctx) { if (ctx->ops.dma_unmap_all) { if (ctx->ops.dma_unmap_all(ctx)) { - log_debug("failed to unmap dma\n"); + log_error("failed to unmap dma\n"); return -1; } @@ -200,6 +208,7 @@ int iommu_unmap_all(struct iommu_ctx *ctx) return 0; } +#ifndef __APPLE__ int iommu_get_iova_ranges(struct iommu_ctx *ctx, struct iommu_iova_range **ranges) { *ranges = ctx->iova_ranges; @@ -210,3 +219,8 @@ int iommu_iova_range_to_string(struct iommu_iova_range *r, char **str) { return asprintf(str, "[0x%llx; 0x%llx]", r->start, r->last); } +#endif + +#ifdef __cplusplus +} +#endif diff --git a/src/iommu/driverkit.c b/src/iommu/driverkit.c new file mode 100644 index 00000000..9cc9ff11 --- /dev/null +++ b/src/iommu/driverkit.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2023 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#define log_fmt(fmt) "iommu/vfio: " fmt + +#include "ccan/str/str.h" +#include "ccan/compiler/compiler.h" +#include "ccan/minmax/minmax.h" + +#include "vfn/driverkit.h" +#include "vfn/support.h" +#include "vfn/iommu.h" +#include "vfn/trace.h" +#include "vfn/pci/util.h" + +#include + +#include "context.h" + +static int driverkit_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + kern_return_t ret; + + IODMACommand **dmaCommand = (IODMACommand **) &m->opaque[0]; + + log_debug("DriverKit DMA mapping (PrepareForDMA) %p %zu, %p, %p", m->vaddr, m->len, + m->opaque[0], m->opaque[1]); + IODMACommandSpecification dmaSpecification = { + .options = kIODMACommandSpecificationNoOptions, + .maxAddressBits = 64, + }; + struct driverkit_pci_device *dev = container_of(ctx, struct driverkit_pci_device, ctx); + + ret = IODMACommand::Create(dev->dev, kIODMACommandCreateNoOptions, &dmaSpecification, + dmaCommand); + if (ret != kIOReturnSuccess) { + log_error("IODMACommand::Create failed with error code: %x", ret); + return -1; + } + + if (!m->opaque[1]) { + log_error("DriverKit DMA mapping: opaque[1] is NULL"); + return -1; + } + + uint64_t dmaFlags = 0; + uint32_t dmaSegmentCount = 1; + IOAddressSegment physicalAddressSegment; + + ret = (*dmaCommand)->PrepareForDMA( + kIODMACommandPrepareForDMANoOptions, + (IOMemoryDescriptor *) m->opaque[1], + 0, + m->len, + &dmaFlags, + &dmaSegmentCount, + &physicalAddressSegment + ); + if (ret != kIOReturnSuccess) { + log_error("failed to PrepareForDMA %x", ret); + return -1; + } + if (dmaSegmentCount != 1) { + log_error("dmaSegmentCount not 1! %u", dmaSegmentCount); + return -1; + } + assert(m->len == physicalAddressSegment.length); + m->iova = physicalAddressSegment.address; + + log_debug("DriverKit DMA mapping: iova %llx", m->iova); + + return 0; +} + +static int driverkit_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) +{ + log_debug("DriverKit DMA unmapping (CompleteDMA) %p %zu", m->vaddr, m->len); + IODMACommand *dmaCommand = (IODMACommand *) m->opaque[0]; + kern_return_t ret = (int) dmaCommand->CompleteDMA(kIODMACommandCompleteDMANoOptions); + + if (ret != kIOReturnSuccess) { + log_error("DriverKit DMA unmapping failed to complete DMA"); + return -1; + } + OSSafeReleaseNULL(dmaCommand); + m->opaque[0] = NULL; + return 0; +} + +static const struct iommu_ctx_ops driverkit_ops = { + .get_device_fd = NULL, + + .iova_reserve = NULL, + .iova_put_ephemeral = NULL, + + .dma_map = driverkit_dma_map, + .dma_unmap = driverkit_dma_unmap, +}; + +void iommu_ctx_init(struct iommu_ctx *ctx) +{ + memcpy(&ctx->ops, &driverkit_ops, sizeof(ctx->ops)); + ctx->lock = IOLockAlloc(); + ctx->map.lock = IOLockAlloc(); + skiplist_init(&ctx->map.list); +} + +struct iommu_ctx *driverkit_get_iommu_context(const char *name) +{ + return NULL; +} + +struct iommu_ctx *driverkit_get_default_iommu_context(void) +{ + return NULL; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/iommu/iommufd.c b/src/iommu/iommufd.c index 8213bddd..956f9a4c 100644 --- a/src/iommu/iommufd.c +++ b/src/iommu/iommufd.c @@ -12,30 +12,17 @@ #define log_fmt(fmt) "iommu/iommufd: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include -#include -#include - -#include #include -#include #include "vfn/trace.h" #include "vfn/support.h" #include "vfn/pci.h" #include "vfn/iommu.h" +#include "ccan/compiler/compiler.h" #include "ccan/list/list.h" #include "context.h" @@ -64,7 +51,7 @@ static int iommu_ioas_update_iova_ranges(struct iommu_ioas *ioas) if (ioctl(__iommufd, IOMMU_IOAS_IOVA_RANGES, &iova_ranges)) { if (errno != EMSGSIZE) { - log_debug("could not get ioas iova ranges\n"); + log_error("could not get ioas iova ranges\n"); return -1; } @@ -75,7 +62,7 @@ static int iommu_ioas_update_iova_ranges(struct iommu_ioas *ioas) iova_ranges.allowed_iovas = (uintptr_t)ioas->ctx.iova_ranges; if (ioctl(__iommufd, IOMMU_IOAS_IOVA_RANGES, &iova_ranges)) { - log_debug("could not get ioas iova ranges\n"); + log_error("could not get ioas iova ranges\n"); return -1; } } @@ -117,33 +104,33 @@ static int iommufd_get_device_fd(struct iommu_ctx *ctx, const char *bdf) vfio_id = pci_get_device_vfio_id(bdf); if (!vfio_id) { - log_debug("could not determine the vfio device id for %s\n", bdf); + log_error("could not determine the vfio device id for %s\n", bdf); return -1; } if (asprintf(&path, "/dev/vfio/devices/%s", vfio_id) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } devfd = open(path, O_RDWR); if (devfd < 0) { - log_debug("could not open the device cdev\n"); + log_error("could not open the device cdev\n"); return -1; } if (ioctl(devfd, VFIO_DEVICE_BIND_IOMMUFD, &bind)) { - log_debug("could not bind device to iommufd\n"); + log_error("could not bind device to iommufd\n"); goto close_dev; } if (ioctl(devfd, VFIO_DEVICE_ATTACH_IOMMUFD_PT, &attach_data)) { - log_debug("could not associate device with ioas\n"); + log_error("could not associate device with ioas\n"); goto close_dev; } if (iommu_ioas_update_iova_ranges(ioas)) { - log_debug("could not update iova ranges\n"); + log_error("could not update iova ranges\n"); goto close_dev; } @@ -156,8 +143,7 @@ static int iommufd_get_device_fd(struct iommu_ctx *ctx, const char *bdf) return -1; } -static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, uint64_t *iova, - unsigned long flags) +static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) { struct iommu_ioas *ioas = container_of_var(ctx, ioas, ctx); @@ -165,48 +151,51 @@ static int iommu_ioas_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, .size = sizeof(map), .flags = IOMMU_IOAS_MAP_READABLE | IOMMU_IOAS_MAP_WRITEABLE, .ioas_id = ioas->id, - .user_va = (uint64_t)vaddr, - .length = len, + .user_va = (uint64_t)m->vaddr, + .length = m->len, }; - if (flags & IOMMU_MAP_FIXED_IOVA) { + if (m->flags & IOMMU_MAP_FIXED_IOVA) { map.flags |= IOMMU_IOAS_MAP_FIXED_IOVA; - map.iova = *iova; + map.iova = m->iova; } - if (flags & IOMMU_MAP_NOWRITE) + if (m->flags & IOMMU_MAP_NOWRITE) map.flags &= ~IOMMU_IOAS_MAP_WRITEABLE; - if (flags & IOMMU_MAP_NOREAD) + if (m->flags & IOMMU_MAP_NOREAD) map.flags &= ~IOMMU_IOAS_MAP_READABLE; trace_guard(IOMMUFD_IOAS_MAP_DMA) { - if (flags & IOMMU_MAP_FIXED_IOVA) - trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", vaddr, *iova, len); + if (m->flags & IOMMU_MAP_FIXED_IOVA) + trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", m->vaddr, m->iova, + m->len); else - trace_emit("vaddr %p iova AUTO len %zu\n", vaddr, len); + trace_emit("vaddr %p iova AUTO len %zu\n", m->vaddr, m->len); } if (ioctl(__iommufd, IOMMU_IOAS_MAP, &map)) { - log_debug("failed to map\n"); + log_error("failed to map\n"); return -1; } - if (flags & IOMMU_MAP_FIXED_IOVA) + if (m->flags & IOMMU_MAP_FIXED_IOVA) return 0; - *iova = map.iova; + m->iova = map.iova; trace_guard(IOMMUFD_IOAS_MAP_DMA) { - trace_emit("allocated iova 0x%" PRIx64 "\n", *iova); + trace_emit("allocated iova 0x%" PRIx64 "\n", m->iova); } return 0; } -static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t len) +static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) { struct iommu_ioas *ioas = container_of_var(ctx, ioas, ctx); + uint64_t iova = m->iova; + size_t len = m->len; struct iommu_ioas_unmap unmap = { .size = sizeof(unmap), @@ -220,7 +209,7 @@ static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t } if (ioctl(__iommufd, IOMMU_IOAS_UNMAP, &unmap)) { - log_debug("failed to unmap\n"); + log_error("failed to unmap\n"); return -1; } @@ -229,7 +218,11 @@ static int iommu_ioas_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t static int iommu_ioas_do_dma_unmap_all(struct iommu_ctx *ctx) { - return iommu_ioas_do_dma_unmap(ctx, 0, UINT64_MAX); + struct iova_mapping m = { + .iova = 0, + .len = UINT64_MAX + }; + return iommu_ioas_do_dma_unmap(ctx, &m); } static const struct iommu_ctx_ops iommufd_ops = { @@ -248,7 +241,7 @@ static int iommu_ioas_init(struct iommu_ioas *ioas) }; if (ioctl(__iommufd, IOMMU_IOAS_ALLOC, &alloc_data)) { - log_debug("could not allocate ioas\n"); + log_error("could not allocate ioas\n"); return -1; } diff --git a/src/iommu/vfio.c b/src/iommu/vfio.c index 3f497922..4f91e1db 100644 --- a/src/iommu/vfio.c +++ b/src/iommu/vfio.c @@ -12,30 +12,11 @@ #define log_fmt(fmt) "iommu/vfio: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include - #include "ccan/str/str.h" #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" -#include -#include - #include "vfn/support.h" #include "vfn/iommu.h" #include "vfn/trace.h" @@ -221,7 +202,7 @@ static int vfio_iommu_type1_iova_reserve(struct iommu_ctx *ctx, size_t len, uint __autolock(&vfio->lock); if (!ALIGNED(len, __VFN_PAGESIZE)) { - log_debug("len is not page aligned\n"); + log_error("len is not page aligned\n"); errno = EINVAL; return -1; } @@ -251,7 +232,7 @@ static int vfio_iommu_type1_init(struct vfio_container *vfio) return 0; if (ioctl(vfio->fd, VFIO_SET_IOMMU, VFIO_TYPE1_IOMMU)) { - log_debug("failed to set vfio iommu type\n"); + log_error("failed to set vfio iommu type\n"); return -1; } @@ -259,13 +240,13 @@ static int vfio_iommu_type1_init(struct vfio_container *vfio) #ifdef VFIO_IOMMU_INFO_CAPS if (vfio_iommu_type1_get_capabilities(vfio)) { - log_debug("failed to get iommu capabilities\n"); + log_error("failed to get iommu capabilities\n"); return -1; } #endif if (vfio_iommu_type1_iova_reserve(&vfio->ctx, VFIO_IOMMU_TYPE1_IOVA_RESERVED, &iova, 0x0)) { - log_debug("could not reserve iova range\n"); + log_error("could not reserve iova range\n"); return -1; } @@ -289,12 +270,12 @@ static int vfio_group_set_container(struct vfio_group *group, struct vfio_contai log_info("adding group '%s' to container\n", group->path); if (ioctl(group->fd, VFIO_GROUP_SET_CONTAINER, &vfio->fd)) { - log_debug("failed to add group to vfio container\n"); + log_error("failed to add group to vfio container\n"); return -1; } if (vfio_iommu_type1_init(vfio)) { - log_debug("failed to configure iommu\n"); + log_error("failed to configure iommu\n"); log_fatal_if(ioctl(group->fd, VFIO_GROUP_UNSET_CONTAINER), "unset container\n"); @@ -314,18 +295,18 @@ static int vfio_group_open(const char *path) fd = open(path, O_RDWR); if (fd < 0) { - log_debug("failed to open vfio group file: %s\n", strerror(errno)); + log_error("failed to open vfio group file: %s\n", strerror(errno)); return -1; } if (ioctl(fd, VFIO_GROUP_GET_STATUS, &group_status)) { - log_debug("failed to get vfio group status\n"); + log_error("failed to get vfio group status\n"); goto close_fd; } if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { errno = EINVAL; - log_debug("vfio group is not viable\n"); + log_error("vfio group is not viable\n"); goto close_fd; } @@ -367,7 +348,7 @@ static int vfio_get_group_fd(struct vfio_container *vfio, const char *path) group->fd = vfio_group_open(group->path); if (group->fd < 0) { - log_debug("failed to open vfio group\n"); + log_error("failed to open vfio group\n"); goto free_group_path; } @@ -393,7 +374,7 @@ static int vfio_get_device_fd(struct iommu_ctx *ctx, const char *bdf) group = pci_get_iommu_group(bdf); if (!group) { - log_debug("could not determine iommu group for device %s\n", bdf); + log_error("could not determine iommu group for device %s\n", bdf); errno = EINVAL; return -1; } @@ -405,53 +386,54 @@ static int vfio_get_device_fd(struct iommu_ctx *ctx, const char *bdf) ret_fd = ioctl(gfd, VFIO_GROUP_GET_DEVICE_FD, bdf); if (ret_fd < 0) { - log_debug("failed to get device fd\n"); + log_error("failed to get device fd\n"); return -1; } return ret_fd; } -static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, void *vaddr, size_t len, - uint64_t *iova, unsigned long flags) +static int vfio_iommu_type1_do_dma_map(struct iommu_ctx *ctx, struct iova_mapping *m) { struct vfio_container *vfio = container_of_var(ctx, vfio, ctx); struct vfio_iommu_type1_dma_map dma_map = { .argsz = sizeof(dma_map), - .vaddr = (uintptr_t)vaddr, - .size = len, - .iova = *iova, + .vaddr = (uintptr_t)m->vaddr, + .size = m->len, + .iova = m->iova, .flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE, }; - if (flags & IOMMU_MAP_NOWRITE) + if (m->flags & IOMMU_MAP_NOWRITE) dma_map.flags &= ~VFIO_DMA_MAP_FLAG_WRITE; - if (flags & IOMMU_MAP_NOREAD) + if (m->flags & IOMMU_MAP_NOREAD) dma_map.flags &= ~VFIO_DMA_MAP_FLAG_READ; trace_guard(VFIO_IOMMU_TYPE1_MAP_DMA) { - trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", vaddr, *iova, len); + trace_emit("vaddr %p iova 0x%" PRIx64 " len %zu\n", m->vaddr, m->iova, m->len); } - if (!ALIGNED(((uintptr_t)vaddr | len | *iova), __VFN_PAGESIZE)) { - log_debug("vaddr, len or iova not page aligned\n"); + if (!ALIGNED(((uintptr_t)m->vaddr | m->len | m->iova), __VFN_PAGESIZE)) { + log_error("vaddr, len or iova not page aligned\n"); errno = EINVAL; return -1; } if (ioctl(vfio->fd, VFIO_IOMMU_MAP_DMA, &dma_map)) { - log_debug("could not map\n"); + log_error("could not map\n"); return -1; } return 0; } -static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, size_t len) +static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, struct iova_mapping *m) { struct vfio_container *vfio = container_of_var(ctx, vfio, ctx); + uint64_t iova = m->iova; + size_t len = m->len; struct vfio_iommu_type1_dma_unmap dma_unmap = { .argsz = sizeof(dma_unmap), @@ -464,7 +446,7 @@ static int vfio_iommu_type1_do_dma_unmap(struct iommu_ctx *ctx, uint64_t iova, s } if (ioctl(vfio->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap)) { - log_debug("could not unmap\n"); + log_error("could not unmap\n"); return -1; } @@ -502,7 +484,7 @@ static int vfio_iommu_type1_do_dma_unmap_all(struct iommu_ctx *ctx) }; if (ioctl(vfio->fd, VFIO_IOMMU_UNMAP_DMA, &dma_unmap)) { - log_debug("failed to unmap dma\n"); + log_error("failed to unmap dma\n"); return -1; } @@ -527,17 +509,17 @@ static int vfio_init_container(struct vfio_container *vfio) { vfio->fd = open("/dev/vfio/vfio", O_RDWR); if (vfio->fd < 0) { - log_debug("failed to open vfio device\n"); + log_error("failed to open vfio device\n"); return -1; } if (ioctl(vfio->fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) { - log_debug("invalid vfio version\n"); + log_error("invalid vfio version\n"); return -1; } if (!ioctl(vfio->fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU)) { - log_debug("vfio type 1 iommu not supported\n"); + log_error("vfio type 1 iommu not supported\n"); return -1; } diff --git a/src/nvme/core.c b/src/nvme/core.c index 96073c3e..81dd17d0 100644 --- a/src/nvme/core.c +++ b/src/nvme/core.c @@ -12,42 +12,57 @@ #define log_fmt(fmt) "nvme/core: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include -#include -#include #include +#include +#include #include "ccan/compiler/compiler.h" #include "ccan/minmax/minmax.h" +#ifndef __APPLE__ #include "ccan/time/time.h" +#else +#include +int errno; +#endif #include "types.h" +#ifdef __APPLE__ +inline void *cqhdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar *) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid + 1) * (4 << dstrd); + + return new_mapping; +} + +inline void *sqtdbl(void *doorbells, int qid, int dstrd) +{ + struct macvfn_pci_map_bar *doorbell_mapping = (struct macvfn_pci_map_bar *) doorbells; + struct macvfn_pci_map_bar *new_mapping = (struct macvfn_pci_map_bar *) zmallocn(1, + sizeof(struct macvfn_pci_map_bar)); + + new_mapping->pci = doorbell_mapping->pci; + new_mapping->idx = doorbell_mapping->idx; + new_mapping->len = doorbell_mapping->len; + new_mapping->offset = doorbell_mapping->offset + (2 * qid) * (4 << dstrd); + + return new_mapping; +} +#else #define cqhdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid + 1) * (4 << dstrd)) #define sqtdbl(doorbells, qid, dstrd) \ (doorbells + (2 * qid) * (4 << dstrd)) +#endif + enum nvme_ctrl_feature_flags { NVME_CTRL_F_ADMINISTRATIVE = 1 << 0, @@ -59,19 +74,20 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec uint64_t cap; uint8_t dstrd; size_t len; + void *opaque; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.ncqa + 1) { - log_debug("qid %d invalid; max qid is %d\n", qid, ctrl->config.ncqa + 1); + log_error("qid %d invalid; max qid is %d\n", qid, ctrl->config.ncqa + 1); errno = EINVAL; return -1; } if (qsize < 2) { - log_debug("qsize must be at least 2\n"); + log_error("qsize must be at least 2\n"); errno = EINVAL; return -1; } @@ -92,12 +108,12 @@ static int nvme_configure_cq(struct nvme_ctrl *ctrl, int qid, int qsize, int vec cq->dbbuf.eventidx = cqhdbl(ctrl->dbbuf.eventidxs, qid, dstrd); } - len = pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES); + len = __pgmapn(&cq->vaddr, qsize, 1 << NVME_CQES, &opaque); - if (iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0)) { - log_debug("failed to map vaddr\n"); + if (_iommu_map_vaddr(__iommu_ctx(ctrl), cq->vaddr, len, &cq->iova, 0x0, opaque)) { + log_error("failed to map vaddr\n"); - pgunmap(cq->vaddr, len); + __pgunmap(cq->vaddr, len, opaque); return -1; } @@ -112,9 +128,9 @@ static void nvme_discard_cq(struct nvme_ctrl *ctrl, struct nvme_cq *cq) return; if (iommu_unmap_vaddr(__iommu_ctx(ctrl), cq->vaddr, &len)) - log_debug("failed to unmap vaddr\n"); + log_error("failed to unmap vaddr\n"); - pgunmap(cq->vaddr, len); + __pgunmap(cq->vaddr, len, cq->vaddr_opaque); if (ctrl->dbbuf.doorbells) { __STORE_PTR(uint32_t *, cq->dbbuf.doorbell, 0); @@ -132,18 +148,18 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, uint8_t dstrd; ssize_t len; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); if (qid && qid > ctrl->config.nsqa + 1) { - log_debug("qid %d invalid; max qid is %d\n", qid, ctrl->config.nsqa + 1); + log_error("qid %d invalid; max qid is %d\n", qid, ctrl->config.nsqa + 1); errno = EINVAL; return -1; } if (qsize < 2) { - log_debug("qsize must be at least 2\n"); + log_error("qsize must be at least 2\n"); errno = EINVAL; return -1; } @@ -168,13 +184,15 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, * Use ctrl->config.mps instead of host page size, as we have the * opportunity to pack the allocations. */ - len = pgmapn(&sq->pages.vaddr, qsize, __mps_to_pagesize(ctrl->config.mps)); + len = __pgmapn(&sq->pages.vaddr, qsize, __mps_to_pagesize(ctrl->config.mps), + &sq->pages.vaddr_opaque); if (len < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, len, &sq->pages.iova, 0x0)) { - log_debug("failed to map vaddr\n"); + if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, len, &sq->pages.iova, 0x0, + sq->pages.vaddr_opaque)) { + log_error("failed to map vaddr\n"); goto unmap_pages; } @@ -187,33 +205,35 @@ static int nvme_configure_sq(struct nvme_ctrl *ctrl, int qid, int qsize, rq->sq = sq; rq->cid = (uint16_t)i; - rq->page.vaddr = sq->pages.vaddr + (i << __mps_to_pageshift(ctrl->config.mps)); - rq->page.iova = sq->pages.iova + (i << __mps_to_pageshift(ctrl->config.mps)); + rq->page.vaddr = (void *)((uintptr_t)sq->pages.vaddr + + ((uint64_t)i << __mps_to_pageshift(ctrl->config.mps))); + rq->page.iova = sq->pages.iova + + ((uint64_t)i << __mps_to_pageshift(ctrl->config.mps)); if (i > 0) rq->rq_next = &sq->rqs[i - 1]; } - len = pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES); + len = __pgmapn(&sq->vaddr, qsize, 1 << NVME_SQES, &sq->vaddr_opaque); if (len < 0) goto free_sq_rqs; - if (iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0)) { - log_debug("failed to map vaddr\n"); + if (_iommu_map_vaddr(__iommu_ctx(ctrl), sq->vaddr, len, &sq->iova, 0x0, sq->vaddr_opaque)) { + log_error("failed to map vaddr\n"); goto unmap_sq; } return 0; unmap_sq: - pgunmap(sq->vaddr, len); + __pgunmap(sq->vaddr, len, sq->vaddr_opaque); free_sq_rqs: free(sq->rqs); unmap_pages: if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, (size_t *)&len)) - log_debug("failed to unmap vaddr\n"); + log_error("failed to unmap vaddr\n"); - pgunmap(sq->pages.vaddr, len); + __pgunmap(sq->pages.vaddr, len, sq->pages.vaddr_opaque); return -1; } @@ -228,14 +248,14 @@ static void nvme_discard_sq(struct nvme_ctrl *ctrl, struct nvme_sq *sq) if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->vaddr, &len)) log_debug("failed to unmap vaddr\n"); - pgunmap(sq->vaddr, len); + __pgunmap(sq->vaddr, len, sq->vaddr_opaque); free(sq->rqs); if (iommu_unmap_vaddr(__iommu_ctx(ctrl), sq->pages.vaddr, &len)) log_debug("failed to unmap vaddr\n"); - pgunmap(sq->pages.vaddr, len); + __pgunmap(sq->pages.vaddr, len, sq->pages.vaddr_opaque); if (ctrl->dbbuf.doorbells) { __STORE_PTR(uint32_t *, sq->dbbuf.doorbell, 0); @@ -253,12 +273,12 @@ static int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags) struct nvme_sq *sq = &ctrl->sq[NVME_AQ]; if (nvme_configure_cq(ctrl, NVME_AQ, NVME_AQ_QSIZE, 0)) { - log_debug("failed to configure admin completion queue\n"); + log_error("failed to configure admin completion queue\n"); return -1; } if (nvme_configure_sq(ctrl, NVME_AQ, NVME_AQ_QSIZE, cq, sq_flags)) { - log_debug("failed to configure admin submission queue\n"); + log_error("failed to configure admin submission queue\n"); goto discard_cq; } @@ -268,9 +288,9 @@ static int nvme_configure_adminq(struct nvme_ctrl *ctrl, unsigned long sq_flags) aqa = NVME_AQ_QSIZE - 1; aqa |= aqa << 16; - mmio_write32(ctrl->regs + NVME_REG_AQA, cpu_to_le32(aqa)); - mmio_hl_write64(ctrl->regs + NVME_REG_ASQ, cpu_to_le64(sq->iova)); - mmio_hl_write64(ctrl->regs + NVME_REG_ACQ, cpu_to_le64(cq->iova)); + mmio_write32(ctrl->regs, NVME_REG_AQA, cpu_to_le32(aqa)); + mmio_hl_write64(ctrl->regs, NVME_REG_ASQ, cpu_to_le64(sq->iova)); + mmio_hl_write64(ctrl->regs, NVME_REG_ACQ, cpu_to_le64(cq->iova)); return 0; @@ -293,7 +313,7 @@ int nvme_create_iocq(struct nvme_ctrl *ctrl, int qid, int qsize, int vector) uint16_t iv = 0; if (nvme_configure_cq(ctrl, qid, qsize, vector)) { - log_debug("could not configure io completion queue\n"); + log_error("could not configure io completion queue\n"); return -1; } @@ -335,7 +355,7 @@ int nvme_create_iosq(struct nvme_ctrl *ctrl, int qid, int qsize, struct nvme_cq union nvme_cmd cmd; if (nvme_configure_sq(ctrl, qid, qsize, cq, flags)) { - log_debug("could not configure io submission queue\n"); + log_error("could not configure io submission queue\n"); return -1; } @@ -368,12 +388,12 @@ int nvme_delete_iosq(struct nvme_ctrl *ctrl, int qid) int nvme_create_ioqpair(struct nvme_ctrl *ctrl, int qid, int qsize, int vector, unsigned long flags) { if (nvme_create_iocq(ctrl, qid, qsize, vector)) { - log_debug("could not create io completion queue\n"); + log_error("could not create io completion queue\n"); return -1; } if (nvme_create_iosq(ctrl, qid, qsize, &ctrl->cq[qid], flags)) { - log_debug("could not create io submission queue\n"); + log_error("could not create io submission queue\n"); return -1; } @@ -383,38 +403,44 @@ int nvme_create_ioqpair(struct nvme_ctrl *ctrl, int qid, int qsize, int vector, int nvme_delete_ioqpair(struct nvme_ctrl *ctrl, int qid) { if (nvme_delete_iosq(ctrl, qid)) { - log_debug("could not delete io submission queue\n"); + log_error("could not delete io submission queue\n"); return -1; } if (nvme_delete_iocq(ctrl, qid)) { - log_debug("could not delete io completion queue\n"); + log_error("could not delete io completion queue\n"); return -1; } return 0; } +#ifndef __APPLE__ +#define _time_ns time_now().ts.tv_nsec +#else +#define _time_ns clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) +#endif + static int nvme_wait_rdy(struct nvme_ctrl *ctrl, unsigned short rdy) { uint64_t cap; + uint64_t timeout_ns; + uint64_t deadline_ns; uint32_t csts; - unsigned long timeout_ms; - struct timeabs deadline; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); - timeout_ms = 500 * (NVME_FIELD_GET(cap, CAP_TO) + 1); - deadline = timeabs_add(time_now(), time_from_msec(timeout_ms)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); + timeout_ns = 500 * (NVME_FIELD_GET(cap, CAP_TO) + 1) * 1000000; + deadline_ns = _time_ns + timeout_ns; do { - if (time_after(time_now(), deadline)) { - log_debug("timed out\n"); + if ((uint64_t)_time_ns > deadline_ns) { + log_error("timed out\n"); errno = ETIMEDOUT; return -1; } - csts = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CSTS)); + csts = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CSTS)); } while (NVME_FIELD_GET(csts, CSTS_RDY) != rdy); return 0; @@ -426,7 +452,7 @@ int nvme_enable(struct nvme_ctrl *ctrl) uint32_t cc; uint64_t cap; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); css = NVME_FIELD_GET(cap, CAP_CSS); cc = @@ -444,7 +470,7 @@ int nvme_enable(struct nvme_ctrl *ctrl) else cc |= NVME_FIELD_SET(NVME_CC_CSS_NVM, CC_CSS); - mmio_write32(ctrl->regs + NVME_REG_CC, cpu_to_le32(cc)); + mmio_write32(ctrl->regs, NVME_REG_CC, cpu_to_le32(cc)); return nvme_wait_rdy(ctrl, 1); } @@ -453,8 +479,8 @@ int nvme_reset(struct nvme_ctrl *ctrl) { uint32_t cc; - cc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CC)); - mmio_write32(ctrl->regs + NVME_REG_CC, cpu_to_le32(cc & 0xfe)); + cc = le32_to_cpu(mmio_read32(ctrl->regs, NVME_REG_CC)); + mmio_write32(ctrl->regs, NVME_REG_CC, cpu_to_le32(cc & 0xfe)); return nvme_wait_rdy(ctrl, 0); } @@ -463,17 +489,20 @@ static int nvme_init_dbconfig(struct nvme_ctrl *ctrl) { uint64_t prp1, prp2; union nvme_cmd cmd; + void *opaque; - if (pgmap((void **)&ctrl->dbbuf.doorbells, __VFN_PAGESIZE) < 0) + if (__pgmap((void **)&ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &opaque) < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &prp1, 0x0)) + if (_iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.doorbells, __VFN_PAGESIZE, &prp1, 0x0, + opaque)) return -1; - if (pgmap((void **)&ctrl->dbbuf.eventidxs, __VFN_PAGESIZE) < 0) + if (__pgmap((void **)&ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &opaque) < 0) return -1; - if (iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &prp2, 0x0)) + if (_iommu_map_vaddr(__iommu_ctx(ctrl), ctrl->dbbuf.eventidxs, __VFN_PAGESIZE, &prp2, 0x0, + opaque)) return -1; cmd = (union nvme_cmd) { @@ -489,7 +518,7 @@ static int nvme_init_dbconfig(struct nvme_ctrl *ctrl) uint64_t cap; uint8_t dstrd; - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); dstrd = NVME_FIELD_GET(cap, CAP_DSTRD); ctrl->adminq.cq->dbbuf.doorbell = cqhdbl(ctrl->dbbuf.doorbells, NVME_AQ, dstrd); @@ -509,6 +538,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op uint8_t mpsmin, mpsmax; uint16_t oacs; ssize_t len; + void *opaque; void *vaddr; int ret; @@ -521,14 +551,14 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op memcpy(&ctrl->opts, &nvme_ctrl_opts_default, sizeof(*opts)); if (pci_device_info_get_ull(bdf, "class", &classcode)) { - log_debug("could not get device class code\n"); + log_error("could not get device class code\n"); return -1; } log_info("pci class code is 0x%06llx\n", classcode); if ((classcode & 0xffff00) != 0x010800) { - log_debug("%s is not an NVMe device\n", bdf); + log_error("%s is not an NVMe device\n", bdf); errno = EINVAL; return -1; } @@ -541,11 +571,11 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->regs = vfio_pci_map_bar(&ctrl->pci, 0, 0x1000, 0, PROT_READ | PROT_WRITE); if (!ctrl->regs) { - log_debug("could not map controller registersn\n"); + log_error("could not map controller registersn\n"); return -1; } - cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); + cap = le64_to_cpu(mmio_read64(ctrl->regs, NVME_REG_CAP)); mpsmin = NVME_FIELD_GET(cap, CAP_MPSMIN); mpsmax = NVME_FIELD_GET(cap, CAP_MPSMAX); @@ -563,14 +593,14 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->config.mqes = NVME_FIELD_GET(cap, CAP_MQES); if (nvme_reset(ctrl)) { - log_debug("could not reset controller\n"); + log_error("could not reset controller\n"); return -1; } /* map admin queue doorbells */ ctrl->doorbells = vfio_pci_map_bar(&ctrl->pci, 0, 0x1000, 0x1000, PROT_WRITE); if (!ctrl->doorbells) { - log_debug("could not map doorbells\n"); + log_error("could not map doorbells\n"); return -1; } @@ -579,12 +609,12 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->cq = znew_t(struct nvme_cq, ctrl->opts.ncqr + 2); if (nvme_configure_adminq(ctrl, 0x0)) { - log_debug("could not configure admin queue\n"); + log_error("could not configure admin queue\n"); return -1; } if (nvme_enable(ctrl)) { - log_debug("could not enable controller\n"); + log_error("could not enable controller\n"); return -1; } @@ -601,7 +631,7 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op NVME_FIELD_SET(ctrl->opts.ncqr, FEAT_NRQS_NCQR)); if (nvme_admin(ctrl, &cmd, NULL, 0, &cqe)) { - log_debug("could not set number of queues\n"); + log_error("could not set number of queues\n"); return -1; } @@ -610,10 +640,15 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ctrl->config.ncqa = min_t(int, ctrl->opts.ncqr, NVME_FIELD_GET(le32_to_cpu(cqe.dw0), FEAT_NRQS_NCQR)); - len = pgmap(&vaddr, NVME_IDENTIFY_DATA_SIZE); + len = __pgmap(&vaddr, NVME_IDENTIFY_DATA_SIZE, &opaque); if (len < 0) return -1; + if (_iommu_map_vaddr(__iommu_ctx(ctrl), vaddr, len, NULL, IOMMU_MAP_EPHEMERAL, opaque)) { + log_error("failed to map vaddr\n"); + return -1; + } + cmd.identify = (struct nvme_cmd_identify) { .opcode = NVME_ADMIN_IDENTIFY, .cns = NVME_IDENTIFY_CNS_CTRL, @@ -621,17 +656,20 @@ int nvme_init(struct nvme_ctrl *ctrl, const char *bdf, const struct nvme_ctrl_op ret = nvme_admin(ctrl, &cmd, vaddr, len, NULL); if (ret) { - log_debug("could not identify\n"); + log_error("could not identify\n"); goto out; } - oacs = le16_to_cpu(*(leint16_t *)(vaddr + NVME_IDENTIFY_CTRL_OACS)); + memcpy(ctrl->serial, ((char *) vaddr) + + NVME_IDENTIFY_CTRL_SERIAL_NUMBER, sizeof(ctrl->serial)); + oacs = le16_to_cpu(*(leint16_t *)(((uintptr_t) vaddr) + NVME_IDENTIFY_CTRL_OACS)); if (oacs & NVME_IDENTIFY_CTRL_OACS_DBCONFIG) ret = nvme_init_dbconfig(ctrl); out: - pgunmap(vaddr, len); + log_fatal_if(iommu_unmap_vaddr(__iommu_ctx(ctrl), vaddr, NULL), "iommu_unmap_vaddr\n"); + __pgunmap(vaddr, len, opaque); return ret; } diff --git a/src/nvme/queue.c b/src/nvme/queue.c index c89d0c26..cd16b5fd 100644 --- a/src/nvme/queue.c +++ b/src/nvme/queue.c @@ -10,32 +10,11 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#include -#include -#include -#include -#include -#include -#include -#include - +#include +#include "iommu/context.h" +#ifndef __APPLE__ #include "ccan/time/time.h" +#endif void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n) { @@ -53,19 +32,18 @@ void nvme_cq_get_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n) } while (n > 0); } -int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, struct timespec *ts) +int nvme_cq_wait_cqes(struct nvme_cq *cq, struct nvme_cqe *cqes, int n, uint64_t timeout_ns) { struct nvme_cqe *cqe; - struct timerel rel = {.ts = *ts}; uint64_t timeout; - if (!ts) { + if (!timeout_ns) { nvme_cq_get_cqes(cq, cqes, n); return 0; } - timeout = get_ticks() + time_to_usec(rel) * (__vfn_ticks_freq / 1000000ULL); + timeout = get_ticks() + (timeout_ns*1000) * (__vfn_ticks_freq / 1000000ULL); do { cqe = nvme_cq_get_cqe(cq); diff --git a/src/nvme/rq.c b/src/nvme/rq.c index 332ba4e0..036ad79e 100644 --- a/src/nvme/rq.c +++ b/src/nvme/rq.c @@ -12,32 +12,9 @@ #define log_fmt(fmt) "nvme/rq: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include #include - #include "ccan/minmax/minmax.h" -#include "iommu/context.h" static inline int __map_first(leint64_t *prp1, leint64_t *prplist, uint64_t iova, size_t len, int pageshift) @@ -100,7 +77,7 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * size_t len) { int prpcount; - leint64_t *prplist = rq->page.vaddr; + leint64_t *prplist = (leint64_t *)rq->page.vaddr; prpcount = __map_first(&cmd->dptr.prp1, prplist, iova, len, __mps_to_pageshift(ctrl->config.mps)); @@ -119,11 +96,12 @@ int nvme_rq_map_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd * return 0; } +#ifndef __APPLE__ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd *cmd, struct iovec *iov, int niov) { int prpcount, _prpcount; - leint64_t *prplist = rq->page.vaddr; + leint64_t *prplist = (leint64_t *)rq->page.vaddr; uint64_t iova = (uint64_t)iov->iov_base; size_t len = iov->iov_len; int pageshift = __mps_to_pageshift(ctrl->config.mps); @@ -164,7 +142,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd if (!ALIGNED(iova, pagesize)) { - log_error("unaligned iov[%u].iov_base (0x%"PRIx64")\n", i, iova); + log_error("unaligned iov[%u].iov_base (0x%" PRIx64 ")\n", i, iova); goto invalid; } @@ -192,6 +170,7 @@ int nvme_rq_mapv_prp(struct nvme_ctrl *ctrl, struct nvme_rq *rq, union nvme_cmd errno = EINVAL; return -1; } +#endif int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) { @@ -213,7 +192,7 @@ int nvme_rq_spin(struct nvme_rq *rq, struct nvme_cqe *cqe_copy) if (logv(LOG_DEBUG)) { uint16_t status = le16_to_cpu(cqe.sfp) >> 1; - log_debug("cqe status 0x%" PRIx16 "\n", status & 0x7ff); + log_debug("cqe status 0x%" PRIx16 "\n", (uint16_t)(status & 0x7ff)); } return nvme_set_errno_from_cqe(&cqe); diff --git a/src/nvme/types.h b/src/nvme/types.h index 5afd0984..7714cfb4 100644 --- a/src/nvme/types.h +++ b/src/nvme/types.h @@ -102,7 +102,8 @@ enum nvme_identify_cns { }; enum nvme_identify_ctrl_offset { - NVME_IDENTIFY_CTRL_OACS = 0x100, + NVME_IDENTIFY_CTRL_SERIAL_NUMBER = 0x4, + NVME_IDENTIFY_CTRL_OACS = 0x100, }; enum nvme_identify_ctrl_oacs { diff --git a/src/nvme/util.c b/src/nvme/util.c index a509ceb7..7684ea28 100644 --- a/src/nvme/util.c +++ b/src/nvme/util.c @@ -10,32 +10,9 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include - -#include -#include -#include -#include #include - +#include "iommu/context.h" #include "types.h" - #include "crc64table.h" uint64_t nvme_crc64(uint64_t crc, const unsigned char *buffer, size_t len) @@ -84,14 +61,15 @@ int nvme_sync(struct nvme_ctrl *ctrl, struct nvme_sq *sq, union nvme_cmd *sqe, v bool do_unmap = false; int ret = 0; + void *opaque = NULL; // Could be null, as we're always mapped? if (buf) { struct iommu_ctx *ctx = __iommu_ctx(ctrl); if (!iommu_translate_vaddr(ctx, buf, &iova)) { do_unmap = true; - if (iommu_map_vaddr(ctx, buf, len, &iova, IOMMU_MAP_EPHEMERAL)) { - log_debug("failed to map vaddr\n"); + if (_iommu_map_vaddr(ctx, buf, len, &iova, IOMMU_MAP_EPHEMERAL, opaque)) { + log_error("failed to map vaddr\n"); return -1; } } @@ -113,7 +91,7 @@ int nvme_sync(struct nvme_ctrl *ctrl, struct nvme_sq *sq, union nvme_cmd *sqe, v while (nvme_rq_spin(rq, &cqe) < 0) { if (errno == EAGAIN) { log_error("SPURIOUS CQE (cq %" PRIu16 " cid %" PRIu16 ")\n", - rq->sq->cq->id, cqe.cid); + (uint16_t)rq->sq->cq->id, cqe.cid); continue; } diff --git a/src/pci/util.c b/src/pci/util.c index 3a87daeb..960aac9a 100644 --- a/src/pci/util.c +++ b/src/pci/util.c @@ -10,29 +10,33 @@ * COPYING and LICENSE files for more information. */ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include +#ifdef __cplusplus +extern "C" { +#endif +#ifndef __APPLE__ #include +#include +#endif -#include -#include -#include -#include -#include +#include #include +#include + + +#ifdef __APPLE__ +int pci_device_info_get_ull(const char *bdf, const char *prop, unsigned long long *v) +{ + if (!strcmp(prop, "class")) { + // TODO: Actually read this from device once it makes sense + *v = 0x010800; + } else { + return -1; + } + return 0; +} +#else int pci_unbind(const char *bdf) { char *path = NULL; @@ -40,7 +44,7 @@ int pci_unbind(const char *bdf) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/devices/%s/driver/unbind", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -62,7 +66,7 @@ int pci_bind(const char *bdf, const char *driver) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/bind", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -80,12 +84,12 @@ int pci_driver_new_id(const char *driver, uint16_t vid, uint16_t did) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/new_id", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } if (asprintf(&id, "%x %x", vid, did) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); free(path); return -1; } @@ -105,12 +109,12 @@ int pci_driver_remove_id(const char *driver, uint16_t vid, uint16_t did) ssize_t ret; if (asprintf(&path, "/sys/bus/pci/drivers/%s/remove_id", driver) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } if (asprintf(&id, "%x %x", vid, did) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); free(path); return -1; } @@ -129,7 +133,7 @@ int pci_device_info_get_ull(const char *bdf, const char *prop, unsigned long lon ssize_t ret; if (asprintf(&path, "/sys/bus/pci/devices/%s/%s", bdf, prop) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return -1; } @@ -157,7 +161,7 @@ char *pci_get_driver(const char *bdf) if (asprintf(&link, "/sys/bus/pci/devices/%s/driver", bdf) < 0) { link = NULL; - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -168,7 +172,7 @@ char *pci_get_driver(const char *bdf) if (errno == ENOENT) goto out; - log_debug("failed to resolve driver link\n"); + log_error("failed to resolve driver link\n"); goto out; } @@ -176,13 +180,13 @@ char *pci_get_driver(const char *bdf) p = strrchr(driver, '/'); if (!p) { - log_debug("failed to determine driver name\n"); + log_error("failed to determine driver name\n"); goto out; } if (asprintf(&name, "%s", p + 1) < 0) { name = NULL; - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -199,7 +203,7 @@ char *pci_get_iommu_group(const char *bdf) ssize_t ret; if (asprintf(&link, "/sys/bus/pci/devices/%s/iommu_group", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); goto out; } @@ -207,7 +211,7 @@ char *pci_get_iommu_group(const char *bdf) ret = readlink(link, group, PATH_MAX - 1); if (ret < 0) { - log_debug("failed to resolve iommu group link\n"); + log_error("failed to resolve iommu group link\n"); goto out; } @@ -215,7 +219,7 @@ char *pci_get_iommu_group(const char *bdf) p = strrchr(group, '/'); if (!p) { - log_debug("failed to find iommu group number\n"); + log_error("failed to find iommu group number\n"); goto out; } @@ -239,13 +243,13 @@ char *pci_get_device_vfio_id(const char *bdf) DIR *dp; if (asprintf(&path, "/sys/bus/pci/devices/%s/vfio-dev", bdf) < 0) { - log_debug("asprintf failed\n"); + log_error("asprintf failed\n"); return NULL; } dp = opendir(path); if (!dp) { - log_debug("could not open directory; is %s bound to vfio-pci?\n", bdf); + log_error("could not open directory; is %s bound to vfio-pci?\n", bdf); return NULL; } @@ -280,3 +284,8 @@ char *pci_get_device_vfio_id(const char *bdf) return vfio_id; } +#endif + +#ifdef __cplusplus +} +#endif diff --git a/src/support/arch/x86_64/rdtsc.c b/src/support/arch/x86_64/rdtsc.c index ac1b325f..693de4cc 100644 --- a/src/support/arch/x86_64/rdtsc.c +++ b/src/support/arch/x86_64/rdtsc.c @@ -7,6 +7,12 @@ * From DPDK; modified by the libvfn Authors. */ +#if defined(__x86_64__) + +#ifdef __cplusplus +extern "C" { +#endif + #include #include @@ -34,3 +40,9 @@ uint64_t get_ticks_freq_arch(void) { return __x86_64_get_tsc_freq(); } + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/support/io.c b/src/support/io.c index da347d9a..b508e8d6 100644 --- a/src/support/io.c +++ b/src/support/io.c @@ -10,18 +10,6 @@ * COPYING and LICENSE files for more information. */ -#include -#ifdef __GLIBC__ -#include -#endif -#include -#include -#include -#include -#include -#include -#include - #include "vfn/support.h" ssize_t writeallfd(int fd, const void *buf, size_t count) diff --git a/src/support/log.c b/src/support/log.c index f24fb064..60752164 100644 --- a/src/support/log.c +++ b/src/support/log.c @@ -22,7 +22,13 @@ struct log_state __log_state; static void __attribute__((constructor)) init_log_level(void) { - char *buf = getenv("LOGV"); +#ifdef __APPLE__ + goto set_default; + // dummy to satisfy compiler with getenv + #define getenv +#endif + char *buf; + buf = getenv("LOGV"); char *endptr; long v; diff --git a/src/support/meson.build b/src/support/meson.build index 30e01276..c435d615 100644 --- a/src/support/meson.build +++ b/src/support/meson.build @@ -1,11 +1,12 @@ support_sources = files( 'io.c', 'log.c', - 'mem.c', 'ticks.c', 'timer.c', ) +subdir('platform/linux') + if host_machine.cpu_family() == 'x86_64' subdir('arch/x86_64') endif diff --git a/src/support/mem.c b/src/support/platform/linux/mem.c similarity index 77% rename from src/support/mem.c rename to src/support/platform/linux/mem.c index 5d7f2fea..9a051e93 100644 --- a/src/support/mem.c +++ b/src/support/platform/linux/mem.c @@ -11,27 +11,12 @@ */ #define log_fmt(fmt) "support/mem: " fmt - -#include +#include #ifdef __GLIBC__ #include #endif -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include -#include -#include -#include -#include +#include "ccan/compiler/compiler.h" +#include size_t __VFN_PAGESIZE; int __VFN_PAGESHIFT; @@ -88,4 +73,24 @@ ssize_t pgmapn(void **mem, unsigned int n, size_t sz) return pgmap(mem, n * sz); } +ssize_t __pgmap(void **mem, size_t sz, void **opaque UNUSED) +{ + return pgmap(mem, sz); +} + +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque UNUSED) +{ + return pgmapn(mem, n, sz); +} + +void __pgunmap(void *mem, size_t len, void *opaque UNUSED) +{ + return pgunmap(mem, len); +} + +void pgunmap(void *mem, size_t len) +{ + if (munmap(mem, len)) + backtrace_abort(); +} diff --git a/src/support/platform/linux/meson.build b/src/support/platform/linux/meson.build new file mode 100644 index 00000000..b59fba9c --- /dev/null +++ b/src/support/platform/linux/meson.build @@ -0,0 +1,5 @@ +support_sources_platform_linux = files( + 'mem.c', +) + +support_sources += support_sources_platform_linux diff --git a/src/support/platform/macos/mem.c b/src/support/platform/macos/mem.c new file mode 100644 index 00000000..5d141676 --- /dev/null +++ b/src/support/platform/macos/mem.c @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later or MIT + +/* + * This file is part of libvfn. + * + * Copyright (C) 2022 The libvfn Authors. All Rights Reserved. + * + * This library (libvfn) is dual licensed under the GNU Lesser General + * Public License version 2.1 or later or the MIT license. See the + * COPYING and LICENSE files for more information. + */ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +size_t __VFN_PAGESIZE; +int __VFN_PAGESHIFT; + +static void __attribute__((constructor)) init_page_size(void) +{ + __VFN_PAGESIZE = IOVMPageSize; + __VFN_PAGESHIFT = 63u - __builtin_clzl(__VFN_PAGESIZE); + + log_debug("pagesize is %zu (shift %d)\n", __VFN_PAGESIZE, __VFN_PAGESHIFT); +} + +void backtrace_abort(void) +{ + abort(); +} + +ssize_t __pgmap(void **mem, size_t sz, void **opaque) +{ + IOAddressSegment virtualAddressSegment; + ssize_t len = ALIGN_UP(sz, __VFN_PAGESIZE); + + IOBufferMemoryDescriptor **mem_descriptor = (IOBufferMemoryDescriptor **) opaque; + IOBufferMemoryDescriptor::Create( + kIOMemoryDirectionInOut, + len, + __VFN_PAGESIZE, + mem_descriptor + ); + (*mem_descriptor)->SetLength(len); + (*mem_descriptor)->GetAddressRange(&virtualAddressSegment); + bzero((void *) virtualAddressSegment.address, virtualAddressSegment.length); + *mem = (void *) virtualAddressSegment.address; + + return len; +} + +ssize_t __pgmapn(void **mem, unsigned int n, size_t sz, void **opaque) +{ + return __pgmap(mem, n * sz, opaque); +} + +void __pgunmap(void *mem, size_t len, void *opaque) +{ + IOBufferMemoryDescriptor *_mem = (IOBufferMemoryDescriptor *) opaque; + + OSSafeReleaseNULL(_mem); +} + +ssize_t pgmap(void **mem, size_t sz) +{ + log_fatal("Use ::__pgmap on macOS"); +} + +ssize_t pgmapn(void **mem, unsigned int n, size_t sz) +{ + log_fatal("Use ::__pgmapn on macOS"); +} + +void pgunmap(void *mem, size_t len) +{ + log_fatal("Use ::__pgunmap on macOS"); +} + +#ifdef __cplusplus +} +#endif diff --git a/src/support/ticks.c b/src/support/ticks.c index a8d26c30..8d183b2f 100644 --- a/src/support/ticks.c +++ b/src/support/ticks.c @@ -9,26 +9,17 @@ #define log_fmt(fmt) "support/ticks: " fmt -#include -#include -#include -#include -#include -#include -#include - -#include "vfn/support/align.h" -#include "vfn/support/atomic.h" -#include "vfn/support/log.h" -#include "vfn/support/ticks.h" -#include "vfn/support/timer.h" +#include "vfn/support.h" +#ifndef __APPLE__ #include "ccan/time/time.h" +#endif #define TICKS_PER_10MHZ 10000000ULL uint64_t __vfn_ticks_freq; +#ifndef __APPLE__ static uint64_t measure_ticks_freq(void) { struct timemono t_start, t_end; @@ -72,6 +63,7 @@ static uint64_t estimate_ticks_freq(void) return ROUND(get_ticks() - start, TICKS_PER_10MHZ); } +#endif static void __attribute__((constructor)) init_ticks_freq(void) { @@ -79,11 +71,15 @@ static void __attribute__((constructor)) init_ticks_freq(void) freq = get_ticks_freq_arch(); if (!freq) + #ifndef __APPLE__ freq = measure_ticks_freq(); if (!freq) freq = estimate_ticks_freq(); + #else + abort(); + #endif log_debug("tick frequency is ~%" PRIu64 " Hz\n", freq); __vfn_ticks_freq = freq; diff --git a/src/util/skiplist.c b/src/util/skiplist.c index 2decc242..c686ead0 100644 --- a/src/util/skiplist.c +++ b/src/util/skiplist.c @@ -12,6 +12,12 @@ #include +#ifndef __APPLE__ +#define _rand rand +#else +#define _rand arc4random +#endif + #include "skiplist.h" void skiplist_init(struct skiplist *list) @@ -81,7 +87,7 @@ static inline int __skiplist_random_level(void) { int k = 0; - while (k < SKIPLIST_LEVELS - 1 && (rand() > (RAND_MAX / 2))) + while (k < SKIPLIST_LEVELS - 1 && (_rand() > (RAND_MAX / 2))) k++; return k; diff --git a/src/util/skiplist.h b/src/util/skiplist.h index c22ea290..f8906402 100644 --- a/src/util/skiplist.h +++ b/src/util/skiplist.h @@ -10,6 +10,9 @@ * COPYING and LICENSE files for more information. */ +#ifndef LIBVFN_SKIPLIST_H +#define LIBVFN_SKIPLIST_H + #include "ccan/list/list.h" #ifndef SKIPLIST_LEVELS @@ -58,3 +61,5 @@ void skiplist_link(struct skiplist *list, struct skiplist_node *n, struct skiplist_node *update[SKIPLIST_LEVELS]); void skiplist_erase(struct skiplist *list, struct skiplist_node *n, struct skiplist_node *update[SKIPLIST_LEVELS]); + +#endif /* LIBVFN_SKIPLIST_H */ diff --git a/src/vfio/device.c b/src/vfio/device.c index 6ad7275a..40d136d5 100644 --- a/src/vfio/device.c +++ b/src/vfio/device.c @@ -12,25 +12,9 @@ #define log_fmt(fmt) "vfio/device: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include -#include #include - #include -#include #include #include "vfn/support.h" @@ -48,7 +32,7 @@ int vfio_set_irq(struct vfio_device *dev, int *eventfds, int count) if (!(dev->irq_info.flags & VFIO_IRQ_INFO_EVENTFD)) { errno = EINVAL; - log_debug("device irq does not support eventfd\n"); + log_error("device irq does not support eventfd\n"); return -1; } @@ -69,7 +53,7 @@ int vfio_set_irq(struct vfio_device *dev, int *eventfds, int count) free(irq_set); if (ret) { - log_debug("failed to set device irq\n"); + log_error("failed to set device irq\n"); return -1; } @@ -90,7 +74,7 @@ int vfio_disable_irq(struct vfio_device *dev) ret = ioctl(dev->fd, VFIO_DEVICE_SET_IRQS, &irq_set); if (ret) { - log_debug("failed to disable device irq\n"); + log_error("failed to disable device irq\n"); return -1; } diff --git a/src/vfio/pci.c b/src/vfio/pci.c index 76d9c1f3..0eb1bdd8 100644 --- a/src/vfio/pci.c +++ b/src/vfio/pci.c @@ -12,26 +12,10 @@ #define log_fmt(fmt) "vfio/pci: " fmt -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - #include #include #include +#include #include "vfn/support.h" #include "vfn/iommu.h" @@ -50,14 +34,14 @@ static int pci_set_bus_master(struct vfio_pci_device *pci) uint16_t pci_cmd; if (vfio_pci_read_config(pci, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND) < 0) { - log_debug("failed to read pci config region\n"); + log_error("failed to read pci config region\n"); return -1; } pci_cmd |= PCI_COMMAND_MASTER; if (vfio_pci_write_config(pci, &pci_cmd, sizeof(pci_cmd), PCI_COMMAND) < 0) { - log_debug("failed to write pci config region\n"); + log_error("failed to write pci config region\n"); return -1; } @@ -74,7 +58,7 @@ static int vfio_pci_init_bar(struct vfio_pci_device *pci, int idx) }; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_REGION_INFO, &pci->bar_region_info[idx])) { - log_debug("failed to get bar region info\n"); + log_error("failed to get bar region info\n"); return -1; } @@ -92,14 +76,14 @@ static int vfio_pci_init_irq(struct vfio_pci_device *pci) do { if (irq_index < 0) { errno = EINVAL; - log_debug("no supported irq types\n"); + log_error("no supported irq types\n"); return -1; } pci->dev.irq_info.index = irq_index--; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_IRQ_INFO, &pci->dev.irq_info)) { - log_debug("failed to get device irq info\n"); + log_error("failed to get device irq info\n"); return -1; } } while (!pci->dev.irq_info.count); @@ -148,14 +132,14 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) pci->dev.fd = pci->dev.ctx->ops.get_device_fd(pci->dev.ctx, bdf); if (pci->dev.fd < 0) { - log_debug("failed to get device fd\n"); + log_error("failed to get device fd\n"); return -1; } pci->dev.device_info.argsz = sizeof(struct vfio_device_info); if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_INFO, &pci->dev.device_info)) { - log_debug("failed to get device info\n"); + log_error("failed to get device info\n"); return -1; } @@ -167,7 +151,7 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) }; if (ioctl(pci->dev.fd, VFIO_DEVICE_GET_REGION_INFO, &pci->config_region_info)) { - log_debug("failed to get config region info\n"); + log_error("failed to get config region info\n"); return -1; } @@ -177,12 +161,12 @@ int vfio_pci_open(struct vfio_pci_device *pci, const char *bdf) } if (pci_set_bus_master(pci)) { - log_debug("failed to set pci bus master\n"); + log_error("failed to set pci bus master\n"); return -1; } if (vfio_pci_init_irq(pci)) { - log_debug("failed to initialize irq\n"); + log_error("failed to initialize irq\n"); return -1; } diff --git a/tests/device/aer.c b/tests/device/aer.c index 9ffd0b7c..d8a74624 100644 --- a/tests/device/aer.c +++ b/tests/device/aer.c @@ -85,7 +85,7 @@ static int test_aer(void) union nvme_cmd cmd; struct nvme_rq *rq; struct nvme_cqe cqe, cqes[2]; - struct timespec timeout = {.tv_sec = 1}; + uint64_t timeout_ns = 1000000000; int ret; uint32_t temp_thresh; @@ -133,7 +133,7 @@ static int test_aer(void) nvme_rq_exec(rq, &cmd); - ret = nvme_cq_wait_cqes(ctrl.adminq.cq, cqes, 2, &timeout); + ret = nvme_cq_wait_cqes(ctrl.adminq.cq, cqes, 2, timeout_ns); if (ret < 2 && errno == ETIMEDOUT) err(errno, "not enough completions in time"); diff --git a/tests/device/timeout.c b/tests/device/timeout.c index 24a8fb61..e333b487 100644 --- a/tests/device/timeout.c +++ b/tests/device/timeout.c @@ -32,13 +32,13 @@ static int test_timeout(void) { struct nvme_cqe cqe; - struct timespec timeout = {.tv_sec = 1}; + uint64_t timeout_ns = 1000000000; int ret; if (nvme_aer(&ctrl, NULL)) err(1, "could not enable aen"); - ret = nvme_cq_wait_cqes(ctrl.adminq.cq, &cqe, 1, &timeout); + ret = nvme_cq_wait_cqes(ctrl.adminq.cq, &cqe, 1, timeout_ns); if (ret != 1 || errno != ETIMEDOUT) return -1;