NVIDIA/open-gpu-kernel-modules
1
0
Fork 0
mirror of https://github.com/NVIDIA/open-gpu-kernel-modules.git synced 2026-10-12 02:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

615.71.09

Browse Source
This commit is contained in:
Maneet Singh
2026-09-09 11:55:37 -07:00
parent e4a5faa256
commit 61dcc93722
1178 changed files with 621297 additions and 542126 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+6 -7
View File
@@ -1,7 +1,7 @@
# NVIDIA Linux Open GPU Kernel Module Source
This is the source release of the NVIDIA Linux open GPU kernel modules,
version 610.57.04.
version 615.71.09.
## How to Build
@@ -17,7 +17,7 @@ as root:
Note that the kernel modules built here must be used with GSP
firmware and user-space NVIDIA GPU driver components from a corresponding
610.57.04 driver release. This can be achieved by installing
615.71.09 driver release. This can be achieved by installing
the NVIDIA GPU driver from the .run file using the `--no-kernel-modules`
option. E.g.,
@@ -41,6 +41,7 @@ E.g.,
# compile on x86_64 for aarch64
make modules -j$(nproc) \
TARGET_ARCH=aarch64 \
ARCH=arm64 \
CC=aarch64-linux-gnu-gcc \
LD=aarch64-linux-gnu-ld \
AR=aarch64-linux-gnu-ar \
@@ -185,7 +186,7 @@ table below).
For details on feature support and limitations, see the NVIDIA GPU driver
end user README here:
https://us.download.nvidia.com/XFree86/Linux-x86_64/610.57.04/README/kernel_open.html
https://us.download.nvidia.com/XFree86/Linux-x86_64/615.71.09/README/kernel_open.html
For vGPU support, please refer to the README.vgpu packaged in the vGPU Host
Package for more details.
@@ -952,6 +953,7 @@ Subsystem Device ID.
| NVIDIA GB200 | 2941 10DE 20D5 |
| NVIDIA GB200 | 2941 10DE 21C9 |
| NVIDIA GB200 | 2941 10DE 21CA |
| NVIDIA GB200 | 2941 10DE 22C4 |
| NVIDIA DRIVE P2021 | 29BB 10DE 207C |
| NVIDIA GeForce RTX 5090 | 2B85 |
| NVIDIA GeForce RTX 5090 D | 2B87 |
@@ -979,6 +981,7 @@ Subsystem Device ID.
| NVIDIA RTX 6000D | 2BB9 10DE 2279 |
| NVIDIA GeForce RTX 5080 | 2C02 |
| NVIDIA GeForce RTX 5070 Ti | 2C05 |
| NVIDIA GeForce RTX 5070 | 2C09 |
| NVIDIA GeForce RTX 5090 Laptop GPU | 2C18 |
| NVIDIA GeForce RTX 5080 Laptop GPU | 2C19 |
| NVIDIA RTX PRO 4500 Blackwell | 2C31 1028 2051 |
@@ -1018,10 +1021,6 @@ Subsystem Device ID.
| NVIDIA RTX PRO 500 Blackwell Generation Laptop GPU | 2DB9 |
| NVIDIA GeForce RTX 5050 Laptop GPU | 2DD8 |
| NVIDIA RTX PRO 500 Blackwell Embedded GPU | 2DF9 |
| NVIDIA RTX Spark N1X (6144-core Blackwell RTX GPU) | 2E03 1043 1484 |
| NVIDIA RTX Spark N1X (6144-core Blackwell RTX GPU) | 2E03 1043 3309 |
| NVIDIA RTX Spark N1X (5120-core Blackwell RTX GPU) | 2E06 1043 1484 |
| NVIDIA RTX Spark N1X (5120-core Blackwell RTX GPU) | 2E06 1043 3309 |
| NVIDIA GB10 | 2E12 10DE 21EC |
| NVIDIA GeForce RTX 5070 | 2F04 |
| NVIDIA GeForce RTX 5060 | 2F06 |
kernel-open/Kbuild
+31 -4
View File
@@ -38,10 +38,37 @@ NV_BUILD_TYPE ?= release
# just the object files.
#
ASSIGN_PER_OBJ_CFLAGS = \
$(foreach _cflags_variable, \
$(notdir $(1)) $(1), \
$(eval $(addprefix CFLAGS_,$(_cflags_variable)) += $(2)))
$(foreach _obj, \
$(sort $(notdir $(1)) $(1)), \
$(eval $(addprefix CFLAGS_,$(_obj)) += $(2)))
#
# Utility macro ASSIGN_PER_OBJ_CFLAGS_REMOVE: to control CFLAGS on a
# per-object basis, Kbuild honors the 'CFLAGS_REMOVE_$(object)' variable.
# E.g., "CFLAGS_REMOVE_nv.o" will remove those CFLAGS from the nv.o compilation
# unit. It has the same workaround for the change in Kbuild handling of
# object-specific CFLAGS as described in ASSIGN_PER_OBJ_CFLAGS.
#
ASSIGN_PER_OBJ_CFLAGS_REMOVE = \
$(foreach _obj, \
$(sort $(notdir $(1)) $(1)), \
$(eval $(addprefix CFLAGS_REMOVE_,$(_obj)) += $(2)))
#
# As ASSIGN_PER_OBJ_CFLAGS, just for per-object LDFLAGS.
#
ASSIGN_PER_OBJ_LDFLAGS = \
$(foreach _obj, \
$(sort $(notdir $(1)) $(1)), \
$(eval $(addprefix LDFLAGS_,$(_obj)) += $(2)))
#
# As ASSIGN_PER_OBJ_CFLAGS, just for per-object OBJCOPYFLAGS.
#
ASSIGN_PER_OBJ_OBJCOPYFLAGS = \
$(foreach _obj, \
$(sort $(notdir $(1)) $(1)), \
$(eval $(addprefix OBJCOPYFLAGS_,$(_obj)) += $(2)))
#
# Include the specifics of the individual NVIDIA kernel modules.
@@ -79,7 +106,7 @@ ccflags-y += -I$(src)/common/inc
ccflags-y += -I$(src)
ccflags-y += -Wall $(DEFINES) $(INCLUDES) -Wno-cast-qual -Wno-format-extra-args
ccflags-y += -D__KERNEL__ -DMODULE -DNVRM
ccflags-y += -DNV_VERSION_STRING=\"610.57.04\"
ccflags-y += -DNV_VERSION_STRING=\"615.71.09\"
# Include and link Tegra out-of-tree modules.
ifneq ($(wildcard /usr/src/nvidia/nvidia-public),)
kernel-open/common/inc/cpuopsys.h
+6
View File
@@ -178,6 +178,7 @@
#if defined(__riscv) && (__riscv_xlen==64)
# define NVCPU_RISCV64
# define NVCPU_FAMILY_RISCV
# if defined(__nvriscv)
# define NVCPU_NVRISCV64
# endif
@@ -401,6 +402,11 @@
#else
#define NVCPU_IS_FAMILY_ARM 0
#endif
#if defined(NVCPU_FAMILY_RISCV)
#define NVCPU_IS_FAMILY_RISCV 1
#else
#define NVCPU_IS_FAMILY_RISCV 0
#endif
#if defined(NVCPU_SH4)
#define NVCPU_IS_SH4 1
#else
kernel-open/common/inc/nv-caps.h
+2
View File
@@ -24,6 +24,8 @@
#ifndef _NV_CAPS_H_
#define _NV_CAPS_H_
#include <nv-kernel-interface-api.h>
/*
kernel-open/common/inc/nv-dev-printf.h
+33
View File
@@ -0,0 +1,33 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*/
#ifndef _NV_DEV_PRINTF_H_
#define _NV_DEV_PRINTF_H_
#include "nv_stdarg.h"
#include "nv-kernel-interface-api.h"
/*
* Device logging (kernel/system log) levels.
*
* These are distinct from NV_DBG_* which are used for internal driver filtering.
*/
typedef enum NV_LOG_LEVEL
{
NV_LOG_LEVEL_DEBUG = 0,
NV_LOG_LEVEL_INFO = 1,
NV_LOG_LEVEL_NOTICE = 2,
NV_LOG_LEVEL_WARNING = 3,
NV_LOG_LEVEL_ERROR = 4,
NV_LOG_LEVEL_CRIT = 5,
NV_LOG_LEVEL_ALERT = 6,
} NV_LOG_LEVEL;
struct nv_state_t;
int NV_API_CALL nv_dev_printf(struct nv_state_t *, NV_LOG_LEVEL, const char *, ...);
int NV_API_CALL nv_dev_vprintf(struct nv_state_t *, NV_LOG_LEVEL, const char *, va_list);
#endif // _NV_DEV_PRINTF_H_
kernel-open/common/inc/nv-ioctl-numbers.h
+7 -1
View File
@@ -44,5 +44,11 @@
#define NV_ESC_SYS_PARAMS (NV_IOCTL_BASE + 14)
#define NV_ESC_EXPORT_TO_DMABUF_FD (NV_IOCTL_BASE + 17)
#define NV_ESC_WAIT_OPEN_COMPLETE (NV_IOCTL_BASE + 18)
#ifdef NV_VMWARE
/*
* NV_ESC_INIT_GPUS: VMware-only ioctl to initialize all probed GPUs from
* nvidia-ucodeoffload after GSP binary is pushed, avoiding rm_init from open.
*/
#define NV_ESC_INIT_GPUS (NV_IOCTL_BASE + 19)
#endif
#endif
kernel-open/common/inc/nv-linux.h
+64 -16
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2001-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2001-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -38,6 +38,10 @@
#include "nv-chardev-numbers.h"
#include "nv-platform.h"
#if defined(NV_CXL_CXL_H_PRESENT)
#include <cxl/cxl.h>
#endif
#ifndef AUTOCONF_INCLUDED
#if defined(NV_GENERATED_AUTOCONF_H_PRESENT)
#include <generated/autoconf.h>
@@ -103,6 +107,7 @@
#endif
/* task and signal-related items */
#include <linux/sched/mm.h>
#include <linux/sched/signal.h>
#include <linux/sched/task.h>
#include <linux/sched.h>
@@ -342,12 +347,12 @@ NV_STATUS nvos_forward_error_to_cray(struct pci_dev *, NvU32,
/* keep track of memory usage */
#include "nv-memdbg.h"
static inline void *nv_vmalloc(unsigned long size)
static inline void *nv_vmalloc(unsigned long size, gfp_t flags)
{
#if defined(NV_VMALLOC_HAS_PGPROT_T_ARG)
void *ptr = __vmalloc(size, GFP_KERNEL, PAGE_KERNEL);
void *ptr = __vmalloc(size, flags, PAGE_KERNEL);
#else
void *ptr = __vmalloc(size, GFP_KERNEL);
void *ptr = __vmalloc(size, flags);
#endif
NV_MEMDBG_ADD(ptr, size);
return ptr;
@@ -489,8 +494,8 @@ static inline pgprot_t nv_adjust_pgprot(pgprot_t vm_prot)
#define NV_IS_SUSER() capable(CAP_SYS_ADMIN)
#define NV_MAY_SLEEP() (!irqs_disabled() && !in_interrupt() && !in_atomic())
#define NV_MODULE_PARAMETER(x) module_param(x, int, 0)
#define NV_MODULE_STRING_PARAMETER(x) module_param(x, charp, 0)
#define NV_MODULE_PARAMETER(x) module_param(x, int, 0444)
#define NV_MODULE_STRING_PARAMETER(x) module_param(x, charp, 0444)
#undef MODULE_PARM
#define NV_HAVE_MEMORY_ENCRYPT_DECRYPT 0
@@ -677,15 +682,13 @@ static inline int nv_remap_page_range(struct vm_area_struct *vma,
}
static inline int nv_io_remap_page_range(struct vm_area_struct *vma,
NvU64 phys_addr, NvU64 size, NvU64 start)
NvU64 phys_addr, NvU64 size, NvU64 start, pgprot_t prot)
{
int ret = -1;
#if !defined(NV_XEN_SUPPORT_FULLY_VIRTUALIZED_KERNEL)
ret = nv_remap_page_range(vma, start, phys_addr, size,
nv_adjust_pgprot(vma->vm_page_prot));
ret = nv_remap_page_range(vma, start, phys_addr, size, prot);
#else
ret = io_remap_pfn_range(vma, start, (phys_addr >> PAGE_SHIFT),
size, nv_adjust_pgprot(vma->vm_page_prot));
ret = io_remap_pfn_range(vma, start, (phys_addr >> PAGE_SHIFT), size, prot);
#endif
return ret;
}
@@ -902,6 +905,13 @@ struct nv_dma_buf
};
#endif // CONFIG_DMA_SHARED_BUFFER
typedef struct nv_linux_mm_free_work_s {
NvU64 num_pages;
struct mm_struct *mm;
struct work_struct task;
} nv_linux_mm_free_work_t;
typedef struct nv_alloc_s {
struct nv_alloc_s *next;
struct device *dev;
@@ -932,6 +942,7 @@ typedef struct nv_alloc_s {
void *import_priv;
struct sg_table *import_sgt;
dma_addr_t dma_handle; /* dma handle used by dma_alloc_coherent(), dma_free_coherent() */
nv_linux_mm_free_work_t *accounting_mm_work;
} nv_alloc_t;
/**
@@ -1104,12 +1115,30 @@ typedef struct nv_dma_map_s {
* single-page sg elements on Xen Server.
*/
#if !defined(NV_DOM0_KERNEL_PRESENT)
#if defined(NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT)
/*
* See Bug 6419127 for the split-submap GMMU contiguity failure.
* This function and correct max_segment value avoid the overflow
* issue in the kernel.
*/
#define NV_DMA_MAX_SEGMENT_SIZE ((NvU32)(0x80000000)) /* 2G */
#define NV_ALLOC_DMA_SUBMAP_SCATTERLIST(dm, sm, i) \
((sg_alloc_table_from_pages_segment( \
&(sm)->sgt, \
&(dm)->pages[NV_DMA_SUBMAP_IDX_TO_PAGE_IDX(i)], \
(sm)->page_count, \
0, \
(sm)->page_count * PAGE_SIZE, \
NV_DMA_MAX_SEGMENT_SIZE, \
NV_GFP_KERNEL) == 0) ? NV_OK : NV_ERR_OPERATING_SYSTEM)
#else
#define NV_ALLOC_DMA_SUBMAP_SCATTERLIST(dm, sm, i) \
((sg_alloc_table_from_pages(&sm->sgt, \
&dm->pages[NV_DMA_SUBMAP_IDX_TO_PAGE_IDX(i)], \
sm->page_count, 0, \
sm->page_count * PAGE_SIZE, NV_GFP_KERNEL) == 0) ? NV_OK : \
NV_ERR_OPERATING_SYSTEM)
#endif
#else
#define NV_ALLOC_DMA_SUBMAP_SCATTERLIST(dm, sm, i) \
((sg_alloc_table(&sm->sgt, sm->page_count, NV_GFP_KERNEL)) == \
@@ -1235,6 +1264,16 @@ struct nv_pci_tegra_devfreq_dev;
/* linux-specific version of old nv_state_t */
/* this is a general os-specific state structure. the first element *must* be
the general state structure, for the generic unix-based code */
#if defined(NV_CXL_CXL_H_PRESENT)
struct nv_cxl {
struct cxl_dev_state cxlds;
struct cxl_memdev *cxlmd;
struct cxl_endpoint_decoder *cxled;
struct cxl_region *nv_region;
};
#endif
typedef struct nv_linux_state_s {
nv_state_t nv_state;
@@ -1248,6 +1287,11 @@ typedef struct nv_linux_state_s {
/* coherent link information */
coherent_link_info_t coherent_link_info;
#if defined(NV_CXL_CXL_H_PRESENT)
/* CXL device state; NULL for non-CXL GPUs */
struct nv_cxl *cxl;
#endif
/* Dedicated queue to be used for removing FB memory which is onlined
* to kernel as a NUMA node. Refer Bug : 3879845*/
nv_kthread_q_t remove_numa_memory_q;
@@ -1418,11 +1462,10 @@ typedef struct nv_linux_state_s {
struct nv_pci_tegra_devfreq_dev *nvd_devfreq_dev;
struct nv_pci_tegra_devfreq_dev *sys_devfreq_dev;
struct nv_pci_tegra_devfreq_dev *pwr_devfreq_dev;
NvU32 tegra_suspend_freq;
int (*devfreq_suspend)(struct device *dev);
int (*devfreq_resume)(struct device *dev);
int (*devfreq_enable_boost)(struct device *dev, unsigned int duration);
int (*devfreq_enable_boost)(struct device *dev, unsigned int duration, int boost_type);
int (*devfreq_disable_boost)(struct device *dev);
#endif
@@ -1457,6 +1500,9 @@ extern struct semaphore nv_linux_devices_lock;
extern struct rw_semaphore nv_system_pm_lock;
extern NvBool nv_ats_supported;
extern NvBool nv_non_ats_device_present;
void nv_set_max_sysmem_address(void);
/*
* file-private data
@@ -1605,8 +1651,8 @@ extern NvU32 NVreg_EnableNonblockingOpen;
extern NvU32 NVreg_UseKernelSuspendNotifiers;
extern NvU32 NVreg_GpuInitOnProbe;
extern NvU32 num_probed_nv_devices;
extern NvU32 num_nv_devices;
extern atomic_t num_probed_nv_devices;
extern atomic_t num_nv_devices;
#define NV_FILE_INODE(file) (file)->f_inode
@@ -1724,7 +1770,7 @@ typedef enum
#else
#include <linux/gpio/driver.h>
static inline int __to_hwgpio(const struct gpio_device *gdev,
static inline int __to_hwgpio(struct gpio_device *gdev,
const struct of_phandle_args *gpiospec)
{
#if defined(CONFIG_OF_GPIO)
@@ -1797,5 +1843,7 @@ static inline int of_get_named_gpio(const struct device_node *np,
#define NV_EXPORT_SYMBOL(symbol) EXPORT_SYMBOL_GPL(symbol)
#define NV_CHECK_EXPORT_SYMBOL(symbol) NV_IS_EXPORT_SYMBOL_PRESENT_##symbol
// Async PCI probe requires wait_for_device_probe() as a post-registration barrier
#define NV_PCI_ASYNC_PROBE_SUPPORTED NV_IS_EXPORT_SYMBOL_PRESENT_wait_for_device_probe
#endif /* _NV_LINUX_H_ */
kernel-open/common/inc/nv-pci.h
+83 -24
View File
@@ -26,6 +26,15 @@
#include <linux/pci.h>
#include "nv-linux.h"
#include <uapi/linux/pci_regs.h>
#if defined(NV_CXL_CXL_H_PRESENT)
#include <cxl/cxl.h>
#endif
#if defined(NV_CXL_PCI_H_PRESENT)
#include <cxl/pci.h>
#endif
#define NV_GPU_BAR1 1
#define NV_GPU_BAR3 3
@@ -33,36 +42,86 @@
int nv_pci_register_driver(void);
void nv_pci_unregister_driver(void);
int nv_pci_count_devices(void);
void nv_pci_wait_for_probe_complete(void);
NvU8 nv_find_pci_capability(struct pci_dev *, NvU8);
int nvidia_dev_get_pci_info(const NvU8 *, struct pci_dev **, NvU64 *, NvU64 *);
nv_linux_state_t * find_pci(NvU32, NvU8, NvU8, NvU8);
NvBool nv_pci_is_valid_topology_for_direct_pci(nv_state_t *, struct pci_dev *);
NvBool nv_pci_has_common_pci_switch(nv_state_t *nv, struct pci_dev *);
void nv_pci_tegra_boost_clocks(struct device *dev);
/* Register Block Identifier (RBI) */
enum cxl_regloc_type {
CXL_REGLOC_RBI_EMPTY = 0,
CXL_REGLOC_RBI_COMPONENT,
CXL_REGLOC_RBI_VIRT,
CXL_REGLOC_RBI_MEMDEV,
CXL_REGLOC_RBI_PMU,
CXL_REGLOC_RBI_TYPES
};
#define NV_PCI_EXT_CAP_START_OFFSET 0x100
#define NV_PCI_EXT_CAP_END_OFFSET 0x1000
/* Maximum number of iterations to traverse the PCI extended capability chain.
Each capability is minimum 8 bytes long. */
#define NV_PCI_EXT_CAP_MAX_ITERATIONS ((NV_PCI_EXT_CAP_END_OFFSET - NV_PCI_EXT_CAP_START_OFFSET) / 8)
#define NV_PCI_EXT_CAP_ID(hdr0) ((hdr0) & 0xffff)
#define NV_PCI_EXT_CAP_VERSION(hdr0) (((hdr0) & 0x000f0000) >> 16)
#define NV_PCI_EXT_CAP_NEXT_OFFSET(hdr0) (((hdr0) & 0xfff00000) >> 20)
#define NV_PCI_EXT_CAP_ID_DVSEC 0x0023
#define NV_PCI_DVSEC_HEADER_1_OFFSET 0x4
#define NV_PCI_DVSEC_VENDOR_ID(hdr1) ((hdr1) & 0xffff)
#define NV_PCI_DVSEC_REVISION(hdr1) (((hdr1) & 0x000f0000) >> 16)
#define NV_PCI_DVSEC_LEN(hdr1) (((hdr1) & 0xfff00000) >> 20)
#define NV_PCI_DVSEC_HEADER_2_OFFSET 0x8
#define NV_PCI_DVSEC_DESIGNATED_ID(hdr2) ((hdr2) & 0xffff)
#define NV_PCI_DVSEC_NVIDIA_VENDOR_ID 0x10de
#define CXL_CM_OFFSET 0x1000
#define CXL_CM_CAP_HDR_OFFSET 0x0
#define CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0)
#define CM_CAP_HDR_CAP_ID 1
#define CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24)
#define CXL_CM_CAP_PTR_MASK GENMASK(31, 20)
#define CXL_CM_CAP_CAP_ID_HDM 0x5
#define CXL_HDM_DECODER_CAP_OFFSET 0x0
#define CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
#define CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)
#define CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)
#define CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)
#define CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)
#define CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
/* DVSEC 3 specific constants */
#define NV_PCI_DVSEC_GPU_PDI_DESIGNATED_ID 0x3
#define NV_PCI_DVSEC_GPU_PDI_CAP_LEN 0x14
#define NV_PCI_DVSEC_GPU_PDI_LOW_OFFSET 0xc
#define NV_PCI_DVSEC_GPU_PDI_HIGH_OFFSET 0x10
#define NV_CXL_REGLOC_RBI_COMPONENT 0x1
#define NV_CXL_CM_OFFSET 0x1000
#define NV_CXL_CM_CAP_HDR_OFFSET 0x0
#define NV_CXL_CM_CAP_HDR_ID_MASK GENMASK(15, 0)
#define NV_CM_CAP_HDR_CAP_ID 1
#define NV_CXL_CM_CAP_HDR_ARRAY_SIZE_MASK GENMASK(31, 24)
#define NV_CXL_CM_CAP_PTR_MASK GENMASK(31, 20)
#define NV_CXL_CM_CAP_CAP_ID_HDM 0x5
#define NV_CXL_HDM_DECODER_CAP_OFFSET 0x0
#define NV_CXL_HDM_DECODER0_BASE_LOW_OFFSET(i) (0x20 * (i) + 0x10)
#define NV_CXL_HDM_DECODER0_BASE_HIGH_OFFSET(i) (0x20 * (i) + 0x14)
#define NV_CXL_HDM_DECODER0_SIZE_LOW_OFFSET(i) (0x20 * (i) + 0x18)
#define NV_CXL_HDM_DECODER0_SIZE_HIGH_OFFSET(i) (0x20 * (i) + 0x1c)
#define NV_CXL_HDM_DECODER0_CTRL_OFFSET(i) (0x20 * (i) + 0x20)
#define NV_CXL_HDM_DECODER0_CTRL_COMMITTED BIT(10)
/* CXL r4.0, 8.1.8: Flex Bus DVSEC */
#ifndef PCI_DVSEC_CXL_FLEXBUS_PORT
#define PCI_DVSEC_CXL_FLEXBUS_PORT 7
#endif
#ifndef PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS 0xE
#endif
#ifndef PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_CACHE
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_CACHE BIT(0)
#endif
#ifndef PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_MEM
#define PCI_DVSEC_CXL_FLEXBUS_PORT_STATUS_MEM BIT(2)
#endif
/* CXL r4.0, 8.1.9: Register Locator DVSEC */
#ifndef PCI_DVSEC_CXL_REG_LOCATOR
#define PCI_DVSEC_CXL_REG_LOCATOR 8
#endif
#ifndef PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK1 0xC
#endif
#ifndef PCI_DVSEC_CXL_REG_LOCATOR_BIR
#define PCI_DVSEC_CXL_REG_LOCATOR_BIR GENMASK(2, 0)
#endif
#ifndef PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_ID GENMASK(15, 8)
#endif
#ifndef PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW
#define PCI_DVSEC_CXL_REG_LOCATOR_BLOCK_OFF_LOW GENMASK(31, 16)
#endif
#ifndef PCI_VENDOR_ID_CXL
#define PCI_VENDOR_ID_CXL 0x1e98
#endif
#endif /* _NV_PCI_H_ */
kernel-open/common/inc/nv-proto.h
+7 -2
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -49,6 +49,7 @@ void nv_revoke_gpu_mappings_locked(nv_state_t *);
NvUPtr nv_vm_map_pages (struct page **, NvU32, NvBool, NvBool);
void nv_vm_unmap_pages (NvUPtr, NvU32);
NvU64 nv_get_reclaimable_memory_usage(void);
NV_STATUS nv_alloc_contig_pages (nv_state_t *, nv_alloc_t *);
void nv_free_contig_pages (nv_alloc_t *);
@@ -84,7 +85,10 @@ void nvidia_modeset_remove (NvU32 gpuId);
void nvidia_modeset_probe (const nv_linux_state_t *nvl);
NvBool nv_is_uuid_in_gpu_exclusion_list (const char *);
NV_STATUS nv_parse_per_device_option_string(nvidia_stack_t *sp);
NV_STATUS nv_parse_per_device_option_string(nvidia_stack_t *sp,
nv_state_t *current_nv);
NV_STATUS nv_parse_per_device_binary_option_string(nvidia_stack_t *sp,
nv_state_t *current_nv);
void nv_enable_cdmm_mode(nvidia_stack_t *sp);
NvBool nv_is_galaxy_workstation (void);
@@ -103,6 +107,7 @@ NvBool nv_acpi_power_resource_method_present(struct pci_dev *);
int nv_linux_init_open_q(nv_linux_state_t *);
void nv_linux_stop_open_q(nv_linux_state_t *);
void nv_set_init_on_probe(nv_state_t *nv);
int nv_start_device(nv_state_t *nv, nvidia_stack_t *sp);
void nv_stop_device(nv_state_t *nv, nvidia_stack_t *sp);
#endif /* _NV_PROTO_H_ */
kernel-open/common/inc/nv-rethunk.h
+44
View File
@@ -0,0 +1,44 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _NV_RETHUNK_H_
#define _NV_RETHUNK_H_
#include "cpuopsys.h"
#if defined(NVCPU_X86_64)
__asm__(".pushsection .text");
__asm__(
".globl __x86_return_thunk;"
".type __x86_return_thunk,@function;"
"__x86_return_thunk:"
" .cfi_startproc;"
" ret;"
" int3;"
" .cfi_endproc;"
".size __x86_return_thunk, .-__x86_return_thunk;");
__asm__(".popsection");
#endif // NVCPU_X86_64
#endif // _NV_RETHUNK_H_
kernel-open/common/inc/nv-tegra-clk.h
+98
View File
@@ -0,0 +1,98 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef _NV_TEGRA_CLK_H_
#define _NV_TEGRA_CLK_H_
#include "nv.h"
struct nv_pci_tegra_devfreq_data {
const char *clk_name;
const char *icc_name;
const unsigned int gpc_fuse_field;
const TEGRASOC_DEVFREQ_CLK devfreq_clk;
};
static const struct nv_pci_tegra_devfreq_data tegra_devfreq_tables[][6] = {
{
{
.clk_name = "gpc0clk",
.icc_name = "gpu-write",
.gpc_fuse_field = BIT(0),
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_GPC,
},
{
.clk_name = "gpc1clk",
.icc_name = "gpu-write",
.gpc_fuse_field = BIT(1),
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_GPC,
},
{
.clk_name = "gpc2clk",
.icc_name = "gpu-write",
.gpc_fuse_field = BIT(2),
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_GPC,
},
{
.clk_name = "nvdclk",
.icc_name = "video-write",
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_NVD,
},
{
.clk_name = "sysclk"
},
{
.clk_name = "uprocclk"
},
},
{
{
.clk_name = "gpc0clk",
.icc_name = "gpu-write",
.gpc_fuse_field = BIT(0),
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_GPC,
},
{
.clk_name = "gpc1clk",
.icc_name = "gpu-write",
.gpc_fuse_field = BIT(1),
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_GPC,
},
{
.clk_name = "nvdclk",
.icc_name = "video-write",
.devfreq_clk = TEGRASOC_DEVFREQ_CLK_NVD,
},
{
.clk_name = "sysclk"
},
{
.clk_name = "uprocclk"
},
{
.clk_name = NULL
},
},
};
#endif /* _NV_TEGRA_CLK_H_ */
kernel-open/common/inc/nv.h
+32 -11
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 1999-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 1999-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -29,13 +29,7 @@
#include <nvlimits.h>
#if defined(NV_KERNEL_INTERFACE_LAYER) && defined(__FreeBSD__)
#include <sys/stddef.h> // NULL
#elif defined(NV_KERNEL_INTERFACE_LAYER) && defined(NV_LINUX)
#include <linux/stddef.h> // NULL
#else
#include <stddef.h> // NULL
#endif
#include <nv_stddef.h> // NULL
#include <nvstatus.h>
#include "nv_stdarg.h"
@@ -43,6 +37,8 @@
#include <nv-firmware.h>
#include <nv-ioctl.h>
#include <nv-ioctl-numa.h>
#include <nv-dev-printf.h>
#include <nv-kernel-interface-api.h>
#include <nvmisc.h>
#include <os/nv_memory_area.h>
@@ -59,8 +55,6 @@ extern const NvBool nv_is_rm_firmware_supported_os;
#include <nvi2c.h>
#include <nvimpshared.h>
#include <nv-kernel-interface-api.h>
#define GPU_UUID_LEN (16)
/*
@@ -106,6 +100,13 @@ typedef enum _TEGRASOC_DEVFREQ_CLK
TEGRASOC_DEVFREQ_CLK_NVD,
} TEGRASOC_DEVFREQ_CLK;
typedef enum _TEGRASOC_DEVFREQ_CLK_BOOST_TYPE
{
TEGRASOC_DEVFREQ_CLK_BOOST_TYPE_DEFAULT,
TEGRASOC_DEVFREQ_CLK_BOOST_TYPE_FMAX,
TEGRASOC_DEVFREQ_CLK_BOOST_TYPE_RATED_TDP,
} TEGRASOC_DEVFREQ_CLK_BOOST_TYPE;
/*!
* @brief The order of the display clocks in the below defined enum
* should be synced with below mapping array and macro.
@@ -478,6 +479,7 @@ typedef struct nv_state_t
NvBool valid;
NvU8 uuid[GPU_UUID_LEN];
NvBool pci_uuid_read_attempted;
NvBool pci_uuid_from_dvsec_pdi;
NV_STATUS pci_uuid_status;
} nv_uuid_cache;
void *handle;
@@ -514,6 +516,7 @@ typedef struct nv_state_t
NvBool supports_tegra_igpu_rg;
NvBool is_tegra_pci_igpu_rg_enabled;
NvU32 tegra_pci_igpu_pg_mask;
NvU32 gpc_fuse_status_offset;
NvBool primary_vga;
@@ -643,6 +646,15 @@ typedef struct nv_state_t
/* Bool to check if the GPU is a CXL device */
NvBool is_cxl_dev;
/*
* Tracks user-requested persistence mode for backward compatibility
* when init-on-probe is active. Existing scripts and tools expect
* PM0/PM1 state to be reflected in queries even though persistence
* mode has no functional effect under init-on-probe.
* This field will be removed when legacy persistence mode is
* deprecated in the future.
*/
NvBool cached_persistence_mode;
} nv_state_t;
#define NVFP_TYPE_NONE ((NvU32)0x0)
@@ -850,6 +862,7 @@ typedef enum {
// Flags needed by osAllocPagesNode / os_alloc_pages_node
#define NV_ALLOC_PAGES_NODE_NONE 0x0
#define NV_ALLOC_PAGES_NODE_SKIP_RECLAIM 0x1
#define NV_ALLOC_PAGES_NODE_DO_ACCOUNT 0x2
/*
** where we hide our nv_state_t * ...
@@ -990,6 +1003,8 @@ void NV_API_CALL nv_dma_unmap_mmio (nv_dma_device_t *, NvU64, NvU6
void NV_API_CALL nv_dma_cache_invalidate (nv_dma_device_t *, void *);
void* NV_API_CALL nv_dma_get_dev_pagemap (NvU64);
void NV_API_CALL nv_dma_put_dev_pagemap (void *);
NV_STATUS NV_API_CALL nv_dma_init_sysmem_window_for_fabric_access (nv_state_t *, NvU64, NvU64 *, NvU64 *, NvBool *);
void NV_API_CALL nv_dma_destroy_sysmem_window_for_fabric_access (nv_state_t *, NvU64, NvU64);
NvBool NV_API_CALL nv_grdma_pci_topology_supported(nv_state_t *, nv_dma_device_t *);
NvS32 NV_API_CALL nv_start_rc_timer (nv_state_t *);
@@ -1061,6 +1076,7 @@ NvBool NV_API_CALL nv_match_gpu_os_info(nv_state_t *, void *);
void NV_API_CALL nv_get_updated_emu_seg(NvU32 *start, NvU32 *end);
void NV_API_CALL nv_get_screen_info(nv_state_t *, NvU64 *, NvU32 *, NvU32 *, NvU32 *, NvU32 *, NvU64 *);
void NV_API_CALL nv_set_gpu_pg_mask(nv_state_t *);
void NV_API_CALL nv_trigger_gpu_flr(nv_state_t *);
struct dma_buf;
typedef struct nv_dma_buf nv_dma_buf_t;
@@ -1083,6 +1099,7 @@ NvBool NV_API_CALL nv_s2idle_pm_configured (void);
NvBool NV_API_CALL nv_pci_tegra_register_power_domain (nv_state_t *, NvBool);
NvBool NV_API_CALL nv_pci_tegra_pm_init (nv_state_t *);
void NV_API_CALL nv_pci_tegra_pm_deinit (nv_state_t *);
NV_STATUS NV_API_CALL nv_pci_read_gpu_pdi_from_dvsec(nv_state_t *, NvU64 *);
NvBool NV_API_CALL nv_is_chassis_notebook (void);
void NV_API_CALL nv_allow_runtime_suspend (nv_state_t *nv);
@@ -1217,6 +1234,10 @@ NvBool NV_API_CALL rm_is_supported_pci_device(NvU8 pci_class,
NvU16 subsystem_vendor,
NvU16 subsystem_device,
NvBool print_legacy_warning);
NvBool NV_API_CALL rm_is_nvidia_gpu_device(NvU8 pci_class,
NvU8 pci_subclass,
NvU16 vendor,
NvU16 device);
void NV_API_CALL rm_i2c_remove_adapters (nvidia_stack_t *, nv_state_t *);
NvBool NV_API_CALL rm_i2c_is_smbus_capable (nvidia_stack_t *, nv_state_t *, void *);
@@ -1251,7 +1272,7 @@ void NV_API_CALL rm_dma_buf_unmap_mem_handle(nvidia_stack_t *, nv_state_t
NV_STATUS NV_API_CALL rm_dma_buf_get_client_and_device(nvidia_stack_t *,
nv_state_t *, NvHandle, NvHandle,
NvU8, NvHandle *, NvHandle *,
NvHandle *, void **, NvBool *, NvBool *);
NvHandle *, void **, NvBool *);
void NV_API_CALL rm_dma_buf_put_client_and_device(nvidia_stack_t *, nv_state_t *, NvHandle, NvHandle, NvHandle, void *);
void NV_API_CALL rm_kernel_rmapi_op(nvidia_stack_t *sp, void *ops_cmd);
kernel-open/common/inc/nv_stddef.h
+38
View File
@@ -0,0 +1,38 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
* DEALINGS IN THE SOFTWARE.
*/
#ifndef NV_STDDEF_H
#define NV_STDDEF_H
#include "cpuopsys.h"
#if defined(NV_KERNEL_INTERFACE_LAYER) && NVOS_IS_FREEBSD
#include <sys/stddef.h>
#include <sys/types.h>
#elif defined(NV_KERNEL_INTERFACE_LAYER) && NVOS_IS_LINUX
#include <linux/types.h>
#else
#include <stddef.h>
#endif
#endif // NV_STDDEF_H
kernel-open/common/inc/nv_uvm_types.h
+26
View File
@@ -563,6 +563,10 @@ typedef struct UvmPlatformInfo_tag
// Out: ATS (Address Translation Services) is supported
NvBool atsSupported;
// Out: Indicates that at least one non-ats device is present in the
// system
NvBool nonAtsDevicePresent;
// Out: True if HW trusted execution, such as AMD's SEV-SNP or Intel's TDX,
// is enabled in the VM, indicating that Confidential Computing must be
// also enabled in the GPU(s); these two security features are either both
@@ -718,6 +722,28 @@ typedef struct UvmGpuInfo_tag
// to direct NVLink peers.
NvBool nvlDirectConnect;
NvU64 nvlDirectConnectMemoryWindowStart;
// Compute GPU has no vidmem, used to detect iGPUs.
NvBool gpuArchIsZeroFb;
struct
{
// Indicates if fabric-based sysmem access is supported on this GPU
NvBool bSupported;
// Start of the FLA window mapped to local CPU sysmem
NvU64 flaStart;
// Size of the FLA window starting from flaWindowStart
NvU64 flaSize;
// DMA/IOVA address corresponding to the start of local CPU sysmem
NvU64 dmaAddrBase;
// Indicates if DMA addresses are identity-mapped (no IOMMU translation).
// If false, physical sysmem ranges need to be linked into the IOVA range.
NvBool bIdentityDmaMap;
} flaWindowForCpuMemNode;
} UvmGpuInfo;
typedef struct UvmGpuFbInfo_tag
kernel-open/common/inc/nvkms-api-types.h
+31 -14
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2014-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2014-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -52,6 +52,9 @@
#define NVKMS_OLUT_FP_NORM_SCALE_DEFAULT 0xffffffff
#define NVKMS_HDMI_VSIF_METADATA_MIN_PAYLOAD_SIZE 3
#define NVKMS_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE 27
typedef NvU32 NvKmsDeviceHandle;
typedef NvU32 NvKmsDispHandle;
typedef NvU32 NvKmsConnectorHandle;
@@ -186,6 +189,8 @@ enum NvKmsEventType {
NVKMS_EVENT_TYPE_DPY_ATTRIBUTE_CHANGED,
NVKMS_EVENT_TYPE_FRAMELOCK_ATTRIBUTE_CHANGED,
NVKMS_EVENT_TYPE_FLIP_OCCURRED,
NVKMS_EVENT_TYPE_DPY_CONTENT_PROTECTION_CHANGED,
NVKMS_EVENT_TYPE_DPY_CP_TOPOLOGY_CHANGED,
};
enum NvKmsFlipResult {
@@ -497,19 +502,6 @@ struct NvKmsCompositionCapabilities {
};
struct NvKmsLayerCapabilities {
/*!
* Whether Layer supports the window mode. If window mode is supported,
* then clients can set the layer's dimensions so that they're smaller than
* the viewport, and can also change the output position of the layer to a
* non-(0, 0) position.
*
* NOTE: Dimension changes are currently unsupported for the main layer,
* and output position changes for the main layer are currently only
* supported via IOCTL_SET_LAYER_POSITION but not via flips. Support for
* these is coming soon, via changes to flip code.
*/
NvBool supportsWindowMode :1;
/*!
* Whether layer supports ICtCp pipe.
*/
@@ -782,6 +774,14 @@ struct NvKmsVblankSemControlData {
struct NvKmsVblankSemControlDataOneHead head[NV_MAX_HEADS];
};
enum NvKmsContentProtection {
NVKMS_CONTENT_PROTECTION_OFF = 0,
NVKMS_CONTENT_PROTECTION_HDCP1X_ON = 1,
NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE0_ON = 2,
NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE1_ON = 3,
NVKMS_CONTENT_PROTECTION_FAILED = 4,
};
/*
* Dithering control enums, shared between NVKMS API and KAPI.
* These are defined here (instead of nvkms-api.h) so that nvkms-kapi.h
@@ -806,4 +806,21 @@ enum NvKmsDpyAttributeRequestedDitheringModeValue {
typedef void (*NVRgInterruptCallbackProc)(NvU64 clientData,
NvU64 timestamp);
/*! hdcp topology struct defined in drivers/unix/common/inc/nv_hdcp_topology.h */
#define NVKMS_HDCP_TOPOLOGY_SIZE (1296)
/*
* HDMI Vendor Specific InfoFrame (VSIF) metadata.
* The payload is a 27 byte array where the first three bytes are
* the vendor OUI, and the remaining bytes are the infoframe content.
* payloadSize == 0 means disabled (no VSIF override).
* When payloadSize > 0, it must be in the range
* [NVKMS_HDMI_VSIF_METADATA_MIN_PAYLOAD_SIZE,
* NVKMS_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE] to enable the VSIF.
*/
struct NvKmsHdmiVsifMetadata {
NvU8 payloadSize;
NvU8 payload[NVKMS_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE];
};
#endif /* NVKMS_API_TYPES_H */
kernel-open/common/inc/nvkms-kapi.h
+55 -59
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2015-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2015-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -174,7 +174,18 @@ struct NvKmsKapiDeviceResourcesInfo {
NvBool supportsSyncpts;
NvBool supportsColorPassthrough;
NvBool contiguousPhysicalMappings;
/*
* Whether this device is an SoC product whose display is driven by
* the dGPU-style display class and needs the resume connector
* re-probe workaround (bug 6422321). Do not use this for anything
* else: it exists only for that workaround and is tracked for
* removal by bug 6556017.
*/
NvBool isSocDgpuDisplayNeedingWar;
} caps;
NvU64 supportedSurfaceMemoryFormats[NVKMS_KAPI_LAYER_MAX];
@@ -317,25 +328,28 @@ struct NvKmsKapiLayerConfig {
NvBool blendCtm : 1;
} enabled;
} matrixOverrides;
NvBool precompColorPassthrough;
};
struct NvKmsKapiLayerRequestedConfig {
struct NvKmsKapiLayerConfig config;
struct {
NvBool surfaceChanged : 1;
NvBool srcXYChanged : 1;
NvBool srcWHChanged : 1;
NvBool dstXYChanged : 1;
NvBool dstWHChanged : 1;
NvBool cscChanged : 1;
NvBool inputTfChanged : 1;
NvBool outputTfChanged : 1;
NvBool inputColorSpaceChanged : 1;
NvBool inputColorRangeChanged : 1;
NvBool hdrMetadataChanged : 1;
NvBool matrixOverridesChanged : 1;
NvBool ilutChanged : 1;
NvBool tmoChanged : 1;
NvBool surfaceChanged : 1;
NvBool srcXYChanged : 1;
NvBool srcWHChanged : 1;
NvBool dstXYChanged : 1;
NvBool dstWHChanged : 1;
NvBool cscChanged : 1;
NvBool inputTfChanged : 1;
NvBool outputTfChanged : 1;
NvBool inputColorSpaceChanged : 1;
NvBool inputColorRangeChanged : 1;
NvBool hdrMetadataChanged : 1;
NvBool matrixOverridesChanged : 1;
NvBool ilutChanged : 1;
NvBool tmoChanged : 1;
NvBool precompColorPassthroughChanged : 1;
} flags;
};
@@ -386,22 +400,10 @@ struct NvKmsKapiHeadModeSetConfig {
struct NvKmsHDRStaticMetadata staticMetadata;
} hdrInfoFrame;
struct NvKmsHdmiVsifMetadata hdmiVsifMetadata;
enum NvKmsOutputColorimetry colorimetry;
struct {
struct {
NvU32 depth;
NvU32 start;
NvU32 end;
struct NvKmsLutRamps *pRamps;
} input;
struct {
NvBool enabled;
struct NvKmsLutRamps *pRamps;
} output;
} lut;
struct {
NvBool enabled;
struct NvKmsKapiSurface *lutSurface;
@@ -416,21 +418,23 @@ struct NvKmsKapiHeadModeSetConfig {
enum NvKmsDpyAttributeRequestedDitheringValue state;
enum NvKmsDpyAttributeRequestedDitheringModeValue mode;
} dithering;
NvBool postcompColorPassthrough;
};
struct NvKmsKapiHeadRequestedConfig {
struct NvKmsKapiHeadModeSetConfig modeSetConfig;
struct {
NvBool activeChanged : 1;
NvBool displaysChanged : 1;
NvBool modeChanged : 1;
NvBool hdrInfoFrameChanged : 1;
NvBool colorimetryChanged : 1;
NvBool legacyIlutChanged : 1;
NvBool legacyOlutChanged : 1;
NvBool olutChanged : 1;
NvBool olutFpNormScaleChanged : 1;
NvBool ditheringChanged : 1;
NvBool activeChanged : 1;
NvBool displaysChanged : 1;
NvBool modeChanged : 1;
NvBool hdrInfoFrameChanged : 1;
NvBool colorimetryChanged : 1;
NvBool olutChanged : 1;
NvBool olutFpNormScaleChanged : 1;
NvBool ditheringChanged : 1;
NvBool hdmiVsifMetadataChanged : 1;
NvBool postcompColorPassthroughChanged : 1;
} flags;
struct NvKmsKapiCursorRequestedConfig cursorRequestedConfig;
@@ -465,6 +469,16 @@ struct NvKmsKapiEventDisplayChanged {
NvKmsKapiDisplay display;
};
struct NvKmsKapiEventDisplayCpChanged {
NvKmsKapiDisplay display;
enum NvKmsContentProtection cp;
};
struct NvKmsKapiEventDisplayCpTopologyChanged {
NvKmsKapiDisplay display;
const void *topology;
};
struct NvKmsKapiEventDynamicDisplayConnected {
NvKmsKapiDisplay display;
};
@@ -496,6 +510,8 @@ struct NvKmsKapiEvent {
struct NvKmsKapiEventDisplayChanged displayChanged;
struct NvKmsKapiEventDynamicDisplayConnected dynamicDisplayConnected;
struct NvKmsKapiEventFlipOccurred flipOccurred;
struct NvKmsKapiEventDisplayCpChanged displayCpChanged;
struct NvKmsKapiEventDisplayCpTopologyChanged displayCpTopologyChanged;
} u;
};
@@ -1606,26 +1622,6 @@ struct NvKmsKapiFunctionsTable {
NvU32 semaphoreIndex
);
/*!
* Check or wait on a head's LUT notifier.
*
* \param [in] device A device allocated using allocateDevice().
*
* \param [in] head The head to check for LUT completion.
*
* \param [in] waitForCompletion If true, wait for the notifier in NvKms
* before returning.
*
* \param [out] complete Returns whether the notifier has completed.
*/
NvBool
(*checkLutNotifier)
(
struct NvKmsKapiDevice *device,
NvU32 head,
NvBool waitForCompletion
);
/*
* Notify NVKMS that the system's framebuffer console has been disabled and
* the reserved allocation for the old framebuffer console can be unmapped.
kernel-open/common/inc/nvmisc.h
+5
View File
@@ -259,8 +259,13 @@ extern "C" {
//
// 64 Bit Versions
//
#if defined(NV_MISRA_COMPLIANCE_REQUIRED)
#define DRF_SHIFT64(drf) ((DRF_BASE(drf)) % 64U)
#define DRF_MASK64(drf) (NV_U64_MAX>>(63U-((DRF_EXTENT(drf)) % 64U)+((DRF_BASE(drf)) % 64U)))
#else
#define DRF_SHIFT64(drf) ((DRF_ISBIT(0,drf)) % 64)
#define DRF_MASK64(drf) (NV_U64_MAX>>(63-((DRF_ISBIT(1,drf)) % 64)+((DRF_ISBIT(0,drf)) % 64)))
#endif
#define DRF_SHIFTMASK64(drf) (DRF_MASK64(drf)<<(DRF_SHIFT64(drf)))
#define DRF_DEF64(d,r,f,c) (((NvU64)(NV ## d ## r ## f ## c))<<DRF_SHIFT64(NV ## d ## r ## f))
kernel-open/common/inc/nvstatuscodes.h
+16
View File
@@ -168,6 +168,22 @@ NV_STATUS_CODE(NV_ERR_MC_FLA_OFFSET_TABLE_FULL, 0x0000008A, "Mul
NV_STATUS_CODE(NV_ERR_OPERATION_ABORTED, 0x0000008B, "Operation has aborted")
NV_STATUS_CODE(NV_ERR_DMA_XFER_FAILED, 0x0000008C, "DMA transfer failed")
NV_STATUS_CODE(NV_ERR_RESOURCE_ACCOUNTING_HARD_LIMIT_EXCEEDED, 0x0000008D, "Resource accounting rejected request (hard limit)")
NV_STATUS_CODE(NV_ERR_NVLINK_FW_NOT_LOADED, 0x0000008E, "MSE firmware is not loaded")
NV_STATUS_CODE(NV_ERR_INTR_STUCK, 0x0000008F, "Interrupt is stuck")
NV_STATUS_CODE(NV_ERR_GMCAPI_INVALID_MCTP, 0x00000090, "Invalid MCTP header")
NV_STATUS_CODE(NV_ERR_GMCAPI_INVALID_NVDM_TYPE, 0x00000091, "Invalid NVDM message type")
NV_STATUS_CODE(NV_ERR_GMCAPI_INVALID_COMMAND_ID, 0x00000092, "The given GMCAPI command ID is not valid")
NV_STATUS_CODE(NV_ERR_GMCAPI_REQUEST_SIZE_TOO_LARGE, 0x00000093, "GMCAPI request size is too large")
NV_STATUS_CODE(NV_ERR_GMCAPI_REQUEST_SIZE_TOO_SMALL, 0x00000094, "GMCAPI request size is too small")
NV_STATUS_CODE(NV_ERR_GMCAPI_MAX_RESPONSE_SIZE_TOO_SMALL, 0x00000095, "GMCAPI maximum response size is too small")
// bug 6141566: Unable to add new NV_STATUS values temporarily
//NV_STATUS_CODE(NV_ERR_GMCAPI_RESERVED_FIELD_NOT_ZERO, 0x00000096, "GMCAPI reserved field not set to zero")
//NV_STATUS_CODE(NV_ERR_GMCAPI_INVALID_FLAGS, 0x00000097, "GMCAPI flags for this call are not valid")
#define NV_ERR_GMCAPI_RESERVED_FIELD_NOT_ZERO ((NV_STATUS)0x00000096)
#define NV_ERR_GMCAPI_INVALID_FLAGS ((NV_STATUS)0x00000097)
// Status codes 0x98..0x9D are reserved for GMCAPI
NV_STATUS_CODE(NV_ERR_GMCAPI_INTERNAL_RM_ERROR, 0x0000009E, "RM has hit an internal error while servicing GMCAPI")
NV_STATUS_CODE(NV_ERR_GMCAPI_GENERIC_ERROR, 0x0000009F, "Failed to service GMCAPI: Generic error")
// Warnings:
NV_STATUS_CODE(NV_WARN_HOT_SWITCH, 0x00010001, "WARNING Hot switch")
kernel-open/common/inc/nvtypes.h
+14
View File
@@ -524,6 +524,20 @@ typedef struct
#define NV_ALIGNOF(type) __alignof__(type)
#elif defined(_MSC_VER)
#define NV_ALIGNOF(type) __alignof(type)
#endif
//
// This tag can be applied to a type to mark it as ABI-stable and block most
// changes to the internal layout. In all driver builds, this tag is a NOP and
// is there for human benefit only. However, static checker tools may support
// the custom attribute and use it to verify no changes across releases.
// Tag comes between struct/enum keyword and the name, like GCC type attributes.
// e.g. typedef struct NV_ABI_STABLE MyStableStruct { ... } MyStableStruct;
//
#if defined(NV_ABI_STABLE_CHECKED_BUILD)
#define NV_ABI_STABLE __attribute__((nv_abi_stable))
#else
#define NV_ABI_STABLE
#endif
/***************************************************************************\
kernel-open/common/inc/os-interface.h
+27 -6
View File
@@ -191,7 +191,7 @@ NV_STATUS NV_API_CALL os_put_page (NvU64 address);
NvU32 NV_API_CALL os_get_page_refcount (NvU64 address);
NvU32 NV_API_CALL os_count_tail_pages (NvU64 address);
void NV_API_CALL os_free_pages_phys (NvU64, NvU32);
NV_STATUS NV_API_CALL os_open_temporary_file (void **);
NV_STATUS NV_API_CALL os_allocate_temporary_file (void **, NvU64);
void NV_API_CALL os_close_file (void *);
NV_STATUS NV_API_CALL os_write_file (void *, NvU8 *, NvU64, NvU64);
NV_STATUS NV_API_CALL os_read_file (void *, NvU8 *, NvU64, NvU64);
@@ -213,7 +213,7 @@ int NV_API_CALL os_nv_cap_validate_and_dup_fd (const nv_cap_t *, int
void NV_API_CALL os_nv_cap_close_fd (int);
NvS32 NV_API_CALL os_imex_channel_get (NvU64);
NvS32 NV_API_CALL os_imex_channel_count (void);
NV_STATUS NV_API_CALL os_tegra_igpu_perf_boost (void *, NvBool, NvU32);
NV_STATUS NV_API_CALL os_tegra_igpu_perf_boost (void *, NvBool, NvU32, int);
NV_STATUS NV_API_CALL os_get_tegra_platform (NvU32 *);
enum os_pci_req_atomics_type {
@@ -223,6 +223,7 @@ enum os_pci_req_atomics_type {
};
NV_STATUS NV_API_CALL os_enable_pci_req_atomics (void *, enum os_pci_req_atomics_type);
void NV_API_CALL os_pci_trigger_flr(void *handle);
NvU64 NV_API_CALL os_get_reclaimable_memory_usage(void);
NV_STATUS NV_API_CALL os_get_numa_node_memory_usage (NvS32, NvU64 *, NvU64 *);
NV_STATUS NV_API_CALL os_numa_add_gpu_memory (void *, NvU64, NvU64, NvU32 *);
NV_STATUS NV_API_CALL os_numa_remove_gpu_memory (void *, NvU64, NvU64, NvU32);
@@ -264,6 +265,7 @@ extern NvBool os_imex_channel_is_supported;
void NV_API_CALL out_string(const char *str);
int NV_API_CALL nv_printf(NvU32 debuglevel, const char *printf_format, ...);
int NV_API_CALL nv_vprintf(NvU32 debuglevel, const char *printf_format, va_list arglist);
#define NV_DEV_PRINTF(debuglevel, nv, format, ... ) \
nv_printf(debuglevel, "NVRM: GPU " NV_PCI_DEV_FMT ": " format, NV_PCI_DEV_FMT_ARGS(nv), ## __VA_ARGS__)
@@ -289,14 +291,33 @@ int NV_API_CALL nv_printf(NvU32 debuglevel, const char *printf_format, ...);
#define OS_CGROUP_IMPL_NONE 0
#define OS_CGROUP_IMPL_MISC 1
#define OS_CGROUP_IMPL_DMEM 2
void* NV_API_CALL os_cgroup_for_pid(int pid, void *pidInfo, int impl);
#if defined(NV_LINUX)
NvU32 NV_API_CALL os_cgroup_implementation(void);
void* NV_API_CALL os_dmem_cgroup_register_region(NvU64 size, const char *name);
void NV_API_CALL os_dmem_cgroup_unregister_region(void *region);
void* NV_API_CALL os_dmem_cgroup_register_region(const char *name, NvU64 size, NvU64 precharge, void **prechargePool);
void NV_API_CALL os_dmem_cgroup_unregister_region(void *region, void *prechargePool, NvU64 precharge);
NV_STATUS NV_API_CALL os_dmem_cgroup_try_charge(void *region, NvU64 size, void **ret_pool, void **ret_limit_pool);
void NV_API_CALL os_dmem_cgroup_uncharge(void *pool, NvU64 size);
void* NV_API_CALL os_cgroup_for_pid(int pid, void *pidInfo);
void NV_API_CALL os_dmem_cgroup_pool_state_put(void *pool);
NvBool NV_API_CALL os_dmem_cgroup_state_evict_valuable(void *limit_pool, void *test_pool, NvBool ignore_low, NvBool *ret_hit_low);
void* NV_API_CALL os_cgroup_get_from_fd(NvU32 fd);
void NV_API_CALL os_cgroup_put(void *cgroup);
void NV_API_CALL os_cgroup_get(void *cgroup);
void* NV_API_CALL os_cgroup_parent(void *cgroup);
#else // !defined(NV_LINUX)
#define os_cgroup_implementation() OS_CGROUP_IMPL_NONE
#define os_dmem_cgroup_register_region(...) NULL
#define os_dmem_cgroup_unregister_region(...)
#define os_dmem_cgroup_try_charge(...) NV_OK
#define os_dmem_cgroup_uncharge(...)
#define os_dmem_cgroup_pool_state_put(...)
#define os_dmem_cgroup_state_evict_valuable(...) NV_FALSE
#define os_cgroup_get_from_fd(...) NULL
#define os_cgroup_put(...)
#define os_cgroup_get(...)
#define os_cgroup_parent(...) NULL
#endif // defined(NV_LINUX)
#endif /* OS_INTERFACE_H */
kernel-open/conftest.sh
+347 -71
View File
@@ -3,8 +3,8 @@
PATH="${PATH}:/bin:/sbin:/usr/bin"
# make sure we are in the directory containing this script
SCRIPTDIR=`dirname $0`
cd $SCRIPTDIR
SCRIPTDIR=$(dirname "$0")
cd "$SCRIPTDIR" || exit 1
CC="$1"
ARCH=$2
@@ -58,8 +58,8 @@ append_conftest() {
# older driver versions
#
while read LINE; do
echo ${LINE}
while read -r LINE; do
echo "${LINE}"
done
}
@@ -80,7 +80,7 @@ test_header_presence() {
TEST_CFLAGS="-E -M -I${NVIDIA_OOT_PATH}/include $CFLAGS"
file="$1"
file_define=NV_`echo $file | tr '/.-' '___' | tr 'a-z' 'A-Z'`_PRESENT
file_define=NV_$(echo "$file" | tr '/.-' '___' | tr 'a-z' 'A-Z')_PRESENT
CODE="#include <$file>"
@@ -101,7 +101,7 @@ test_header_presence() {
}
build_cflags() {
ISYSTEM=`$CC -print-file-name=include 2> /dev/null`
ISYSTEM=$($CC -print-file-name=include 2> /dev/null)
BASE_CFLAGS="-O2 -D__KERNEL__ \
-DKBUILD_BASENAME=\"#conftest$$\" -DKBUILD_MODNAME=\"#conftest$$\" \
-nostdinc -isystem $ISYSTEM \
@@ -130,7 +130,7 @@ build_cflags() {
KERNEL_ARCH="$ARCH"
if [ "$ARCH" = "i386" -o "$ARCH" = "x86_64" ]; then
if [ "$ARCH" = "i386" ] || [ "$ARCH" = "x86_64" ]; then
if [ -d "$SOURCES/arch/x86" ]; then
KERNEL_ARCH="x86"
fi
@@ -145,12 +145,11 @@ build_cflags() {
# includes if that platform is enabled in the configuration file, which
# may have a definition like this:
# #define CONFIG_ARCH_<MACHUPPERCASE> 1
for _mach_dir in `ls -1d $SOURCES/arch/$KERNEL_ARCH/mach-* 2>/dev/null`; do
_mach=`echo $_mach_dir | \
for _mach_dir in $(ls -1d "$SOURCES/arch/$KERNEL_ARCH"/mach-* 2>/dev/null); do
_mach=$(echo "$_mach_dir" | \
sed -e "s,$SOURCES/arch/$KERNEL_ARCH/mach-,," | \
tr 'a-z' 'A-Z'`
grep "CONFIG_ARCH_$_mach \+1" $AUTOCONF_FILE > /dev/null 2>&1
if [ $? -eq 0 ]; then
tr 'a-z' 'A-Z')
if grep "CONFIG_ARCH_$_mach \+1" "$AUTOCONF_FILE" > /dev/null 2>&1; then
MACH_CFLAGS="$MACH_CFLAGS -I$_mach_dir/include"
fi
done
@@ -189,7 +188,7 @@ build_cflags() {
# Newer versions of gcc-goto.sh don't print anything on success, but
# this is okay, since it's no longer necessary to set CC_HAVE_ASM_GOTO
# based on the output of those versions of gcc-goto.sh.
if [ `/bin/sh "$GCC_GOTO_SH" "$CC"` = "y" ]; then
if [ "$(/bin/sh "$GCC_GOTO_SH" "$CC")" = "y" ]; then
CFLAGS="$CFLAGS -DCC_HAVE_ASM_GOTO"
fi
fi
@@ -202,8 +201,7 @@ build_cflags() {
# fails to get compiled, because conftest.sh runs outside of Kbuild it ends
# up building without -mfentry and CC_USING_FENTRY flags.
#
grep "CONFIG_HAVE_FENTRY \+1" $AUTOCONF_FILE > /dev/null 2>&1
if [ $? -eq 0 ]; then
if grep "CONFIG_HAVE_FENTRY \+1" "$AUTOCONF_FILE" > /dev/null 2>&1; then
echo "" > conftest$$.c
$CC -mfentry -c -x c conftest$$.c > /dev/null 2>&1
@@ -247,7 +245,7 @@ test_configuration_option() {
# Check to see if the given configuration option is defined
#
get_configuration_option $1 >/dev/null 2>&1
get_configuration_option "$1" >/dev/null 2>&1
return $?
@@ -355,7 +353,7 @@ check_symbol_exists() {
# space separated list instead of semicolon separated so we can iterate
# over it.
if [ -z "${CONFTEST_BSD_KMODPATHS}" ] ; then
KMODPATHS=`sysctl -n kern.module_path | sed -e "s/;/ /g"`
KMODPATHS=$(sysctl -n kern.module_path | sed -e "s/;/ /g")
else
KMODPATHS="${CONFTEST_BSD_KMODPATHS}"
fi
@@ -379,7 +377,7 @@ export_symbol_present_conftest() {
SYMBOL="$1"
if check_symbol_exists $SYMBOL; then
if check_symbol_exists "$SYMBOL"; then
echo "#define NV_IS_EXPORT_SYMBOL_PRESENT_$SYMBOL 1" |
append_conftest "symbols"
else
@@ -422,9 +420,9 @@ get_configuration_option() {
NEW_FILE="generated/autoconf.h"
FILE=""
if [ -f $HEADERS/$NEW_FILE -o -f $OUTPUT/include/$NEW_FILE ]; then
if [ -f "$HEADERS/$NEW_FILE" ] || [ -f "$OUTPUT/include/$NEW_FILE" ]; then
FILE=$NEW_FILE
elif [ -f $HEADERS/$OLD_FILE -o -f $OUTPUT/include/$OLD_FILE ]; then
elif [ -f "$HEADERS/$OLD_FILE" ] || [ -f "$OUTPUT/include/$OLD_FILE" ]; then
FILE=$OLD_FILE
fi
@@ -435,9 +433,9 @@ get_configuration_option() {
# via a compile check, and print the option's value.
#
if [ -f $HEADERS/$FILE ]; then
if [ -f "$HEADERS/$FILE" ]; then
INCLUDE_DIRECTORY=$HEADERS
elif [ -f $OUTPUT/include/$FILE ]; then
elif [ -f "$OUTPUT/include/$FILE" ]; then
INCLUDE_DIRECTORY=$OUTPUT/include
else
return 1
@@ -451,7 +449,7 @@ get_configuration_option() {
$OPTION
" > conftest$$.c
$CC -E -P -I$INCLUDE_DIRECTORY -o conftest$$ conftest$$.c > /dev/null 2>&1
$CC -E -P -I"$INCLUDE_DIRECTORY" -o conftest$$ conftest$$.c > /dev/null 2>&1
if [ -e conftest$$ ]; then
tr -d '\r\n\t ' < conftest$$
@@ -461,8 +459,8 @@ get_configuration_option() {
rm -f conftest$$.c conftest$$
else
CONFIG=$OUTPUT/.config
if [ -f $CONFIG ] && grep "^$OPTION=" $CONFIG; then
grep "^$OPTION=" $CONFIG | cut -f 2- -d "="
if [ -f "$CONFIG" ] && grep "^$OPTION=" "$CONFIG"; then
grep "^$OPTION=" "$CONFIG" | cut -f 2- -d "="
RET=$?
fi
fi
@@ -1461,6 +1459,40 @@ compile_test() {
compile_check_conftest "$CODE" "NV_DRM_SYSFS_CONNECTOR_STATUS_EVENT_PRESENT" "" "functions"
;;
drm_connector_attach_content_protection_property)
#
# Determine if drm_connector_attach_content_protection_property() is present.
#
# Commit 6a99099fe1d6c ("drm/display: Move HDCP helpers into display-helper module")
# in Linux v5.19.
#
CODE="
#if defined(NV_DRM_DISPLAY_DRM_HDCP_HELPER_H_PRESENT)
#include <drm/display/drm_hdcp_helper.h>
#endif
void conftest_drm_connector_attach_content_protection_property(void) {
drm_connector_attach_content_protection_property();
}"
compile_check_conftest "$CODE" "NV_DRM_CONNECTOR_ATTACH_CONTENT_PROTECTION_PROPERTY_PRESENT" "" "functions"
;;
drm_hdcp_update_content_protection)
#
# Determine if drm_hdcp_update_content_protection() is present.
#
# Commit 6a99099fe1d6c ("drm/display: Move HDCP helpers into display-helper module")
# in Linux v5.19.
#
CODE="
#if defined(NV_DRM_DISPLAY_DRM_HDCP_HELPER_H_PRESENT)
#include <drm/display/drm_hdcp_helper.h>
#endif
void conftest_drm_hdcp_update_content_protection(void) {
drm_hdcp_update_content_protection();
}"
compile_check_conftest "$CODE" "NV_DRM_HDCP_UPDATE_CONTENT_PROTECTION_PRESENT" "" "functions"
;;
pde_data)
#
# Determine if the pde_data() function is present.
@@ -2086,6 +2118,23 @@ compile_test() {
compile_check_conftest "$CODE" "NV_PCI_DRIVER_HAS_DRIVER_MANAGED_DMA" "" "types"
;;
device_has_tdi_enabled)
#
# Determine if "struct device" has .tdi_enabled member.
#
# tdi_enabled is provided by the downstream RMEDA module which
# supports CCA. This conftest is needed until upstream support
# is available.
#
CODE="
#include <linux/device.h>
int conftest_device_has_tdi_enabled(void) {
return offsetof(struct device, tdi_enabled);
}"
compile_check_conftest "$CODE" "NV_DEVICE_HAS_TDI_ENABLED" "" "types"
;;
drm_file_get_master)
#
# Determine if function drm_file_get_master() is present.
@@ -2202,11 +2251,11 @@ compile_test() {
;;
is_export_symbol_present_*)
export_symbol_present_conftest $(echo $1 | cut -f5- -d_)
export_symbol_present_conftest "$(echo "$1" | cut -f5- -d_)"
;;
is_export_symbol_gpl_*)
export_symbol_gpl_conftest $(echo $1 | cut -f5- -d_)
export_symbol_gpl_conftest "$(echo "$1" | cut -f5- -d_)"
;;
pcie_is_cxl)
@@ -2226,6 +2275,73 @@ compile_test() {
compile_check_conftest "$CODE" "NV_PCIE_IS_CXL_PRESENT" "" "functions"
;;
cxl_init_supported)
#
# Probe CXL bring-up helpers (e.g. devm_cxl_dev_state_create(),
# cxl_pci_setup_regs(), and related calls below).
#
CODE="
#include <linux/pci.h>
#include <cxl/cxl.h>
#include <cxl/pci.h>
struct nv_conftest_cxl_dev_state {
struct cxl_dev_state cxlds;
};
struct nv_conftest_cxl_dev_state *
conftest_devm_cxl_dev_state_create(void)
{
return devm_cxl_dev_state_create(
NULL, CXL_DEVTYPE_DEVMEM, 0, 0,
struct nv_conftest_cxl_dev_state, cxlds, false);
}
typeof(cxl_pci_setup_regs) conftest_cxl_pci_setup_regs;
int conftest_cxl_pci_setup_regs(struct pci_dev *pdev,
enum cxl_regloc_type type,
struct cxl_register_map *map)
{
return 0;
}
typeof(cxl_set_capacity) conftest_cxl_set_capacity;
int conftest_cxl_set_capacity(struct cxl_dev_state *cxlds,
u64 capacity)
{
return 0;
}
typeof(devm_cxl_add_memdev) conftest_devm_cxl_add_memdev;
struct cxl_memdev *conftest_devm_cxl_add_memdev(
struct cxl_dev_state *cxlds,
const struct cxl_memdev_attach *attach)
{
return NULL;
}
typeof(cxl_get_committed_decoder) conftest_cxl_get_committed_decoder;
struct cxl_endpoint_decoder *conftest_cxl_get_committed_decoder(
struct cxl_memdev *cxlmd, struct cxl_region **cxlr)
{
return NULL;
}
typeof(cxl_get_region_range) conftest_cxl_get_region_range;
int conftest_cxl_get_region_range(struct cxl_region *region,
struct range *range)
{
return 0;
}
typeof(cxl_unregister_region) conftest_cxl_unregister_region;
void conftest_cxl_unregister_region(struct cxl_region *cxlr)
{
}"
compile_check_conftest "$CODE" "NV_CXL_INIT_SUPPORTED" "" "types"
;;
get_backlight_device_by_name)
#
# Determine if the get_backlight_device_by_name() function is present
@@ -2367,6 +2483,57 @@ compile_test() {
fi
;;
dma_buf_supports_dynamic_importer)
#
# Determine if a move_notify() callback is present in struct
# dma_buf_attach_ops. The presence of this callback allows importers
# to safely revoke mappings when the underlying resource is moved.
#
# It will be superseded by the invalidate_mappings() callback in
# linux-next and likely to appear in v7.1.
#
# Added by commit: bb42df4662a4
# ("dma-buf: add dynamic DMA-buf handling v15") in v5.10 (2018-07-03)
#
# conftest #1: Check if dma_buf_attach_ops has the move_notify()
# callback
echo "$CONFTEST_PREAMBLE
#include <linux/dma-buf.h>
int conftest_dma_buf_move_notify(void) {
return offsetof(struct dma_buf_attach_ops, move_notify);
}" > conftest$$.c
$CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
rm -f conftest$$.c
if [ -f conftest$$.o ]; then
echo "#define NV_DMA_BUF_ATTACH_OPS_HAS_MOVE_NOTIFY" | append_conftest "types"
rm -f conftest$$.o
return
else
echo "#undef NV_DMA_BUF_ATTACH_OPS_HAS_MOVE_NOTIFY" | append_conftest "types"
fi
# conftest #2: Check if dma_buf_attach_ops has the
# invalidate_mappings() callback
echo "$CONFTEST_PREAMBLE
#include <linux/dma-buf.h>
int conftest_dma_buf_invalidate_mappings(void) {
return offsetof(struct dma_buf_attach_ops, invalidate_mappings);
}" > conftest$$.c
$CC $CFLAGS -c conftest$$.c > /dev/null 2>&1
rm -f conftest$$.c
if [ -f conftest$$.o ]; then
echo "#define NV_DMA_BUF_ATTACH_OPS_HAS_INVALIDATE_MAPPINGS" | append_conftest "types"
rm -f conftest$$.o
else
echo "#undef NV_DMA_BUF_ATTACH_OPS_HAS_INVALIDATE_MAPPINGS" | append_conftest "types"
fi
;;
drm_fb_create_takes_format_info)
#
# Determine if a `struct drm_format_info *` is passed into
@@ -3798,6 +3965,22 @@ compile_test() {
compile_check_conftest "$CODE" "NV_THERMAL_UNBIND_CDEV_FROM_TRIP_PRESENT" "" "functions"
;;
pm_qos_read_value_supported)
#
# Determine if 'dev_pm_qos' structure has member 'freq'
#
# Added by commit 36a8015f89e40
# ("PM / QoS: Restore DEV_PM_QOS_MIN/MAX_FREQUENCY") in v5.5-rc1
#
CODE="
#include <linux/pm_qos.h>
int conftest_pm_qos_read_value_supported(void) {
return offsetof(struct dev_pm_qos, freq);
}
"
compile_check_conftest "$CODE" "NV_PM_QOS_READ_VALUE_SUPPORTED" "" "types"
;;
update_devfreq)
#
# Determine if update_devfreq() function is present
@@ -4218,6 +4401,22 @@ compile_test() {
compile_check_conftest "$CODE" "NV_MEMORY_FAILURE_MF_SW_SIMULATED_DEFINED" "" "types"
;;
dma_fence_chain_alloc)
#
# Determine if the dma_fence_chain_alloc function is present.
#
# Added by commit 440d0f1 ("dma-buf: add dma_fence_chain_alloc/free v3")
# in 5.15.
#
CODE="
#include <linux/dma-fence-chain.h>
void conftest_dma_fence_chain_alloc(void) {
dma_fence_chain_alloc();
}"
compile_check_conftest "$CODE" "NV_DMA_FENCE_CHAIN_ALLOC_PRESENT" "" "functions"
;;
fence_ops_use_64bit_seqno)
#
# Determine if dma_fence_ops has the use_64bit_seqno member
@@ -4577,6 +4776,24 @@ compile_test() {
compile_check_conftest "$CODE" "NV_MMU_INTERVAL_NOTIFIER" "" "types"
;;
drm_connector_attach_broadcast_rgb_property)
#
# Determine if the function
# drm_connector_attach_broadcast_rgb_property() is present.
#
# Added by commit ab52af4ba7c7d ("drm/connector: Introduce
# drm_connector_attach_broadcast_rgb_property()") in v6.8.
#
CODE="
#include <drm/drm_connector.h>
void conftest_drm_connector_attach_broadcast_rgb_property(void) {
drm_connector_attach_broadcast_rgb_property();
}"
compile_check_conftest "$CODE" "NV_DRM_CONNECTOR_ATTACH_BROADCAST_RGB_PROPERTY_PRESENT" "" "functions"
;;
drm_mode_create_dp_colorspace_property_has_supported_colorspaces_arg)
# Determine if drm_mode_create_dp_colorspace_property() takes the
# 'supported_colorspaces' argument.
@@ -5033,8 +5250,9 @@ compile_test() {
CODE="
#include <drm/drm_atomic_helper.h>
static int conftest_drm_connector_mode_valid(struct drm_connector *connector,
const struct drm_display_mode *mode) {
static enum drm_mode_status
conftest_drm_connector_mode_valid(struct drm_connector *connector,
const struct drm_display_mode *mode) {
return 0;
}
@@ -5085,6 +5303,25 @@ compile_test() {
compile_check_conftest "$CODE" "NV_SHRINKER_ALLOC_PRESENT" "" "functions"
;;
nr_kernel_misc_reclaimable)
#
# Determine if NR_KERNEL_MISC_RECLAIMABLE is present in
# enum node_stat_item.
#
# Added by commit a4a33ef5657a ("mm: add NR_KERNEL_MISC_RECLAIMABLE
# node stat for misc reclaimable kernel memory") in v5.9.
#
CODE="
#include <linux/mmzone.h>
void conftest_nr_kernel_misc_reclaimable(void) {
enum node_stat_item x = NR_KERNEL_MISC_RECLAIMABLE;
(void)x;
}"
compile_check_conftest "$CODE" "NV_NR_KERNEL_MISC_RECLAIMABLE_PRESENT" "" "types"
;;
memory_device_coherent_present)
#
# Determine if MEMORY_DEVICE_COHERENT support is present or not
@@ -5118,26 +5355,6 @@ compile_test() {
compile_check_conftest "$CODE" "NV_DRM_CRTC_FUNCS_HAS_GET_VBLANK_TIMESTAMP" "" "types"
;;
is_vma_write_locked_has_mm_lock_seq_arg)
#
# Determine if __is_vma_write_locked() takes only a single
# 'struct vm_area_struct *' argument.
#
# Commit 22f7639f2f03 ("mm/vma: improve and document
# __is_vma_write_locked()") removed the 'unsigned int *mm_lock_seq'
# parameter in v7.0-rc1.
#
CODE="
#include <linux/mm.h>
#include <linux/mmap_lock.h>
int conftest_is_vma_write_locked_has_mm_lock_seq_arg(struct vm_area_struct *vma) {
unsigned int mm_lock_seq;
return __is_vma_write_locked(vma, &mm_lock_seq);
}"
compile_check_conftest "$CODE" "NV_IS_VMA_WRITE_LOCKED_HAS_MM_LOCK_SEQ_ARG" "" "types"
;;
pfn_is_map_memory)
#
# Determine if pfn_is_map_memory() function is present.
@@ -5174,7 +5391,7 @@ compile_test() {
#
# Check to see if the misc cgroup type is available
#
# dmem_cgrp_id was added by commit id a72232e ("cgroup: Add misc cgroup
# misc_cgrp_id was added by commit id a72232e ("cgroup: Add misc cgroup
# controller") in 5.13
#
CODE="
@@ -5185,6 +5402,26 @@ compile_test() {
compile_check_conftest "$CODE" "NV_MISC_CGROUP_PRESENT" "" "types"
;;
is_vma_write_locked_has_mm_lock_seq_arg)
#
# Determine if __is_vma_write_locked() takes only a single
# 'struct vm_area_struct *' argument.
#
# Commit ("mm/vma: improve and document
# __is_vma_write_locked()") removed the 'unsigned int *mm_lock_seq'
# parameter in linux-next, expected in v7.0-rc1.
#
CODE="
#include <linux/mm.h>
#include <linux/mmap_lock.h>
int conftest_is_vma_write_locked_has_mm_lock_seq_arg(struct vm_area_struct *vma) {
unsigned int mm_lock_seq;
return __is_vma_write_locked(vma, &mm_lock_seq);
}"
compile_check_conftest "$CODE" "NV_IS_VMA_WRITE_LOCKED_HAS_MM_LOCK_SEQ_ARG" "" "types"
;;
drm_color_lut32_present)
# Determine if struct drm_color_lut32 is present.
#
@@ -5235,6 +5472,21 @@ compile_test() {
compile_check_conftest "$CODE" "NV_USE_DMA_IOMMU_PRESENT" "" "functions"
;;
percpu_mm_counter)
#
# Determine if the mm_counter variable is percpu or an atomic
#
# Added by commit f1a7941 ("mm: convert mm's rss stats into percpu_counter") in 6.2.
#
CODE="
#include <linux/mm.h>
void conftest_percpu_mm_counter(struct mm_struct *mm) {
percpu_counter_add(&mm->rss_stat[0], 0);
}"
compile_check_conftest "$CODE" "NV_PERCPU_MM_COUNTER" "" "types"
;;
drm_atomic_commit_struct_present)
#
# Determine if 'struct drm_atomic_state' has been renamed to
@@ -5251,6 +5503,32 @@ compile_test() {
compile_check_conftest "$CODE" "NV_DRM_ATOMIC_COMMIT_STRUCT_PRESENT" "" "types"
;;
sg_alloc_table_from_pages_segment)
#
# Determine if the sg_alloc_table_from_pages_segment() function is
# present. This variant accepts an explicit max_segment cap and
# internally relies on sg_chain() to handle scatterlist tables
# whose total size exceeds 4 GB, letting the RM DMA path use a
# single submap instead of splitting at NV_DMA_SUBMAP_MAX_PAGES
# (~4 GB) boundaries.
#
# Added by commit 90e7a6de62781c27d6a111fccfb19b807f9b6887
# ("lib/scatterlist: Provide a dedicated function to support table
# append") in v5.15-rc1.
#
# See Bug 6419127 for the split-submap GMMU contiguity failure.
# This function and correct max_segment value avoid the overflow
# issue in the kernel.
#
CODE="
#include <linux/scatterlist.h>
void conftest_sg_alloc_table_from_pages_segment(void) {
sg_alloc_table_from_pages_segment();
}"
compile_check_conftest "$CODE" "NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT" "" "functions"
;;
# When adding a new conftest entry, please use the correct format for
# specifying the relevant upstream Linux kernel commit. Please
# avoid specifying -rc kernels, and only use SHAs that actually exist
@@ -5344,7 +5622,7 @@ case "$5" in
kernel_compile_h=$OUTPUT/include/generated/compile.h
if [ ! -f ${kernel_compile_h} ]; then
if [ ! -f "${kernel_compile_h}" ]; then
# The kernel's compile.h file is not present, so there
# isn't a convenient way to identify the compiler version
# used to build the kernel.
@@ -5355,15 +5633,15 @@ case "$5" in
exit 0
fi
kernel_cc_string=`cat ${kernel_compile_h} | \
grep LINUX_COMPILER | cut -f 2 -d '"'`
kernel_cc_string=$(cat "${kernel_compile_h}" | \
grep LINUX_COMPILER | cut -f 2 -d '"')
kernel_cc_version=`echo ${kernel_cc_string} | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+' | head -n 1`
kernel_cc_version=$(echo "${kernel_cc_string}" | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+' | head -n 1)
if [ -z "${kernel_cc_version}" ]; then
kernel_cc_version=`echo ${kernel_cc_string} | grep -o '[0-9]\+\.[0-9]\+' | head -n 1`
kernel_cc_version=$(echo "${kernel_cc_string}" | grep -o '[0-9]\+\.[0-9]\+' | head -n 1)
fi
kernel_cc_major=`echo ${kernel_cc_version} | cut -d '.' -f 1`
kernel_cc_minor=`echo ${kernel_cc_version} | cut -d '.' -f 2`
kernel_cc_major=$(echo "${kernel_cc_version}" | cut -d '.' -f 1)
kernel_cc_minor=$(echo "${kernel_cc_version}" | cut -d '.' -f 2)
echo "
#if (__GNUC__ != ${kernel_cc_major}) || (__GNUC_MINOR__ != ${kernel_cc_minor})
@@ -5408,7 +5686,7 @@ case "$5" in
echo "";
elif [ "$VERBOSE" = "just_msg" ]; then
echo "Warning: The kernel was built with ${kernel_cc_string}, but the" \
"current compiler version is `$CC --version | head -n 1`.";
"current compiler version is $($CC --version | head -n 1).";
fi
exit 0;
fi
@@ -5421,7 +5699,7 @@ case "$5" in
#
VERBOSE=$6
if [ -n "$IGNORE_XEN_PRESENCE" -o -n "$VGX_BUILD" ]; then
if [ -n "$IGNORE_XEN_PRESENCE" ] || [ -n "$VGX_BUILD" ]; then
exit 0
fi
@@ -5491,9 +5769,9 @@ case "$5" in
PATCHES=""
for PATCH in patch-*.h; do
if [ -f $PATCH ]; then
if [ -f "$PATCH" ]; then
echo "#include \"$PATCH\""
PATCHES="$PATCHES "`echo $PATCH | sed -s 's/patch-\(.*\)\.h/\1/'`
PATCHES="$PATCHES "$(echo "$PATCH" | sed -s 's/patch-\(.*\)\.h/\1/')
fi
done
@@ -5519,7 +5797,7 @@ case "$5" in
CFLAGS=$1
shift
for i in $*; do compile_test $i; done
for i in "$@"; do compile_test "$i"; done
exit 0
;;
@@ -5532,7 +5810,7 @@ case "$5" in
if [ -n "$VGX_BUILD" ]; then
if [ -f /proc/xen/capabilities ]; then
if [ "`cat /proc/xen/capabilities`" = "control_d" ]; then
if [ "$(cat /proc/xen/capabilities)" = "control_d" ]; then
exit 0
fi
else
@@ -5583,7 +5861,7 @@ case "$5" in
VFIO_PCI_CORE_PRESENT=1
fi
if ([ "$VFIO_IOMMU_PRESENT" != "0" ] || [ "$VFIO_IOMMUFD_VFIO_CONTAINER_PRESENT" != "0" ])&& [ "$KVM_PRESENT" != "0" ] ; then
if { [ "$VFIO_IOMMU_PRESENT" != "0" ] || [ "$VFIO_IOMMUFD_VFIO_CONTAINER_PRESENT" != "0" ]; } && [ "$KVM_PRESENT" != "0" ] ; then
# vGPU requires either MDEV or vfio-pci-core framework to be present.
if [ "$VFIO_MDEV_PRESENT" != "0" ] || [ "$VFIO_PCI_CORE_PRESENT" != "0" ]; then
exit 0
@@ -5622,7 +5900,7 @@ case "$5" in
#
OPTION=$6
test_configuration_option $OPTION
test_configuration_option "$OPTION"
exit $?
;;
@@ -5632,7 +5910,7 @@ case "$5" in
#
OPTION=$6
get_configuration_option $OPTION
get_configuration_option "$OPTION"
exit $?
;;
@@ -5643,10 +5921,8 @@ case "$5" in
# to the extent that is possible.
#
HASH=$(get_configuration_option CONFIG_MODULE_SIG_HASH)
if [ $? -eq 0 ] && [ -n "$HASH" ]; then
echo $HASH
if HASH=$(get_configuration_option CONFIG_MODULE_SIG_HASH) && [ -n "$HASH" ]; then
echo "$HASH"
exit 0
else
for SHA in 512 384 256 224 1; do
@@ -5679,7 +5955,7 @@ case "$5" in
#
build_cflags
echo $CFLAGS
echo "$CFLAGS"
exit 0
;;
kernel-open/header-presence-tests.mk
+2
View File
@@ -40,6 +40,8 @@ NV_HEADER_PRESENCE_TESTS = \
linux/iosys-map.h \
linux/vfio_pci_core.h \
linux/nvgrace-egm.h \
cxl/cxl.h \
cxl/pci.h \
linux/cc_platform.h \
linux/slub_def.h \
linux/of_gpio.h \
kernel-open/nvidia-drm/nv_drm_common_ioctl.h
+41 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -57,6 +57,8 @@
#define DRM_NVIDIA_UNREGISTER_ROI 0x1a /* Unregister ROI */
#define DRM_NVIDIA_GET_CRTC_ROI_CRCS 0x1b /* Read CRCs for registered ROIs */
#define DRM_NVIDIA_GET_ROI_CAPABILITIES 0x1c /* Get ROI related capabilities */
#define DRM_NVIDIA_SEMSURF_EXPORT_TO_SYNCOBJ_POINT 0x1d
#define DRM_NVIDIA_SYNCOBJ_GET_SYNCFD 0x1e
/* Maximum possible telltale CRCs per plane (used for array size)
@@ -177,6 +179,16 @@
DRM_NVIDIA_GET_DRM_FILE_UNIQUE_ID), \
struct drm_nvidia_get_drm_file_unique_id_params)
#define DRM_IOCTL_NVIDIA_SEMSURF_EXPORT_TO_SYNCOBJ_POINT \
DRM_IOW((DRM_COMMAND_BASE + \
DRM_NVIDIA_SEMSURF_EXPORT_TO_SYNCOBJ_POINT), \
struct drm_nvidia_semsurf_export_to_syncobj_point_params)
#define DRM_IOCTL_NVIDIA_SYNCOBJ_GET_SYNCFD \
DRM_IOWR((DRM_COMMAND_BASE + \
DRM_NVIDIA_SYNCOBJ_GET_SYNCFD), \
struct drm_nvidia_syncobj_get_syncfd_params)
#define DRM_IOCTL_NVIDIA_REGISTER_ROI \
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_REGISTER_ROI), \
struct drm_nvidia_register_roi_params)
@@ -193,6 +205,9 @@
DRM_IOWR((DRM_COMMAND_BASE + DRM_NVIDIA_GET_ROI_CAPABILITIES), \
struct drm_nvidia_get_roi_capabilities_params)
#define NV_DRM_HDMI_VSIF_METADATA_MIN_PAYLOAD_SIZE 3
#define NV_DRM_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE 27
struct drm_nvidia_gem_import_nvkms_memory_params {
uint64_t mem_size; /* IN */
@@ -426,6 +441,19 @@ struct drm_nvidia_get_drm_file_unique_id_params {
uint64_t id; /* OUT Unique ID of the DRM file */
};
struct drm_nvidia_semsurf_export_to_syncobj_point_params {
uint32_t fence_context_handle; /* IN GEM handle to fence context */
uint32_t syncobj_handle; /* IN DRM syncobj handle */
uint64_t wait_value; /* IN Semsurf timeline point to export */
uint64_t syncobj_point; /* IN DRM syncobj timeline point to export to */
};
struct drm_nvidia_syncobj_get_syncfd_params {
uint32_t syncobj_handle; /* IN DRM syncobj timeline handle */
int32_t fd; /* OUT sync FD for the syncobj timeline point */
uint64_t syncobj_point; /* IN DRM syncobj timeline point */
};
/**
* @brief Parameters for getting ROI capabilities
*
@@ -583,4 +611,16 @@ struct drm_nvidia_read_crc_params {
NV_DRM_TRANSFER_FUNCTION_MAX = NV_DRM_TRANSFER_FUNCTION_SRGB,
};
/**
* @brief Struct for HDMI VSIF blob property
*
* This structure defines the content of the NV_HDMI_VSIF_METADATA blob property.
* The payload is a 27 byte array where the first three bytes are the vendor OUI,
* and the remaining bytes are the infoframe content.
*/
struct drm_nvidia_hdmi_vsif_metadata {
uint8_t payload[NV_DRM_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE];
};
#endif /* _NV_DRM_COMMON_IOCTL_H_ */
kernel-open/nvidia-drm/nvidia-drm-conftest.h
+13
View File
@@ -62,6 +62,8 @@
#define NV_DRM_AVAILABLE
#endif
#if IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) || defined(__FreeBSD__)
#if defined(NV_DRM_FBDEV_GENERIC_SETUP_PRESENT) && \
defined(NV_DRM_APERTURE_REMOVE_CONFLICTING_PCI_FRAMEBUFFERS_PRESENT)
#define NV_DRM_FBDEV_AVAILABLE
@@ -87,6 +89,7 @@
#endif
#endif
#endif /* IS_ENABLED(CONFIG_DRM_FBDEV_EMULATION) || defined(__FreeBSD__) */
/*
* DRM color pipeline support (drm_colorop) was added by commit cfc27680ee20
@@ -96,6 +99,16 @@
#define NV_DRM_HAS_COLOROP
#endif
#if defined(NV_DRM_CONNECTOR_ATTACH_CONTENT_PROTECTION_PROPERTY_PRESENT) && \
defined(NV_DRM_HDCP_UPDATE_CONTENT_PROTECTION_PRESENT)
#if (IS_ENABLED(CONFIG_DRM_DISPLAY_HDCP_HELPER))
#include <drm/display/drm_hdcp_helper.h>
// Support content protection prop only from K5.19 onwards
#define NV_DRM_SUPPORT_CONTENT_PROTECTION_PROPERTY
#endif
#endif
/*
* Adapt to quirks in FreeBSD's Linux kernel compatibility layer.
*/
kernel-open/nvidia-drm/nvidia-drm-connector.c
+166 -4
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -30,6 +30,7 @@
#include "nvidia-drm-crtc.h"
#include "nvidia-drm-utils.h"
#include "nvidia-drm-encoder.h"
#include "nv_drm_common_ioctl.h"
/*
* Commit fcd70cd36b9b ("drm: Split out drm_probe_helper.h")
@@ -44,6 +45,7 @@
#include <drm/drm_atomic.h>
#include <drm/drm_atomic_helper.h>
#include <drm/drm_edid.h>
#include <drm/drm_sysfs.h>
/*
* Dithering support requires connector atomic_check, which we only enable on
@@ -298,6 +300,16 @@ static struct drm_connector_state* nv_drm_connector_atomic_duplicate_state(struc
__drm_atomic_helper_connector_duplicate_state(connector, &nv_drm_new_connector_state->base);
nv_drm_new_connector_state->dithering_mode = nv_drm_old_connector_state->dithering_mode;
nv_drm_new_connector_state->hdcp_topology_blob = nv_drm_old_connector_state->hdcp_topology_blob;
if (nv_drm_new_connector_state->hdcp_topology_blob) {
drm_property_blob_get(nv_drm_new_connector_state->hdcp_topology_blob);
}
nv_drm_new_connector_state->hdmi_vsif_metadata =
nv_drm_old_connector_state->hdmi_vsif_metadata;
if (nv_drm_new_connector_state->hdmi_vsif_metadata) {
drm_property_blob_get(nv_drm_new_connector_state->hdmi_vsif_metadata);
}
return &nv_drm_new_connector_state->base;
}
@@ -310,6 +322,9 @@ static void nv_drm_connector_atomic_destroy_state(
to_nv_drm_connector_state(state);
__drm_atomic_helper_connector_destroy_state(state);
drm_property_blob_put(nv_drm_connector_state->hdcp_topology_blob);
drm_property_blob_put(nv_drm_connector_state->hdmi_vsif_metadata);
nv_drm_free(nv_drm_connector_state);
}
@@ -341,6 +356,19 @@ static int nv_drm_connector_atomic_set_property(
}
nv_connector_state->dithering_mode = val;
return 0;
} else if (property == nv_dev->nv_hdcp_level_property) {
// nv_hdcp_level_property is read only
return -EINVAL;
}
if (property == nv_dev->nv_connector_hdmi_vsif_metadata_property) {
return nv_drm_atomic_replace_property_blob_from_id_size_range(
nv_dev->dev,
&nv_connector_state->hdmi_vsif_metadata,
val,
NV_DRM_HDMI_VSIF_METADATA_MIN_PAYLOAD_SIZE,
NV_DRM_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE);
}
/* Unknown property - DRM core handles standard connector properties */
@@ -354,12 +382,22 @@ static int nv_drm_connector_atomic_get_property(
uint64_t *val)
{
struct nv_drm_device *nv_dev = to_nv_device(connector->dev);
struct nv_drm_connector *nv_conn = to_nv_connector(connector);
const struct nv_drm_connector_state *nv_connector_state =
to_nv_drm_connector_state_const(state);
if (property == nv_dev->nv_connector_dithering_mode_property) {
*val = nv_connector_state->dithering_mode;
return 0;
} else if (property == nv_dev->nv_hdcp_level_property) {
*val = nv_conn->cp;
return 0;
}
if (property == nv_dev->nv_connector_hdmi_vsif_metadata_property) {
*val = nv_connector_state->hdmi_vsif_metadata ?
nv_connector_state->hdmi_vsif_metadata->base.id : 0;
return 0;
}
/* Unknown property - DRM core handles standard connector properties */
@@ -451,11 +489,11 @@ static int nv_drm_connector_get_modes(struct drm_connector *connector)
return count;
}
static int nv_drm_connector_mode_valid(struct drm_connector *connector,
static enum drm_mode_status nv_drm_connector_mode_valid(struct drm_connector *connector,
#if defined(NV_DRM_CONNECTOR_HELPER_FUNCS_MODE_VALID_HAS_CONST_MODE_ARG)
const struct drm_display_mode *mode)
const struct drm_display_mode *mode)
#else
struct drm_display_mode *mode)
struct drm_display_mode *mode)
#endif
{
struct drm_device *dev = connector->dev;
@@ -541,6 +579,30 @@ __nv_drm_connector_atomic_check(struct drm_connector *connector,
req_config->modeSetConfig.dithering.mode = mode;
}
/* Handle HDMI VSIF metadata changes */
req_config->flags.hdmiVsifMetadataChanged =
!nv_drm_blobs_equal(nv_old_connector_state->hdmi_vsif_metadata,
nv_new_connector_state->hdmi_vsif_metadata);
if (req_config->flags.hdmiVsifMetadataChanged) {
if (nv_new_connector_state->hdmi_vsif_metadata &&
nv_new_connector_state->hdmi_vsif_metadata->data) {
struct drm_nvidia_hdmi_vsif_metadata* vsif_metadata =
(struct drm_nvidia_hdmi_vsif_metadata*)
nv_new_connector_state->hdmi_vsif_metadata->data;
NvU32 payload_size =
nv_new_connector_state->hdmi_vsif_metadata->length;
WARN_ON(payload_size < NV_DRM_HDMI_VSIF_METADATA_MIN_PAYLOAD_SIZE ||
payload_size > NV_DRM_HDMI_VSIF_METADATA_MAX_PAYLOAD_SIZE);
memcpy(req_config->modeSetConfig.hdmiVsifMetadata.payload,
vsif_metadata->payload,
payload_size);
req_config->modeSetConfig.hdmiVsifMetadata.payloadSize =
payload_size;
} else {
req_config->modeSetConfig.hdmiVsifMetadata.payloadSize = 0;
}
}
/*
* Override metadata for the entire head instead of allowing NVKMS to derive
* it from the layers' metadata.
@@ -668,6 +730,7 @@ nv_drm_connector_new(struct drm_device *dev,
nv_connector->internal = internal;
nv_connector->modeset_permission_filep = NULL;
nv_connector->modeset_permission_crtc = NULL;
nv_connector->cp = NVKMS_CONTENT_PROTECTION_OFF;
strcpy(nv_connector->dpAddress, dpAddress);
@@ -693,6 +756,29 @@ nv_drm_connector_new(struct drm_device *dev,
DRM_CONNECTOR_POLL_CONNECT | DRM_CONNECTOR_POLL_DISCONNECT;
}
#ifdef NV_DRM_SUPPORT_CONTENT_PROTECTION_PROPERTY
/* attach content protection properties */
if ((nv_connector->type == NVKMS_CONNECTOR_TYPE_DP) ||
(nv_connector->type == NVKMS_CONNECTOR_TYPE_HDMI)) {
ret = drm_connector_attach_content_protection_property(&nv_connector->base, true);
if (ret != 0) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to attach content protection properties to connector created from physical index %u",
nv_connector->physicalIndex);
goto failed_connector_init;
}
/* attach nvidia defined connector properties */
drm_object_attach_property(&nv_connector->base.base,
nv_dev->nv_hdcp_topology_property,
0);
drm_object_attach_property(&nv_connector->base.base,
nv_dev->nv_hdcp_level_property,
0);
}
#endif
#if defined(NV_DRM_CONNECTOR_ATTACH_HDR_OUTPUT_METADATA_PROPERTY_PRESENT)
if (nv_connector->type == NVKMS_CONNECTOR_TYPE_HDMI) {
#if defined(NV_DRM_MODE_CREATE_DP_COLORSPACE_PROPERTY_HAS_SUPPORTED_COLORSPACES_ARG)
@@ -705,6 +791,12 @@ nv_drm_connector_new(struct drm_device *dev,
drm_connector_attach_colorspace_property(&nv_connector->base);
}
drm_connector_attach_hdr_output_metadata_property(&nv_connector->base);
if (nv_dev->nv_connector_hdmi_vsif_metadata_property) {
drm_object_attach_property(&nv_connector->base.base,
nv_dev->nv_connector_hdmi_vsif_metadata_property,
0);
}
} else if (nv_connector->type == NVKMS_CONNECTOR_TYPE_DP) {
#if defined(NV_DRM_MODE_CREATE_DP_COLORSPACE_PROPERTY_HAS_SUPPORTED_COLORSPACES_ARG)
if (drm_mode_create_dp_colorspace_property(
@@ -816,4 +908,74 @@ bool nv_drm_connector_revoke_permissions(struct drm_device *dev,
return ret;
}
void nv_drm_connector_update_content_protection(struct nv_drm_connector *nv_connector)
{
#ifdef NV_DRM_SUPPORT_CONTENT_PROTECTION_PROPERTY
struct drm_connector *connector = &nv_connector->base;
struct drm_connector_state *state = connector->state;
unsigned int content_protection = state->content_protection;
unsigned int hdcp_content_type = state->hdcp_content_type;
bool update_cp = false;
unsigned int cp_val;
if ((content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) &&
(hdcp_content_type == DRM_MODE_HDCP_CONTENT_TYPE0)) {
if ((nv_connector->cp == NVKMS_CONTENT_PROTECTION_HDCP1X_ON) ||
(nv_connector->cp == NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE0_ON) ||
(nv_connector->cp == NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE1_ON)) {
cp_val = DRM_MODE_CONTENT_PROTECTION_ENABLED;
update_cp = true;
}
} else if ((content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) &&
(hdcp_content_type == DRM_MODE_HDCP_CONTENT_TYPE1)) {
if (nv_connector->cp == NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE1_ON) {
cp_val = DRM_MODE_CONTENT_PROTECTION_ENABLED;
update_cp = true;
}
} else if (content_protection == DRM_MODE_CONTENT_PROTECTION_ENABLED) {
if ((nv_connector->cp == NVKMS_CONTENT_PROTECTION_OFF) ||
(nv_connector->cp == NVKMS_CONTENT_PROTECTION_FAILED)) {
cp_val = DRM_MODE_CONTENT_PROTECTION_DESIRED;
update_cp = true;
}
}
if (update_cp) {
drm_hdcp_update_content_protection(connector, cp_val);
}
#endif
}
int nv_drm_connector_update_topology_property(struct nv_drm_connector *nv_connector,
const void *topology)
{
#ifdef NV_DRM_SUPPORT_CONTENT_PROTECTION_PROPERTY
struct drm_connector *connector = &nv_connector->base;
struct drm_connector_state *state = connector->state;
struct nv_drm_connector_state *nv_state = to_nv_drm_connector_state(state);
struct drm_device *dev = connector->dev;
struct nv_drm_device *nv_dev = to_nv_device(dev);
int ret;
ret = drm_property_replace_global_blob(dev,
&nv_state->hdcp_topology_blob,
NVKMS_HDCP_TOPOLOGY_SIZE,
topology,
&connector->base,
nv_dev->nv_hdcp_topology_property);
// Generate uevent on cp property when topology is updated
#if defined(NV_DRM_SYSFS_CONNECTOR_PROPERTY_EVENT_PRESENT)
drm_sysfs_connector_property_event(connector,
dev->mode_config.content_protection_property);
#elif defined(NV_DRM_SYSFS_CONNECTOR_STATUS_EVENT_PRESENT)
drm_sysfs_connector_status_event(connector,
dev->mode_config.content_protection_property);
#endif
return ret;
#else
return 0;
#endif
}
#endif
kernel-open/nvidia-drm/nvidia-drm-connector.h
+9 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2016-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -47,6 +47,8 @@ struct nv_drm_connector {
struct nv_drm_encoder *nv_detected_encoder;
struct edid *edid;
enum NvKmsContentProtection cp;
atomic_t connection_status_dirty;
/**
@@ -78,6 +80,8 @@ static inline struct nv_drm_connector *to_nv_connector(
struct nv_drm_connector_state {
struct drm_connector_state base;
enum nv_drm_dithering_mode dithering_mode;
struct drm_property_blob *hdcp_topology_blob;
struct drm_property_blob *hdmi_vsif_metadata;
};
static inline struct nv_drm_connector_state *to_nv_drm_connector_state(
@@ -115,6 +119,10 @@ nv_drm_get_connector(struct drm_device *dev,
bool nv_drm_connector_revoke_permissions(struct drm_device *dev,
struct nv_drm_connector *nv_connector);
void nv_drm_connector_update_content_protection(struct nv_drm_connector *nv_connector);
int nv_drm_connector_update_topology_property(struct nv_drm_connector *nv_connector,
const void *topology);
#endif /* NV_DRM_AVAILABLE */
kernel-open/nvidia-drm/nvidia-drm-crtc.c
+237 -212
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -227,6 +227,7 @@ plane_req_config_disable(struct NvKmsKapiLayerRequestedConfig *req_config)
req_config->flags.matrixOverridesChanged = NV_TRUE;
req_config->flags.ilutChanged = NV_TRUE;
req_config->flags.tmoChanged = NV_TRUE;
req_config->flags.precompColorPassthroughChanged = NV_TRUE;
}
static inline void
@@ -484,7 +485,52 @@ static int init_drm_nvkms_surface(struct nv_drm_device *nv_dev,
return 0;
}
static struct nv_drm_lut_surface *alloc_drm_lut_surface(
static struct nv_drm_lut_surface *pool_acquire_drm_lut_surface(
struct nv_drm_device *nv_dev,
NvU32 num_vss_header_entries,
NvU32 num_entries)
{
struct nv_drm_lut_surface *surface, *n;
mutex_lock(&nv_dev->drm_lut_surface_pool_mutex);
list_for_each_entry_safe(surface, n, &nv_dev->drm_lut_surface_pool,
pool_entry) {
if (surface->properties.vssSegmentEntries == num_vss_header_entries &&
surface->properties.lutEntries == num_entries) {
list_del_init(&surface->pool_entry);
kref_init(&surface->base.refcount);
mutex_unlock(&nv_dev->drm_lut_surface_pool_mutex);
return surface;
}
}
mutex_unlock(&nv_dev->drm_lut_surface_pool_mutex);
return NULL;
}
static void pool_release_drm_lut_surface(
struct nv_drm_lut_surface *surface)
{
struct nv_drm_device *nv_dev;
if (surface == NULL) {
return;
}
nv_dev = surface->nv_dev;
// The pool should only hold "freed" lut surfaces
WARN_ON_ONCE(kref_read(&surface->base.refcount) != 0);
mutex_lock(&nv_dev->drm_lut_surface_pool_mutex);
list_add_tail(&surface->pool_entry,
&nv_dev->drm_lut_surface_pool);
mutex_unlock(&nv_dev->drm_lut_surface_pool_mutex);
}
static struct nv_drm_lut_surface *acquire_drm_lut_surface(
struct nv_drm_device *nv_dev,
enum NvKmsLUTFormat entry_format,
enum NvKmsLUTVssType vss_type,
@@ -504,16 +550,25 @@ static struct nv_drm_lut_surface *alloc_drm_lut_surface(
params.height = 1;
params.surface_size = surface_size;
drm_lut_surface = nv_drm_calloc(1, sizeof(struct nv_drm_lut_surface));
drm_lut_surface = pool_acquire_drm_lut_surface(nv_dev,
num_vss_header_entries,
num_entries);
if (drm_lut_surface == NULL) {
return NULL;
}
drm_lut_surface = nv_drm_calloc(1, sizeof(struct nv_drm_lut_surface));
if (drm_lut_surface == NULL) {
return NULL;
}
if (init_drm_nvkms_surface(nv_dev, &drm_lut_surface->base, &params) != 0) {
nv_drm_free(drm_lut_surface);
return NULL;
if (init_drm_nvkms_surface(nv_dev, &drm_lut_surface->base, &params) != 0) {
nv_drm_free(drm_lut_surface);
return NULL;
}
drm_lut_surface->nv_dev = nv_dev;
INIT_LIST_HEAD(&drm_lut_surface->pool_entry);
}
drm_lut_surface->properties.vssSegmentEntries = num_vss_header_entries;
drm_lut_surface->properties.vssSegments = num_vss_header_segments;
drm_lut_surface->properties.vssType = vss_type;
drm_lut_surface->properties.lutEntries = num_entries;
@@ -522,17 +577,30 @@ static struct nv_drm_lut_surface *alloc_drm_lut_surface(
return drm_lut_surface;
}
static void free_drm_lut_surface(struct kref *ref)
static void release_drm_lut_surface(struct kref *ref)
{
struct nv_drm_nvkms_surface *drm_nvkms_surface =
container_of(ref, struct nv_drm_nvkms_surface, refcount);
struct nv_drm_lut_surface *drm_lut_surface =
container_of(drm_nvkms_surface, struct nv_drm_lut_surface, base);
// Clean up base
release_drm_nvkms_surface(drm_nvkms_surface);
pool_release_drm_lut_surface(drm_lut_surface);
}
nv_drm_free(drm_lut_surface);
void nv_free_drm_lut_surface_pool(struct nv_drm_device *nv_dev)
{
struct nv_drm_lut_surface *surface, *n;
mutex_lock(&nv_dev->drm_lut_surface_pool_mutex);
list_for_each_entry_safe(surface, n,
&nv_dev->drm_lut_surface_pool,
pool_entry) {
list_del(&surface->pool_entry);
release_drm_nvkms_surface(&surface->base);
nv_drm_free(surface);
}
mutex_unlock(&nv_dev->drm_lut_surface_pool_mutex);
}
static NvU32 fp32_lut_interp(
@@ -560,7 +628,8 @@ static NvU32 fp32_lut_interp(
static struct nv_drm_lut_surface *create_drm_ilut_surface_vss(
struct nv_drm_device *nv_dev,
struct nv_drm_plane *nv_plane,
struct nv_drm_plane_state *nv_drm_plane_state)
struct nv_drm_plane_state *nv_drm_plane_state,
const struct drm_crtc_state *crtc_state)
{
static const NvU32 fp_norm = 0x42FA0000; // FP32 125.0
static const NvU32 u10_norm = 0x447FC000; // FP32 1023.0
@@ -632,12 +701,12 @@ static struct nv_drm_lut_surface *create_drm_ilut_surface_vss(
* Space for the VSS header must be included even for non-VSS LUTs.
*/
drm_lut_surface =
alloc_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_FP16,
vss_type,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
num_entries);
acquire_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_FP16,
vss_type,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
num_entries);
if (!drm_lut_surface) {
return ERR_PTR(-ENOMEM);
}
@@ -750,6 +819,22 @@ static struct nv_drm_lut_surface *create_drm_ilut_surface_vss(
NvU32 fp32_entry = nvKmsKapiF16ToF32(fp16_entry);
fp32_r = fp32_g = fp32_b = nvKmsKapiF32Div(fp32_entry, fp_norm);
} else if (!multiply && (crtc_state->degamma_lut != NULL)) {
/* Use legacy DEGAMMA_LUT property if color pipeline not in use. */
fp32_r = nvKmsKapiUI32ToF32(
READ_LUT_AS_UNORM32(crtc_state->degamma_lut,
NVKMS_LUT_ARRAY_SIZE, entry_idx, red));
fp32_g = nvKmsKapiUI32ToF32(
READ_LUT_AS_UNORM32(crtc_state->degamma_lut,
NVKMS_LUT_ARRAY_SIZE, entry_idx, green));
fp32_b = nvKmsKapiUI32ToF32(
READ_LUT_AS_UNORM32(crtc_state->degamma_lut,
NVKMS_LUT_ARRAY_SIZE, entry_idx, blue));
/* Convert UNORM32 to 1.0-normalized FP32. */
fp32_r = nvKmsKapiF32Div(fp32_r, u32_norm);
fp32_g = nvKmsKapiF32Div(fp32_g, u32_norm);
fp32_b = nvKmsKapiF32Div(fp32_b, u32_norm);
} else {
/* Use implicit identity. */
// TODO: Use LUT table?
@@ -817,12 +902,12 @@ static struct nv_drm_lut_surface *create_drm_tmo_surface(
* The TMO LUT always uses VSS.
*/
drm_lut_surface =
alloc_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_UNORM16,
NVKMS_LUT_VSS_TYPE_LINEAR,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
NVKMS_LUT_ARRAY_SIZE + 1);
acquire_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_UNORM16,
NVKMS_LUT_VSS_TYPE_LINEAR,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
NVKMS_LUT_ARRAY_SIZE + 1);
if (drm_lut_surface == NULL) {
return ERR_PTR(-ENOMEM);
}
@@ -898,6 +983,13 @@ static struct nv_drm_lut_surface *create_drm_olut_surface_vss(
const NvU16 *vss_entries = NULL;
enum NvKmsLUTVssType vss_type = NVKMS_LUT_VSS_TYPE_NONE;
/*
* The regamma_divisor will be handled later via a separate NvKms parameter,
* but we don't want to use the legacy GAMMA_LUT if we're supplying a
* non-identity regamma_divisor.
*/
NvBool will_divide = (nv_drm_crtc_state->regamma_divisor > NV_DRM_S31_32_ONE);
WARN_ON(!nv_crtc->olut_caps.supported);
WARN_ON(nv_crtc->olut_caps.entryFormat != NVKMS_LUT_FORMAT_UNORM16);
WARN_ON(nv_crtc->olut_caps.vssType != NVKMS_LUT_VSS_TYPE_LOGARITHMIC);
@@ -927,12 +1019,12 @@ static struct nv_drm_lut_surface *create_drm_olut_surface_vss(
* Space for the VSS header must be included even for non-VSS LUTs.
*/
drm_lut_surface =
alloc_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_UNORM16,
vss_type,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
num_entries);
acquire_drm_lut_surface(nv_dev,
NVKMS_LUT_FORMAT_UNORM16,
vss_type,
num_vss_header_segments,
NUM_VSS_HEADER_ENTRIES,
num_entries);
if (!drm_lut_surface) {
return NULL;
}
@@ -1007,6 +1099,14 @@ static struct nv_drm_lut_surface *create_drm_olut_surface_vss(
} else if (vss_entries != NULL) {
/* Use VSS LUT directly. */
r = g = b = vss_entries[entry_idx];
} else if (!will_divide && (nv_drm_crtc_state->base.gamma_lut != NULL)) {
/* Use legacy GAMMA_LUT property if color pipeline not in use. */
const struct drm_color_lut *gamma_lut =
(struct drm_color_lut *) nv_drm_crtc_state->base.gamma_lut->data;
r = gamma_lut[entry_idx].red;
g = gamma_lut[entry_idx].green;
b = gamma_lut[entry_idx].blue;
} else {
/* Use implicit identity. */
WARN_ON_ONCE(num_entries != (NVKMS_LUT_ARRAY_SIZE + 1));
@@ -1120,6 +1220,7 @@ static enum NvKmsInputColorRange nv_drm_color_range_to_nvkms_color_range(
static int
plane_req_config_update(struct drm_plane *plane,
struct drm_plane_state *plane_state,
const struct drm_crtc_state *crtc_state,
struct NvKmsKapiLayerRequestedConfig *req_config)
{
struct nv_drm_device *nv_dev = to_nv_device(plane->dev);
@@ -1384,6 +1485,22 @@ plane_req_config_update(struct drm_plane *plane,
req_config->flags.outputTfChanged = (old_config.outputTf != req_config->config.outputTf);
#endif
if (crtc_state->color_mgmt_changed || (plane->state->crtc != plane_state->crtc)) {
/*
* According to the comment in the Linux kernel's
* drivers/gpu/drm/drm_color_mgmt.c, if this property is NULL,
* the CTM needs to be changed to the identity matrix
*/
if (crtc_state->ctm) {
ctm_to_csc(&req_config->config.csc,
(struct drm_color_ctm *)crtc_state->ctm->data);
} else {
req_config->config.csc = NVKMS_IDENTITY_CSC_MATRIX;
}
req_config->config.cscUseMain = NV_FALSE;
req_config->flags.cscChanged = NV_TRUE;
}
req_config->config.matrixOverrides.enabled.fmtCtm =
update_matrix_override(nv_drm_plane_state->fmt_ctm,
&req_config->config.matrixOverrides.fmtCtm,
@@ -1421,20 +1538,22 @@ plane_req_config_update(struct drm_plane *plane,
&matrix_overrides_changed);
req_config->flags.matrixOverridesChanged = matrix_overrides_changed;
if (nv_drm_plane_state->degamma_changed) {
if (nv_drm_plane_state->degamma_changed || crtc_state->color_mgmt_changed) {
if (nv_drm_plane_state->degamma_drm_lut_surface != NULL) {
kref_put(&nv_drm_plane_state->degamma_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
nv_drm_plane_state->degamma_drm_lut_surface = NULL;
}
if ((nv_drm_plane_state->degamma_tf != NV_DRM_TRANSFER_FUNCTION_DEFAULT) ||
(nv_drm_plane_state->degamma_lut != NULL) ||
(nv_drm_plane_state->degamma_multiplier != NV_DRM_S31_32_ONE)) {
(nv_drm_plane_state->degamma_multiplier != NV_DRM_S31_32_ONE) ||
(crtc_state->degamma_lut != NULL)) {
nv_drm_plane_state->degamma_drm_lut_surface =
create_drm_ilut_surface_vss(nv_dev, nv_plane,
nv_drm_plane_state);
nv_drm_plane_state,
crtc_state);
if (IS_ERR(nv_drm_plane_state->degamma_drm_lut_surface)) {
int ret = PTR_ERR(nv_drm_plane_state->degamma_drm_lut_surface);
nv_drm_plane_state->degamma_drm_lut_surface = NULL;
@@ -1475,7 +1594,7 @@ plane_req_config_update(struct drm_plane *plane,
if (nv_drm_plane_state->tmo_changed) {
if (nv_drm_plane_state->tmo_drm_lut_surface != NULL) {
kref_put(&nv_drm_plane_state->tmo_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
nv_drm_plane_state->tmo_drm_lut_surface = NULL;
}
@@ -1520,6 +1639,12 @@ plane_req_config_update(struct drm_plane *plane,
req_config->flags.tmoChanged = NV_TRUE;
}
if (nv_drm_plane_state->precomp_color_passthrough_changed) {
req_config->config.precompColorPassthrough =
nv_drm_plane_state->precomp_color_passthrough;
req_config->flags.precompColorPassthroughChanged = NV_TRUE;
}
/*
* Unconditionally mark the surface as changed, even if nothing changed,
* so that we always get a flip event: a DRM client may flip with
@@ -1651,27 +1776,12 @@ static int nv_drm_plane_atomic_check(struct drm_plane *plane,
#endif
ret = plane_req_config_update(plane,
plane_state,
crtc_state,
plane_requested_config);
if (ret != 0) {
return ret;
}
if (crtc_state->color_mgmt_changed || (plane->state->crtc != plane_state->crtc)) {
/*
* According to the comment in the Linux kernel's
* drivers/gpu/drm/drm_color_mgmt.c, if this property is NULL,
* the CTM needs to be changed to the identity matrix
*/
if (crtc_state->ctm) {
ctm_to_csc(&plane_requested_config->config.csc,
(struct drm_color_ctm *)crtc_state->ctm->data);
} else {
plane_requested_config->config.csc = NVKMS_IDENTITY_CSC_MATRIX;
}
plane_requested_config->config.cscUseMain = NV_FALSE;
plane_requested_config->flags.cscChanged = NV_TRUE;
}
if (__is_async_flip_requested(plane, crtc_state)) {
/*
* Async flip requests that the flip happen 'as soon as
@@ -1725,6 +1835,9 @@ static int nv_drm_atomic_crtc_get_property(
*/
*val = NVKMS_LUT_ARRAY_SIZE;
return 0;
} else if (property == nv_dev->nv_crtc_color_passthrough_property) {
*val = nv_drm_crtc_state->postcomp_color_passthrough;
return 0;
}
return -EINVAL;
@@ -1765,6 +1878,12 @@ static int nv_drm_atomic_crtc_set_property(
nv_drm_crtc_state->regamma_changed = true;
}
return 0;
} else if (property == nv_dev->nv_crtc_color_passthrough_property) {
if (!!val != nv_drm_crtc_state->postcomp_color_passthrough) {
nv_drm_crtc_state->postcomp_color_passthrough = !!val;
nv_drm_crtc_state->postcomp_color_passthrough_changed = true;
}
return 0;
}
return -EINVAL;
@@ -1879,6 +1998,12 @@ static int nv_drm_plane_set_vendor_color_property(
nv_drm_plane_state->tmo_changed = true;
}
return ret;
} else if (property == nv_dev->nv_plane_color_passthrough_property) {
if (!!val != nv_drm_plane_state->precomp_color_passthrough) {
nv_drm_plane_state->precomp_color_passthrough = !!val;
nv_drm_plane_state->precomp_color_passthrough_changed = true;
}
return 0;
}
/* Not a vendor color property */
@@ -2009,6 +2134,9 @@ static int nv_drm_plane_get_vendor_color_property(
*/
*val = NVKMS_LUT_ARRAY_SIZE;
return 0;
} else if (property == nv_dev->nv_plane_color_passthrough_property) {
*val = nv_drm_plane_state->precomp_color_passthrough;
return 0;
}
/* Not a vendor color property */
@@ -2050,6 +2178,21 @@ static int nv_drm_plane_atomic_get_property(
return -EINVAL;
}
static inline struct nv_drm_plane_state *nv_drm_plane_state_alloc(void)
{
struct nv_drm_plane_state *nv_plane_state =
nv_drm_calloc(1, sizeof(*nv_plane_state));
if (!nv_plane_state) {
return NULL;
}
/* Default to 1 in S31.32 Sign-Magnitude Format */
nv_plane_state->degamma_multiplier = NV_DRM_S31_32_ONE;
return nv_plane_state;
}
/**
* nv_drm_plane_atomic_reset - plane state reset hook
* @plane: DRM plane
@@ -2059,7 +2202,7 @@ static int nv_drm_plane_atomic_get_property(
static void nv_drm_plane_atomic_reset(struct drm_plane *plane)
{
struct nv_drm_plane_state *nv_plane_state =
nv_drm_calloc(1, sizeof(*nv_plane_state));
nv_drm_plane_state_alloc();
if (!nv_plane_state) {
return;
@@ -2153,6 +2296,10 @@ nv_drm_plane_atomic_duplicate_state(struct drm_plane *plane)
nv_plane_state->ctms_default_to_identity =
nv_old_plane_state->ctms_default_to_identity;
nv_plane_state->precomp_color_passthrough =
nv_old_plane_state->precomp_color_passthrough;
nv_plane_state->precomp_color_passthrough_changed = false;
return &nv_plane_state->base;
}
@@ -2174,13 +2321,13 @@ static inline void __nv_drm_plane_atomic_destroy_state(
drm_property_blob_put(nv_drm_plane_state->degamma_lut);
if (nv_drm_plane_state->degamma_drm_lut_surface != NULL) {
kref_put(&nv_drm_plane_state->degamma_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
}
drm_property_blob_put(nv_drm_plane_state->tmo_lut);
if (nv_drm_plane_state->tmo_drm_lut_surface != NULL) {
kref_put(&nv_drm_plane_state->tmo_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
}
}
@@ -2238,32 +2385,6 @@ static inline bool nv_drm_crtc_duplicate_req_head_modeset_config(
old->layerRequestedConfig[i].config;
}
if (old->modeSetConfig.lut.input.pRamps) {
new->modeSetConfig.lut.input.pRamps =
nv_drm_calloc(1, sizeof(*new->modeSetConfig.lut.input.pRamps));
if (!new->modeSetConfig.lut.input.pRamps) {
return false;
}
*new->modeSetConfig.lut.input.pRamps =
*old->modeSetConfig.lut.input.pRamps;
}
if (old->modeSetConfig.lut.output.pRamps) {
new->modeSetConfig.lut.output.pRamps =
nv_drm_calloc(1, sizeof(*new->modeSetConfig.lut.output.pRamps));
if (!new->modeSetConfig.lut.output.pRamps) {
/*
* new->modeSetConfig.lut.input.pRamps is either NULL or it was
* just allocated
*/
nv_drm_free(new->modeSetConfig.lut.input.pRamps);
new->modeSetConfig.lut.input.pRamps = NULL;
return false;
}
*new->modeSetConfig.lut.output.pRamps =
*old->modeSetConfig.lut.output.pRamps;
}
return true;
}
@@ -2280,6 +2401,10 @@ static inline struct nv_drm_crtc_state *nv_drm_crtc_state_alloc(void)
for (i = 0; i < ARRAY_SIZE(nv_state->req_config.layerRequestedConfig); i++) {
plane_config_clear(&nv_state->req_config.layerRequestedConfig[i].config);
}
/* Default to 1 */
nv_state->regamma_divisor = NV_DRM_S31_32_ONE;
return nv_state;
}
@@ -2369,6 +2494,10 @@ nv_drm_atomic_crtc_duplicate_state(struct drm_crtc *crtc)
}
nv_state->regamma_changed = false;
nv_state->postcomp_color_passthrough =
nv_old_state->postcomp_color_passthrough;
nv_state->postcomp_color_passthrough_changed = false;
return &nv_state->base;
}
@@ -2395,12 +2524,9 @@ static void nv_drm_atomic_crtc_destroy_state(struct drm_crtc *crtc,
drm_property_blob_put(nv_state->regamma_lut);
if (nv_state->regamma_drm_lut_surface != NULL) {
kref_put(&nv_state->regamma_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
}
nv_drm_free(nv_state->req_config.modeSetConfig.lut.input.pRamps);
nv_drm_free(nv_state->req_config.modeSetConfig.lut.output.pRamps);
nv_drm_free(nv_state);
}
@@ -2491,118 +2617,6 @@ static int head_modeset_config_attach_connector(
return 0;
}
static int color_mgmt_config_copy_lut(struct NvKmsLutRamps *nvkms_lut,
struct drm_color_lut *drm_lut,
uint64_t lut_len)
{
uint64_t i = 0;
if (lut_len != NVKMS_LUT_ARRAY_SIZE) {
return -EINVAL;
}
/*
* Both NvKms and drm LUT values are 16-bit linear values. NvKms LUT ramps
* are in arrays in a single struct while drm LUT ramps are an array of
* structs.
*/
for (i = 0; i < lut_len; i++) {
nvkms_lut->red[i] = drm_lut[i].red;
nvkms_lut->green[i] = drm_lut[i].green;
nvkms_lut->blue[i] = drm_lut[i].blue;
}
return 0;
}
static int color_mgmt_config_set_luts(struct nv_drm_crtc_state *nv_crtc_state,
struct NvKmsKapiHeadRequestedConfig *req_config)
{
struct NvKmsKapiHeadModeSetConfig *modeset_config =
&req_config->modeSetConfig;
struct drm_crtc_state *crtc_state = &nv_crtc_state->base;
int ret = 0;
/*
* According to the comment in the Linux kernel's
* drivers/gpu/drm/drm_color_mgmt.c, if either property is NULL, that LUT
* needs to be changed to a linear LUT
*
* On failure, any LUT ramps allocated in this function are freed when the
* subsequent atomic state cleanup calls nv_drm_atomic_crtc_destroy_state.
*/
if (crtc_state->degamma_lut) {
struct drm_color_lut *degamma_lut = NULL;
uint64_t degamma_len = 0;
if (!modeset_config->lut.input.pRamps) {
modeset_config->lut.input.pRamps =
nv_drm_calloc(1, sizeof(*modeset_config->lut.input.pRamps));
if (!modeset_config->lut.input.pRamps) {
return -ENOMEM;
}
}
degamma_lut = (struct drm_color_lut *)crtc_state->degamma_lut->data;
degamma_len = crtc_state->degamma_lut->length /
sizeof(struct drm_color_lut);
if ((ret = color_mgmt_config_copy_lut(modeset_config->lut.input.pRamps,
degamma_lut,
degamma_len)) != 0) {
return ret;
}
modeset_config->lut.input.depth = 30; /* specify the full LUT */
modeset_config->lut.input.start = 0;
modeset_config->lut.input.end = degamma_len - 1;
} else {
/* setting input.end to 0 is equivalent to disabling the LUT, which
* should be equivalent to a linear LUT */
modeset_config->lut.input.depth = 30; /* specify the full LUT */
modeset_config->lut.input.start = 0;
modeset_config->lut.input.end = 0;
nv_drm_free(modeset_config->lut.input.pRamps);
modeset_config->lut.input.pRamps = NULL;
}
req_config->flags.legacyIlutChanged = NV_TRUE;
if (crtc_state->gamma_lut) {
struct drm_color_lut *gamma_lut = NULL;
uint64_t gamma_len = 0;
if (!modeset_config->lut.output.pRamps) {
modeset_config->lut.output.pRamps =
nv_drm_calloc(1, sizeof(*modeset_config->lut.output.pRamps));
if (!modeset_config->lut.output.pRamps) {
return -ENOMEM;
}
}
gamma_lut = (struct drm_color_lut *)crtc_state->gamma_lut->data;
gamma_len = crtc_state->gamma_lut->length /
sizeof(struct drm_color_lut);
if ((ret = color_mgmt_config_copy_lut(modeset_config->lut.output.pRamps,
gamma_lut,
gamma_len)) != 0) {
return ret;
}
modeset_config->lut.output.enabled = NV_TRUE;
} else {
/* disabling the output LUT should be equivalent to setting a linear
* LUT */
modeset_config->lut.output.enabled = NV_FALSE;
nv_drm_free(modeset_config->lut.output.pRamps);
modeset_config->lut.output.pRamps = NULL;
}
req_config->flags.legacyOlutChanged = NV_TRUE;
return 0;
}
/**
* nv_drm_crtc_atomic_check() can fail after it has modified
* the 'nv_drm_crtc_state::req_config', that is fine because 'nv_drm_crtc_state'
@@ -2668,21 +2682,17 @@ static int nv_drm_crtc_atomic_check(struct drm_crtc *crtc,
req_config->modeSetConfig.vrrEnabled = crtc_state->vrr_enabled;
#endif
if (crtc_state->color_mgmt_changed) {
if ((ret = color_mgmt_config_set_luts(nv_crtc_state, req_config)) != 0) {
return ret;
}
}
if (nv_crtc_state->regamma_changed) {
if (nv_crtc_state->regamma_changed || crtc_state->color_mgmt_changed) {
if (nv_crtc_state->regamma_drm_lut_surface != NULL) {
kref_put(&nv_crtc_state->regamma_drm_lut_surface->base.refcount,
free_drm_lut_surface);
release_drm_lut_surface);
nv_crtc_state->regamma_drm_lut_surface = NULL;
}
if ((nv_crtc_state->regamma_tf != NV_DRM_TRANSFER_FUNCTION_DEFAULT) ||
(nv_crtc_state->regamma_lut != NULL)) {
(nv_crtc_state->regamma_lut != NULL) ||
((crtc_state->gamma_lut != NULL) &&
(nv_crtc_state->regamma_divisor <= NV_DRM_S31_32_ONE))) {
nv_crtc_state->regamma_drm_lut_surface =
create_drm_olut_surface_vss(nv_dev, nv_crtc,
@@ -2732,6 +2742,12 @@ static int nv_drm_crtc_atomic_check(struct drm_crtc *crtc,
}
}
if (nv_crtc_state->postcomp_color_passthrough_changed) {
req_config->modeSetConfig.postcompColorPassthrough =
nv_crtc_state->postcomp_color_passthrough;
req_config->flags.postcompColorPassthroughChanged = NV_TRUE;
}
return ret;
}
@@ -2762,8 +2778,6 @@ static void nv_drm_crtc_install_properties(
NV_DRM_TRANSFER_FUNCTION_DEFAULT);
}
if (nv_dev->nv_crtc_regamma_divisor_property) {
/* Default to 1 */
nv_crtc_state->regamma_divisor = NV_DRM_S31_32_ONE;
drm_object_attach_property(
&crtc->base, nv_dev->nv_crtc_regamma_divisor_property,
nv_crtc_state->regamma_divisor);
@@ -2778,6 +2792,13 @@ static void nv_drm_crtc_install_properties(
NVKMS_LUT_ARRAY_SIZE);
}
}
if (nv_dev->supportsColorPassthrough) {
if (nv_dev->nv_crtc_color_passthrough_property) {
drm_object_attach_property(
&crtc->base, nv_dev->nv_crtc_color_passthrough_property, 0);
}
}
}
static void nv_drm_plane_install_properties(
@@ -2854,8 +2875,6 @@ static void nv_drm_plane_install_properties(
NV_DRM_TRANSFER_FUNCTION_DEFAULT);
}
if (nv_dev->nv_plane_degamma_multiplier_property) {
/* Default to 1 in S31.32 Sign-Magnitude Format */
nv_plane_state->degamma_multiplier = NV_DRM_S31_32_ONE;
drm_object_attach_property(
&plane->base, nv_dev->nv_plane_degamma_multiplier_property,
nv_plane_state->degamma_multiplier);
@@ -2870,6 +2889,13 @@ static void nv_drm_plane_install_properties(
NVKMS_LUT_ARRAY_SIZE);
}
}
if (nv_dev->supportsColorPassthrough) {
if (nv_dev->nv_plane_color_passthrough_property) {
drm_object_attach_property(
&plane->base, nv_dev->nv_plane_color_passthrough_property, 0);
}
}
}
#if defined(NV_DRM_HAS_COLOROP)
@@ -3047,8 +3073,7 @@ nv_drm_plane_create(struct drm_device *dev,
nv_plane->defaultCompositionMode = defaultCompositionMode;
nv_plane->layer_idx = layer_idx;
if ((nv_plane_state =
nv_drm_calloc(1, sizeof(*nv_plane_state))) == NULL) {
if ((nv_plane_state = nv_drm_plane_state_alloc()) == NULL) {
goto failed_state_alloc;
}
kernel-open/nvidia-drm/nvidia-drm-crtc.h
+14 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016-2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2016-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -177,6 +177,9 @@ struct nv_drm_crtc_state {
uint64_t regamma_divisor;
struct nv_drm_lut_surface *regamma_drm_lut_surface;
NvBool regamma_changed;
NvBool postcomp_color_passthrough;
NvBool postcomp_color_passthrough_changed;
};
static inline struct nv_drm_crtc_state *to_nv_crtc_state(struct drm_crtc_state *state)
@@ -246,6 +249,8 @@ struct nv_drm_nvkms_surface_params {
struct nv_drm_lut_surface {
struct nv_drm_nvkms_surface base;
struct {
NvU32 vssSegmentEntries;
NvU32 vssSegments;
enum NvKmsLUTVssType vssType;
@@ -253,8 +258,13 @@ struct nv_drm_lut_surface {
enum NvKmsLUTFormat entryFormat;
} properties;
struct nv_drm_device *nv_dev;
struct list_head pool_entry;
};
void nv_free_drm_lut_surface_pool(struct nv_drm_device *nv_dev);
struct nv_drm_plane_state {
struct drm_plane_state base;
s32 __user *fd_user_ptr;
@@ -278,6 +288,9 @@ struct nv_drm_plane_state {
struct drm_property_blob *tmo_lut;
struct nv_drm_lut_surface *tmo_drm_lut_surface;
NvBool tmo_changed;
NvBool precomp_color_passthrough;
NvBool precomp_color_passthrough_changed;
};
static inline struct nv_drm_plane_state *to_nv_drm_plane_state(struct drm_plane_state *state)
kernel-open/nvidia-drm/nvidia-drm-drv.c
+169 -82
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -104,66 +104,6 @@ static int nv_drm_revoke_sub_ownership(struct drm_device *dev);
static DEFINE_MUTEX(dev_list_mutex);
static struct nv_drm_device *dev_list = NULL;
static const char* nv_get_input_colorspace_name(
enum nv_drm_input_color_space colorSpace)
{
switch (colorSpace) {
case NV_DRM_INPUT_COLOR_SPACE_NONE:
return "None";
case NV_DRM_INPUT_COLOR_SPACE_SCRGB_LINEAR:
return "scRGB Linear FP16";
case NV_DRM_INPUT_COLOR_SPACE_BT2100_PQ:
return "BT.2100 PQ";
default:
/* We shoudn't hit this */
WARN_ON("Unsupported input colorspace");
return "None";
}
};
static char* nv_get_transfer_function_name(
enum nv_drm_transfer_function tf)
{
switch (tf) {
case NV_DRM_TRANSFER_FUNCTION_LINEAR:
return "Linear";
case NV_DRM_TRANSFER_FUNCTION_PQ:
return "PQ (Perceptual Quantizer)";
default:
/* We shoudn't hit this */
WARN_ON("Unsupported transfer function");
#if defined(fallthrough)
fallthrough;
#else
/* Fallthrough */
#endif
case NV_DRM_TRANSFER_FUNCTION_DEFAULT:
return "Default";
}
};
static const char* nv_get_dithering_mode_name(
enum nv_drm_dithering_mode mode)
{
switch (mode) {
case NV_DRM_DITHERING_MODE_AUTO:
return "auto";
case NV_DRM_DITHERING_MODE_OFF:
return "off";
case NV_DRM_DITHERING_MODE_STATIC_2X2:
return "static 2x2";
case NV_DRM_DITHERING_MODE_DYNAMIC_2X2:
return "dynamic 2x2";
case NV_DRM_DITHERING_MODE_TEMPORAL:
return "temporal";
case NV_DRM_DITHERING_MODE_ON:
return "on";
default:
/* We shouldn't hit this, but return a safe default */
return "auto";
}
};
#if defined(NV_DRM_OUTPUT_POLL_CHANGED_PRESENT)
static void nv_drm_output_poll_changed(struct drm_device *dev)
{
@@ -232,6 +172,20 @@ static void nv_drm_event_callback(const struct NvKmsKapiEvent *event)
event->u.displayChanged.display);
break;
case NVKMS_EVENT_TYPE_DPY_CONTENT_PROTECTION_CHANGED:
nv_drm_handle_display_cp_change(
nv_dev,
event->u.displayCpChanged.display,
event->u.displayCpChanged.cp);
break;
case NVKMS_EVENT_TYPE_DPY_CP_TOPOLOGY_CHANGED:
nv_drm_handle_display_cp_topology_change(
nv_dev,
event->u.displayCpTopologyChanged.display,
event->u.displayCpTopologyChanged.topology);
break;
case NVKMS_EVENT_TYPE_DYNAMIC_DPY_CONNECTED:
nv_drm_handle_dynamic_display_connected(
nv_dev,
@@ -522,26 +476,51 @@ static void nv_drm_enumerate_encoders_and_connectors
*/
static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
{
struct drm_prop_enum_list colorspace_enum_list[NV_DRM_INPUT_COLOR_SPACE_MAX] = { };
struct drm_prop_enum_list tf_enum_list[NV_DRM_TRANSFER_FUNCTION_MAX] = { };
struct drm_prop_enum_list dithering_mode_enum_list[NV_DRM_DITHERING_MODE_MAX] = { };
int i, len = 0;
const struct drm_prop_enum_list colorspace_enum_list[] = {
{ .type = NV_DRM_INPUT_COLOR_SPACE_NONE,
.name = "None" },
{ .type = NV_DRM_INPUT_COLOR_SPACE_SCRGB_LINEAR,
.name = "scRGB Linear FP16" },
{ .type = NV_DRM_INPUT_COLOR_SPACE_BT2100_PQ,
.name = "BT.2100 PQ" },
};
for (i = 0; i < NV_DRM_INPUT_COLOR_SPACE_MAX; i++) {
colorspace_enum_list[len].type = i;
colorspace_enum_list[len].name = nv_get_input_colorspace_name(i);
len++;
}
const struct drm_prop_enum_list tf_enum_list[] = {
{ .type = NV_DRM_TRANSFER_FUNCTION_LINEAR,
.name = "Linear" },
{ .type = NV_DRM_TRANSFER_FUNCTION_PQ,
.name = "PQ (Perceptual Quantizer)" },
{ .type = NV_DRM_TRANSFER_FUNCTION_DEFAULT,
.name = "Default" },
};
for (i = 0; i < NV_DRM_TRANSFER_FUNCTION_MAX; i++) {
tf_enum_list[i].type = i;
tf_enum_list[i].name = nv_get_transfer_function_name(i);
}
const struct drm_prop_enum_list dithering_mode_enum_list[] = {
{ .type = NV_DRM_DITHERING_MODE_AUTO,
.name = "auto" },
{ .type = NV_DRM_DITHERING_MODE_OFF,
.name = "off" },
{ .type = NV_DRM_DITHERING_MODE_STATIC_2X2,
.name = "static 2x2" },
{ .type = NV_DRM_DITHERING_MODE_DYNAMIC_2X2,
.name = "dynamic 2x2" },
{ .type = NV_DRM_DITHERING_MODE_TEMPORAL,
.name = "temporal" },
{ .type = NV_DRM_DITHERING_MODE_ON,
.name = "on" },
};
for (i = 0; i < NV_DRM_DITHERING_MODE_MAX; i++) {
dithering_mode_enum_list[i].type = i;
dithering_mode_enum_list[i].name = nv_get_dithering_mode_name(i);
}
const struct drm_prop_enum_list hdcp_level_enum_list[] = {
{ .type = NVKMS_CONTENT_PROTECTION_OFF,
.name = "HDCP_OFF" },
{ .type = NVKMS_CONTENT_PROTECTION_HDCP1X_ON,
.name = "HDCP1X_ON" },
{ .type = NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE0_ON,
.name = "HDCP2X_TYPE0_ON" },
{ .type = NVKMS_CONTENT_PROTECTION_HDCP2X_TYPE1_ON,
.name = "HDCP2X_TYPE1_ON" },
{ .type = NVKMS_CONTENT_PROTECTION_FAILED,
.name = "HDCP_FAILED" },
};
if (nv_dev->supportsSyncpts) {
nv_dev->nv_out_fence_property =
@@ -554,12 +533,22 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
nv_dev->nv_input_colorspace_property =
drm_property_create_enum(nv_dev->dev, 0, "NV_INPUT_COLORSPACE",
colorspace_enum_list, len);
colorspace_enum_list,
ARRAY_SIZE(colorspace_enum_list));
if (nv_dev->nv_input_colorspace_property == NULL) {
NV_DRM_LOG_ERR("Failed to create NV_INPUT_COLORSPACE property");
return -ENOMEM;
}
nv_dev->nv_hdcp_level_property =
drm_property_create_enum(nv_dev->dev, 0, "NV_HDCP_LEVEL",
hdcp_level_enum_list,
ARRAY_SIZE(hdcp_level_enum_list));
if (nv_dev->nv_hdcp_level_property == NULL) {
NV_DRM_LOG_ERR("Failed to create NV_HDCP_LEVEL property");
return -ENOMEM;
}
#if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
nv_dev->nv_hdr_output_metadata_property =
drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
@@ -602,7 +591,7 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
nv_dev->nv_plane_degamma_tf_property =
drm_property_create_enum(nv_dev->dev, 0,
"NV_PLANE_DEGAMMA_TF", tf_enum_list,
NV_DRM_TRANSFER_FUNCTION_MAX);
ARRAY_SIZE(tf_enum_list));
if (nv_dev->nv_plane_degamma_tf_property == NULL) {
return -ENOMEM;
}
@@ -646,7 +635,7 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
nv_dev->nv_crtc_regamma_tf_property =
drm_property_create_enum(nv_dev->dev, 0,
"NV_CRTC_REGAMMA_TF", tf_enum_list,
NV_DRM_TRANSFER_FUNCTION_MAX);
ARRAY_SIZE(tf_enum_list));
if (nv_dev->nv_crtc_regamma_tf_property == NULL) {
return -ENOMEM;
}
@@ -677,12 +666,42 @@ static int nv_drm_create_properties(struct nv_drm_device *nv_dev)
drm_property_create_enum(nv_dev->dev, 0,
"dithering mode",
dithering_mode_enum_list,
NV_DRM_DITHERING_MODE_MAX);
ARRAY_SIZE(dithering_mode_enum_list));
if (nv_dev->nv_connector_dithering_mode_property == NULL) {
NV_DRM_LOG_ERR("Failed to create dithering mode property");
return -ENOMEM;
}
// Color Pipeline Passthrough Properties
nv_dev->nv_plane_color_passthrough_property =
drm_property_create_bool(nv_dev->dev, 0,
"NV_PLANE_COLOR_PASSTHROUGH");
if (nv_dev->nv_plane_color_passthrough_property == NULL) {
return -ENOMEM;
}
nv_dev->nv_crtc_color_passthrough_property =
drm_property_create_bool(nv_dev->dev, 0,
"NV_CRTC_COLOR_PASSTHROUGH");
if (nv_dev->nv_crtc_color_passthrough_property == NULL) {
return -ENOMEM;
}
/* hdcp_topology is immutable by user space */
nv_dev->nv_hdcp_topology_property =
drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB | DRM_MODE_PROP_IMMUTABLE,
"NV_HDCP_TOPOLOGY", 0);
if (nv_dev->nv_hdcp_topology_property == NULL) {
return -ENOMEM;
}
// HDMI Vendor Specific InfoFrame Metadata
nv_dev->nv_connector_hdmi_vsif_metadata_property =
drm_property_create(nv_dev->dev, DRM_MODE_PROP_BLOB,
"NV_HDMI_VSIF_METADATA", 0);
if (nv_dev->nv_connector_hdmi_vsif_metadata_property == NULL) {
return -ENOMEM;
}
return 0;
}
@@ -800,6 +819,10 @@ static int nv_drm_dev_load(struct drm_device *dev)
nv_dev->supportsSyncpts = resInfo.caps.supportsSyncpts;
nv_dev->supportsColorPassthrough = resInfo.caps.supportsColorPassthrough;
nv_dev->isSocDgpuDisplayNeedingWar = resInfo.caps.isSocDgpuDisplayNeedingWar;
nv_dev->semsurf_stride = resInfo.caps.semsurf.stride;
nv_dev->semsurf_max_submitted_offset =
@@ -829,6 +852,9 @@ static int nv_drm_dev_load(struct drm_device *dev)
nv_dev->modifiers[i++] = DRM_FORMAT_MOD_LINEAR;
nv_dev->modifiers[i++] = DRM_FORMAT_MOD_INVALID;
mutex_init(&nv_dev->drm_lut_surface_pool_mutex);
INIT_LIST_HEAD(&nv_dev->drm_lut_surface_pool);
/* Initialize drm_device::mode_config */
nv_drm_init_mode_config(nv_dev, &resInfo);
@@ -850,6 +876,8 @@ static int nv_drm_dev_load(struct drm_device *dev)
nv_dev->pDevice,
((1 << NVKMS_EVENT_TYPE_DPY_CHANGED) |
(1 << NVKMS_EVENT_TYPE_DYNAMIC_DPY_CONNECTED) |
(1 << NVKMS_EVENT_TYPE_DPY_CONTENT_PROTECTION_CHANGED) |
(1 << NVKMS_EVENT_TYPE_DPY_CP_TOPOLOGY_CHANGED) |
(1 << NVKMS_EVENT_TYPE_FLIP_OCCURRED)))) {
NV_DRM_DEV_LOG_ERR(nv_dev, "Failed to register event mask");
}
@@ -941,6 +969,8 @@ static void nv_drm_dev_unload(struct drm_device *dev)
NV_DRM_DEV_LOG_ERR(nv_dev, "Failed to stop event listening");
}
nv_free_drm_lut_surface_pool(nv_dev);
/* Unset NvKmsKapiDevice */
pDevice = nv_dev->pDevice;
@@ -1841,6 +1871,12 @@ static const struct drm_ioctl_desc nv_drm_ioctls[] = {
DRM_IOCTL_DEF_DRV(NVIDIA_SEMSURF_FENCE_ATTACH,
nv_drm_semsurf_fence_attach_ioctl,
DRM_RENDER_ALLOW|DRM_UNLOCKED),
DRM_IOCTL_DEF_DRV(NVIDIA_SEMSURF_EXPORT_TO_SYNCOBJ_POINT,
nv_drm_semsurf_export_to_syncobj_point_ioctl,
DRM_RENDER_ALLOW|DRM_UNLOCKED),
DRM_IOCTL_DEF_DRV(NVIDIA_SYNCOBJ_GET_SYNCFD,
nv_drm_syncobj_get_syncfd_ioctl,
DRM_RENDER_ALLOW|DRM_UNLOCKED),
/*
* DRM_UNLOCKED is implicit for all non-legacy DRM driver IOCTLs since Linux
@@ -2261,6 +2297,34 @@ void nv_drm_remove_devices(void)
}
}
/*
* Mark the connection status of all connectors dirty and schedule a
* hotplug event, so that both the fbdev helper and userspace DRM clients
* re-detect connectors and restore modes as appropriate.
*/
static void nv_drm_reprobe_connectors(struct nv_drm_device *nv_dev)
{
struct drm_device *dev = nv_dev->dev;
struct drm_connector *connector = NULL;
struct drm_connector_list_iter conn_iter;
drm_connector_list_iter_begin(dev, &conn_iter);
drm_for_each_connector_iter(connector, &conn_iter) {
nv_drm_connector_mark_connection_status_dirty(
to_nv_connector(connector));
}
drm_connector_list_iter_end(&conn_iter);
/*
* Delay the hotplug event: this is called from the resume path while
* other devices are still resuming, and the system workqueue is not
* frozen. An immediate hotplug event could drive a modeset (fbdev
* helper or DRM client) concurrently with the device-resume phase.
*/
schedule_delayed_work(&nv_dev->hotplug_event_work,
msecs_to_jiffies(2000));
}
/*
* Handle system suspend and resume.
*
@@ -2326,6 +2390,29 @@ void nv_drm_suspend_resume(NvBool suspend)
drm_fb_helper_set_suspend_unlocked(dev->fb_helper, 0);
#endif
drm_kms_helper_poll_enable(dev);
/*
* XXX: Workaround for bug 6422321, tracked for removal by
* bug 6556017.
*
* On resume, the affected devices come up with all heads
* shut down and the modeset owner is never notified, and
* display detection can trust a stale deasserted HPD and
* skip DDC (bug 6490670), so a monitor that dropped HPD
* during suspend stays undetected. If a sink stays
* connected across suspend, resume generates no HPD edge,
* and NVKMS only re-detects DisplayPort connectors on
* resume (nvRmResumeDP()). Any connector probe that raced
* with resume may also have latched a stale "disconnected"
* status. Force a re-probe of all connectors and send a
* hotplug event so the DRM client re-detects and restores
* the mode, like nv_drm_dev_load() does at boot. Restrict
* this to SoC products with dGPU-class display so other
* GPUs keep the stock resume behavior.
*/
if (nv_dev->isSocDgpuDisplayNeedingWar) {
nv_drm_reprobe_connectors(nv_dev);
}
}
}
kernel-open/nvidia-drm/nvidia-drm-encoder.c
+41 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -299,6 +299,46 @@ void nv_drm_handle_display_change(struct nv_drm_device *nv_dev,
schedule_delayed_work(&nv_dev->hotplug_event_work, 0);
}
void nv_drm_handle_display_cp_change(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay,
enum NvKmsContentProtection cp)
{
struct drm_device *dev = nv_dev->dev;
struct nv_drm_encoder *nv_encoder = NULL;
nv_encoder = get_nv_encoder_from_nvkms_display(dev, hDisplay);
if (nv_encoder == NULL) {
return;
}
mutex_lock(&dev->mode_config.mutex);
nv_encoder->nv_connector->cp = cp;
nv_drm_connector_update_content_protection(nv_encoder->nv_connector);
mutex_unlock(&dev->mode_config.mutex);
}
void nv_drm_handle_display_cp_topology_change(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay,
const void *topology)
{
struct drm_device *dev = nv_dev->dev;
struct nv_drm_encoder *nv_encoder = NULL;
mutex_lock(&dev->mode_config.mutex);
nv_encoder = get_nv_encoder_from_nvkms_display(dev, hDisplay);
if (nv_encoder == NULL) {
NV_DRM_DEV_LOG_ERR(nv_dev, "Encoder not found for display %d", hDisplay);
goto out;
}
nv_drm_connector_update_topology_property(nv_encoder->nv_connector, topology);
out:
mutex_unlock(&dev->mode_config.mutex);
}
void nv_drm_handle_dynamic_display_connected(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay)
{
kernel-open/nvidia-drm/nvidia-drm-encoder.h
+9 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2016-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -56,6 +56,14 @@ nv_drm_add_encoder(struct drm_device *dev, NvKmsKapiDisplay hDisplay);
void nv_drm_handle_display_change(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay);
void nv_drm_handle_display_cp_change(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay,
enum NvKmsContentProtection cp);
void nv_drm_handle_display_cp_topology_change(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay,
const void *topology);
void nv_drm_handle_dynamic_display_connected(struct nv_drm_device *nv_dev,
NvKmsKapiDisplay hDisplay);
kernel-open/nvidia-drm/nvidia-drm-fence.c
+176
View File
@@ -36,6 +36,10 @@
#include <linux/dma-fence.h>
#if defined(NV_DRM_SYNCOBJ_FEATURES_PRESENT)
#include <drm/drm_syncobj.h>
#endif
#define NV_DRM_SEMAPHORE_SURFACE_FENCE_MAX_TIMEOUT_MS 5000
struct nv_drm_fence_context;
@@ -1826,4 +1830,176 @@ done:
return ret;
}
int nv_drm_semsurf_export_to_syncobj_point_ioctl(struct drm_device *dev,
void *data,
struct drm_file *filep)
{
#if defined(NV_DRM_SYNCOBJ_FEATURES_PRESENT)
struct nv_drm_device *nv_dev = to_nv_device(dev);
struct drm_nvidia_semsurf_export_to_syncobj_point_params *p = data;
struct nv_drm_fence_context *nv_fence_context = NULL;
struct dma_fence *fence = NULL;
struct drm_syncobj *syncobj = NULL;
struct dma_fence_chain *chain = NULL;
int ret = -EINVAL;
if (nv_dev->pDevice == NULL) {
ret = -EOPNOTSUPP;
goto done;
}
/* Lookup the fence context */
nv_fence_context = __nv_drm_fence_context_lookup(
filep,
p->fence_context_handle);
if (!nv_fence_context) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to look up nv fence context: 0x%08x",
p->fence_context_handle);
goto done;
}
if (nv_fence_context->ops != &nv_drm_semsurf_fence_ctx_ops) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Wrong fence context type: 0x%08x",
p->fence_context_handle);
goto done;
}
/* Lookup the DRM syncobj */
syncobj = drm_syncobj_find(filep, p->syncobj_handle);
if (!syncobj) {
ret = -ENOENT;
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to find syncobj: 0x%08x",
p->syncobj_handle);
goto done;
}
#if defined(NV_DMA_FENCE_CHAIN_ALLOC_PRESENT)
chain = dma_fence_chain_alloc();
#else
chain = kzalloc(sizeof(struct dma_fence_chain), GFP_KERNEL);
#endif
if (!chain) {
ret = -ENOMEM;
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to allocate dmabuf chain");
goto done;
}
/* Create a fence for the semsurf timeline point */
fence = __nv_drm_semsurf_fence_ctx_create_fence(
nv_dev,
to_semsurf_fence_ctx(nv_fence_context),
p->wait_value,
NV_DRM_SEMAPHORE_SURFACE_FENCE_MAX_TIMEOUT_MS);
if (IS_ERR(fence)) {
ret = PTR_ERR(fence);
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to create fence for wait_value: %llu",
p->wait_value);
goto done;
}
/* Replace the syncobj timeline point with our fence */
drm_syncobj_add_point(syncobj, chain, fence, p->syncobj_point);
ret = 0;
done:
if (syncobj) {
drm_syncobj_put(syncobj);
}
if (fence) {
dma_fence_put(fence);
}
if (nv_fence_context) {
nv_drm_gem_object_unreference_unlocked(&nv_fence_context->base);
}
return ret;
#else
return -EOPNOTSUPP;
#endif /* NV_DRM_SYNCOBJ_FEATURES_PRESENT */
}
int nv_drm_syncobj_get_syncfd_ioctl(struct drm_device *dev,
void *data,
struct drm_file *filep)
{
#if defined(NV_DRM_SYNCOBJ_FEATURES_PRESENT)
struct nv_drm_device *nv_dev = to_nv_device(dev);
struct drm_nvidia_syncobj_get_syncfd_params *p = data;
struct drm_syncobj *syncobj = NULL;
struct dma_fence *fence = NULL;
int ret = -EINVAL;
int fd;
if (nv_dev->pDevice == NULL) {
ret = -EOPNOTSUPP;
goto done;
}
/* Lookup the DRM syncobj */
syncobj = drm_syncobj_find(filep, p->syncobj_handle);
if (!syncobj) {
ret = -ENOENT;
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to find syncobj: 0x%08x",
p->syncobj_handle);
goto done;
}
/* Find the fence at the specified timeline point */
ret = drm_syncobj_find_fence(filep, p->syncobj_handle, p->syncobj_point,
0 /* flags */, &fence);
if (ret) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to find fence for syncobj 0x%08x at point %llu",
p->syncobj_handle,
p->syncobj_point);
goto done;
}
/* Create a sync FD for the fence */
fd = nv_drm_create_sync_file(fence);
if (fd < 0) {
ret = fd;
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Failed to create sync file from fence on syncobj 0x%08x",
p->syncobj_handle);
goto fence_found;
}
p->fd = fd;
ret = 0;
fence_found:
/* Release this function's reference to the fence */
dma_fence_put(fence);
done:
if (syncobj) {
drm_syncobj_put(syncobj);
}
return ret;
#else
return -EOPNOTSUPP;
#endif /* NV_DRM_SYNCOBJ_FEATURES_PRESENT */
}
#endif /* NV_DRM_AVAILABLE */
kernel-open/nvidia-drm/nvidia-drm-fence.h
+8
View File
@@ -55,6 +55,14 @@ int nv_drm_semsurf_fence_attach_ioctl(struct drm_device *dev,
void *data,
struct drm_file *filep);
int nv_drm_semsurf_export_to_syncobj_point_ioctl(struct drm_device *dev,
void *data,
struct drm_file *filep);
int nv_drm_syncobj_get_syncfd_ioctl(struct drm_device *dev,
void *data,
struct drm_file *filep);
#endif /* NV_DRM_AVAILABLE */
#endif /* __NVIDIA_DRM_PRIME_FENCE_H__ */
kernel-open/nvidia-drm/nvidia-drm-helper.c
+43 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2016-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -229,4 +229,46 @@ nv_drm_atomic_replace_property_blob_from_id(struct drm_device *dev,
return 0;
}
int
nv_drm_atomic_replace_property_blob_from_id_size_range(struct drm_device *dev,
struct drm_property_blob **blob,
uint64_t blob_id,
ssize_t min_size,
ssize_t max_size)
{
struct drm_property_blob *new_blob = NULL;
if (blob_id != 0) {
new_blob = drm_property_lookup_blob(dev, blob_id);
if (new_blob == NULL) {
return -EINVAL;
}
if (((min_size > 0) && (new_blob->length < min_size)) ||
((max_size > 0) && (new_blob->length > max_size))) {
drm_property_blob_put(new_blob);
return -EINVAL;
}
}
nv_drm_atomic_replace_property_blob(blob, new_blob, NULL);
drm_property_blob_put(new_blob);
return 0;
}
NvBool nv_drm_blobs_equal(const struct drm_property_blob *old_blob,
const struct drm_property_blob *new_blob)
{
if (!old_blob || !new_blob) {
return old_blob == new_blob;
}
if (old_blob->length != new_blob->length) {
return NV_FALSE;
}
return (memcmp(old_blob->data, new_blob->data, old_blob->length) == 0);
}
#endif /* NV_DRM_AVAILABLE */
kernel-open/nvidia-drm/nvidia-drm-helper.h
+27
View File
@@ -445,6 +445,33 @@ int nv_drm_atomic_replace_property_blob_from_id(struct drm_device *dev,
ssize_t expected_size,
NvBool *replaced);
/**
* nv_drm_atomic_replace_property_blob_from_id_size_range - Look up and replace a property blob (size range)
* @dev: DRM device
* @blob: Pointer to blob pointer to replace
* @blob_id: Blob ID to lookup (0 = NULL)
* @min_size: Minimum allowed blob size (inclusive)
* @max_size: Maximum allowed blob size (inclusive)
*
* Returns: 0 on success, -EINVAL if blob not found or length outside [min_size, max_size]
*/
int nv_drm_atomic_replace_property_blob_from_id_size_range(struct drm_device *dev,
struct drm_property_blob **blob,
uint64_t blob_id,
ssize_t min_size,
ssize_t max_size);
/**
* nv_drm_blobs_equal - Compare two blobs
* @old_blob: Old blob
* @new_blob: New blob
*
* Returns: NV_TRUE if equal, NV_FALSE otherwise
*/
NvBool nv_drm_blobs_equal(const struct drm_property_blob *old_blob,
const struct drm_property_blob *new_blob);
/*
* S31.32 sign-magnitude fixed-point constant for 1.0
*/
kernel-open/nvidia-drm/nvidia-drm-linux.c
+3 -3
View File
@@ -32,19 +32,19 @@
MODULE_PARM_DESC(
modeset,
"Enable atomic kernel modesetting (1 = enable (default), 0 = disable)");
module_param_named(modeset, nv_drm_modeset_module_param, bool, 0400);
module_param_named(modeset, nv_drm_modeset_module_param, bool, 0444);
#if defined(NV_DRM_FBDEV_AVAILABLE)
MODULE_PARM_DESC(
fbdev,
"Create a framebuffer device (1 = enable (default), 0 = disable)");
module_param_named(fbdev, nv_drm_fbdev_module_param, bool, 0400);
module_param_named(fbdev, nv_drm_fbdev_module_param, bool, 0444);
#endif
MODULE_PARM_DESC(
vblank,
"Enable drm vblank notification support (1 = enable, 0 = disable (default))");
module_param_named(vblank, nv_drm_vblank_module_param, bool, 0400);
module_param_named(vblank, nv_drm_vblank_module_param, bool, 0444);
#if defined(NV_DRM_HAS_COLOROP)
MODULE_PARM_DESC(
kernel-open/nvidia-drm/nvidia-drm-modeset.c
+96 -78
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -26,6 +26,7 @@
#include "nvidia-drm-priv.h"
#include "nvidia-drm-modeset.h"
#include "nvidia-drm-connector.h"
#include "nvidia-drm-crtc.h"
#include "nvidia-drm-os-interface.h"
#include "nvidia-drm-helper.h"
@@ -500,10 +501,22 @@ nv_drm_atomic_apply_modeset_config(struct drm_device *dev,
requested_config,
&reply_config,
commit)) {
if (commit || reply_config.flipResult != NV_KMS_FLIP_RESULT_IN_PROGRESS) {
return -EINVAL;
return -EINVAL;
}
#ifdef NV_DRM_SUPPORT_CONTENT_PROTECTION_PROPERTY
if (commit) {
struct drm_connector *connector;
struct drm_connector_state *connector_state;
int j;
for_each_new_connector_in_state(state, connector, connector_state, j) {
if (connector->state->content_protection == DRM_MODE_CONTENT_PROTECTION_DESIRED) {
nv_drm_connector_update_content_protection(to_nv_connector(connector));
}
}
}
#endif
if (commit && nv_drm_vblank_module_param) {
for_each_oldnew_crtc_in_state(state, crtc, old_crtc_state, new_crtc_state, i) {
@@ -543,6 +556,7 @@ int nv_drm_atomic_check(struct drm_device *dev,
bool cursor_surface_changed;
bool cursor_only_commit;
bool infoframe_changed;
for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
/*
@@ -564,11 +578,60 @@ int nv_drm_atomic_check(struct drm_device *dev,
}
/*
* if the color management changed on the crtc, we need to update the
* crtc's plane's CSC matrices, so add the crtc's planes to the commit
* Check if infoframe-related properties changed on any connector
* attached to this CRTC. If so, we need to serialize the flip
* to ensure infoframe updates are properly sequenced.
*/
infoframe_changed = false;
#if defined(NV_DRM_CONNECTOR_ATTACH_HDR_OUTPUT_METADATA_PROPERTY_PRESENT)
{
struct drm_connector *connector;
struct drm_connector_state *connector_state;
struct drm_connector_state *old_connector_state;
struct nv_drm_connector_state *nv_connector_state;
struct nv_drm_connector_state *nv_old_connector_state;
int k;
bool hdr_changed, colorspace_changed, vsif_metadata_changed;
for_each_new_connector_in_state(state, connector, connector_state, k) {
if (connector_state->crtc != crtc) {
continue;
}
old_connector_state = drm_atomic_get_old_connector_state(state, connector);
nv_connector_state = to_nv_drm_connector_state(connector_state);
nv_old_connector_state = to_nv_drm_connector_state(old_connector_state);
hdr_changed = !drm_connector_atomic_hdr_metadata_equal(old_connector_state,
connector_state);
colorspace_changed = (old_connector_state->colorspace != connector_state->colorspace);
vsif_metadata_changed = !nv_drm_blobs_equal(nv_connector_state->hdmi_vsif_metadata,
nv_old_connector_state->hdmi_vsif_metadata);
/* Check if HDR metadata or colorspace changed */
if (hdr_changed || colorspace_changed || vsif_metadata_changed) {
NV_DRM_LOG_INFO("Infoframe changed on CRTC %d (hdr=%d colorspace=%d vsif=%d), adding planes on CRTC to atomic commit\n",
to_nv_crtc(crtc)->head,
hdr_changed,
colorspace_changed,
vsif_metadata_changed);
infoframe_changed = true;
break;
}
}
}
#endif
/*
* If the color management changed on the crtc, we need to update the
* crtc's plane's CSC matrices and ILUTs, so add the crtc's planes to
* the commit. Similarly, if infoframe properties changed, add all
* planes to ensure the flip is serialized via flip_list.
*/
if (crtc_state->color_mgmt_changed ||
(cursor_surface_changed && cursor_only_commit)) {
(cursor_surface_changed && cursor_only_commit) ||
infoframe_changed) {
if ((ret = drm_atomic_add_affected_planes(state, crtc)) != 0) {
goto done;
}
@@ -711,71 +774,37 @@ int nv_drm_atomic_commit(struct drm_device *dev,
* Our system already implements such a queue, but due to
* bug 4054608, it is currently not used.
*/
for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
if (nonblock) {
for_each_new_crtc_in_state(state, crtc, crtc_state, i) {
struct nv_drm_crtc *nv_crtc = to_nv_crtc(crtc);
/*
* Here you aren't required to hold nv_drm_crtc::flip_list_lock
* because:
*
* The core DRM driver acquires lock for all affected crtcs before
* calling into ->commit() hook, therefore it is not possible for
* other threads to call into ->commit() hook affecting same crtcs
* and enqueue flip objects into flip_list -
*
* nv_drm_atomic_commit_internal()
* |-> nv_drm_atomic_apply_modeset_config(commit=true)
* |-> nv_drm_crtc_enqueue_flip()
*
* Only possibility is list_empty check races with code path
* dequeuing flip object -
*
* __nv_drm_handle_flip_event()
* |-> nv_drm_crtc_dequeue_flip()
*
* But this race condition can't lead list_empty() to return
* incorrect result. nv_drm_crtc_dequeue_flip() in the middle of
* updating the list could not trick us into thinking the list is
* empty when it isn't.
*/
if (nonblock) {
/*
* Here you aren't required to hold nv_drm_crtc::flip_list_lock
* because:
*
* The core DRM driver acquires lock for all affected crtcs before
* calling into ->commit() hook, therefore it is not possible for
* other threads to call into ->commit() hook affecting same crtcs
* and enqueue flip objects into flip_list -
*
* nv_drm_atomic_commit_internal()
* |-> nv_drm_atomic_apply_modeset_config(commit=true)
* |-> nv_drm_crtc_enqueue_flip()
*
* Only possibility is list_empty check races with code path
* dequeuing flip object -
*
* __nv_drm_handle_flip_event()
* |-> nv_drm_crtc_dequeue_flip()
*
* But this race condition can't lead list_empty() to return
* incorrect result. nv_drm_crtc_dequeue_flip() in the middle of
* updating the list could not trick us into thinking the list is
* empty when it isn't.
*/
if (!list_empty(&nv_crtc->flip_list)) {
return -EBUSY;
}
} else {
if (wait_event_timeout(
nv_dev->flip_event_wq,
list_empty(&nv_crtc->flip_list),
3 * HZ /* 3 second */) == 0) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"Flip event timeout on head %u", nv_crtc->head);
}
}
/*
* If the legacy LUT needs to be updated, ensure that the previous LUT
* update is complete first.
*/
if (crtc_state->color_mgmt_changed) {
NvBool complete = nvKms->checkLutNotifier(nv_dev->pDevice,
nv_crtc->head,
!nonblock /* waitForCompletion */);
/* If checking the LUT notifier failed, assume no LUT notifier is set. */
if (!complete) {
if (nonblock) {
return -EBUSY;
} else {
/*
* checkLutNotifier should wait on the notifier in this
* case, so we should only get here if the wait timed out.
*/
NV_DRM_DEV_LOG_ERR(
nv_dev,
"LUT notifier timeout on head %u", nv_crtc->head);
}
}
}
}
@@ -905,17 +934,6 @@ int nv_drm_atomic_commit(struct drm_device *dev,
__nv_drm_handle_flip_event(nv_crtc);
}
}
if (crtc_state->color_mgmt_changed) {
NvBool complete = nvKms->checkLutNotifier(nv_dev->pDevice,
nv_crtc->head,
true /* waitForCompletion */);
if (!complete) {
NV_DRM_DEV_LOG_ERR(
nv_dev,
"LUT notifier timeout on head %u", nv_crtc->head);
}
}
}
}
kernel-open/nvidia-drm/nvidia-drm-os-interface.c
-2
View File
@@ -71,8 +71,6 @@ void nv_drm_free(void *ptr)
#if defined(NVCPU_X86) || defined(NVCPU_X86_64)
#define WRITE_COMBINE_FLUSH() asm volatile("sfence":::"memory")
#elif defined(NVCPU_PPC64LE)
#define WRITE_COMBINE_FLUSH() asm volatile("sync":::"memory")
#else
#define WRITE_COMBINE_FLUSH() mb()
#endif
kernel-open/nvidia-drm/nvidia-drm-priv.h
+14 -1
View File
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015-2025, NVIDIA CORPORATION. All rights reserved.
* Copyright (c) 2015-2026, NVIDIA CORPORATION. All rights reserved.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
@@ -156,11 +156,14 @@ struct nv_drm_device {
NvBool contiguousPhysicalMappings;
NvBool supportsSyncpts;
NvBool supportsColorPassthrough;
NvBool isSocDgpuDisplayNeedingWar;
NvBool subOwnershipGranted;
NvBool hasFramebufferConsole;
struct drm_property *nv_out_fence_property;
struct drm_property *nv_input_colorspace_property;
struct drm_property *nv_hdcp_level_property;
struct {
NvU32 count;
@@ -170,6 +173,7 @@ struct nv_drm_device {
#if defined(NV_DRM_HAS_HDR_OUTPUT_METADATA)
struct drm_property *nv_hdr_output_metadata_property;
#endif
struct drm_property *nv_hdcp_topology_property;
struct drm_property *nv_plane_lms_ctm_property;
struct drm_property *nv_plane_lms_to_itp_ctm_property;
@@ -184,19 +188,28 @@ struct nv_drm_device {
struct drm_property *nv_plane_tmo_lut_property;
struct drm_property *nv_plane_tmo_lut_size_property;
struct drm_property *nv_plane_color_passthrough_property;
struct drm_property *nv_crtc_regamma_tf_property;
struct drm_property *nv_crtc_regamma_lut_property;
struct drm_property *nv_crtc_regamma_lut_size_property;
struct drm_property *nv_crtc_regamma_divisor_property;
struct drm_property *nv_crtc_color_passthrough_property;
struct drm_property *nv_connector_dithering_mode_property;
struct drm_property *nv_connector_hdmi_vsif_metadata_property;
struct nv_drm_device *next;
NvU64 vtFbBaseAddress;
NvU64 vtFbSize;
nv_drm_workthread vblank_worker;
struct mutex drm_lut_surface_pool_mutex;
struct list_head drm_lut_surface_pool;
};
static inline NvU32 nv_drm_next_display_semaphore(
kernel-open/nvidia-drm/nvidia-drm-sources.mk
+3
View File
@@ -65,8 +65,11 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += vmf_insert_mixed
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_gem_prime_mmap
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_sysfs_connector_property_event
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_sysfs_connector_status_event
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_connector_attach_content_protection_property
NV_CONFTEST_FUNCTION_COMPILE_TESTS += drm_hdcp_update_content_protection
NV_CONFTEST_FUNCTION_COMPILE_TESTS += pfn_is_map_memory
NV_CONFTEST_FUNCTION_COMPILE_TESTS += vm_flags_set
NV_CONFTEST_FUNCTION_COMPILE_TESTS += dma_fence_chain_alloc
NV_CONFTEST_TYPE_COMPILE_TESTS += drm_driver_has_legacy_dev_list
NV_CONFTEST_TYPE_COMPILE_TESTS += vm_ops_fault_removed_vma_arg
kernel-open/nvidia-modeset/nvidia-modeset-linux.c
+78 -32
View File
@@ -65,65 +65,111 @@
#include "nv-retpoline.h"
#endif
/*
* Commit 0911b8c52c4d ("x86/bugs: Rename CONFIG_RETHUNK =>
* CONFIG_MITIGATION_RETHUNK)" in v6.9 renamed CONFIG_RETHUNK.
*/
#if !defined(CONFIG_RETHUNK) && !defined(CONFIG_MITIGATION_RETHUNK)
#include "nv-rethunk.h"
#endif
#include <linux/backlight.h>
#if IS_ENABLED(CONFIG_EXTCON_DISP_CABLE_STATE)
#include <linux/extcon/extcon-disp.h>
#endif
#define NVKMS_LOG_PREFIX "nvidia-modeset: "
static bool output_rounding_fix = true;
module_param_named(output_rounding_fix, output_rounding_fix, bool, 0400);
module_param_named(output_rounding_fix, output_rounding_fix, bool, 0444);
static bool disable_hdmi_frl = false;
module_param_named(disable_hdmi_frl, disable_hdmi_frl, bool, 0400);
module_param_named(disable_hdmi_frl, disable_hdmi_frl, bool, 0444);
static bool disable_vrr_memclk_switch = false;
module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0400);
module_param_named(disable_vrr_memclk_switch, disable_vrr_memclk_switch, bool, 0444);
static bool hdmi_deepcolor = true;
module_param_named(hdmi_deepcolor, hdmi_deepcolor, bool, 0400);
module_param_named(hdmi_deepcolor, hdmi_deepcolor, bool, 0444);
static unsigned int max_output_color_depth = 10;
module_param_named(max_output_color_depth, max_output_color_depth, uint, 0444);
static bool opportunistic_display_sync = true;
module_param_named(opportunistic_display_sync, opportunistic_display_sync, bool, 0400);
module_param_named(opportunistic_display_sync, opportunistic_display_sync, bool, 0444);
static enum NvKmsDebugForceColorSpace debug_force_color_space = NVKMS_DEBUG_FORCE_COLOR_SPACE_NONE;
module_param_named(debug_force_color_space, debug_force_color_space, uint, 0400);
module_param_named(debug_force_color_space, debug_force_color_space, uint, 0444);
static bool enable_overlay_layers = true;
module_param_named(enable_overlay_layers, enable_overlay_layers, bool, 0400);
module_param_named(enable_overlay_layers, enable_overlay_layers, bool, 0444);
/* These parameters are used for fault injection tests. Normally the defaults
* should be used. */
MODULE_PARM_DESC(fail_malloc, "Fail the Nth call to nvkms_alloc");
static int fail_malloc_num = -1;
module_param_named(fail_malloc, fail_malloc_num, int, 0400);
module_param_named(fail_malloc, fail_malloc_num, int, 0444);
MODULE_PARM_DESC(malloc_verbose, "Report information about malloc calls on module unload");
static bool malloc_verbose = false;
module_param_named(malloc_verbose, malloc_verbose, bool, 0400);
module_param_named(malloc_verbose, malloc_verbose, bool, 0444);
MODULE_PARM_DESC(force_frl_rate,
"Override the default FRL rate selection (2 = max FRL rate w/ DSC, 1 = max FRL rate, 0 = default rate)");
static int force_frl_rate = 0;
module_param_named(force_frl_rate, force_frl_rate, int, 0400);
module_param_named(force_frl_rate, force_frl_rate, int, 0444);
MODULE_PARM_DESC(conceal_vrr_caps,
"Conceal all display VRR capabilities");
static bool conceal_vrr_caps = false;
module_param_named(conceal_vrr_caps, conceal_vrr_caps, bool, 0400);
module_param_named(conceal_vrr_caps, conceal_vrr_caps, bool, 0444);
MODULE_PARM_DESC(enhanced_pcon_support,
"Enable enhanced display protocol converter features (e.g. VRR over active DP-HDMI converters)");
static bool enhanced_pcon_support = false;
module_param_named(enhanced_pcon_support, enhanced_pcon_support, bool, 0400);
module_param_named(enhanced_pcon_support, enhanced_pcon_support, bool, 0444);
/* Fail allocating the RM core channel for NVKMS using the i-th method (see
* FailAllocCoreChannelMethod). Failures not using the i-th method are ignored. */
MODULE_PARM_DESC(fail_alloc_core_channel, "Control testing for hardware core channel allocation failure");
static int fail_alloc_core_channel_method = -1;
module_param_named(fail_alloc_core_channel, fail_alloc_core_channel_method, int, 0400);
module_param_named(fail_alloc_core_channel, fail_alloc_core_channel_method, int, 0444);
MODULE_PARM_DESC(debug, "Enable debug logging");
static int debug = 0;
module_param_named(debug, debug, int, 0600);
module_param_named(debug, debug, int, 0644);
#if IS_ENABLED(CONFIG_EXTCON_DISP_CABLE_STATE)
/*
* Extcon reporting for HDMI video and audio state.
*
* The extcon-disp-state driver (CONFIG_EXTCON_DISP_CABLE_STATE) is an Android-specific
* kernel driver that exposes HDMI hotplug and audio state to userspace via the
* extcon subsystem. This is required for AOSP's WiredAccessoryManager (audio routing)
* and WindowManager (display management) to receive HDMI state notifications.
*
* These APIs handle device management and duplicate-state checking internally.
*/
void nvkms_extcon_report_hdmi(NvBool state)
{
disp_state_extcon_hdmi_report(state);
}
void nvkms_extcon_report_hdmi_audio(NvBool state)
{
disp_state_extcon_hdmi_audio_report(state);
}
#else
/* Unsupported STUB for nvkms_extcon APIs */
void nvkms_extcon_report_hdmi(NvBool state)
{
}
void nvkms_extcon_report_hdmi_audio(NvBool state)
{
}
#endif
#if NVKMS_CONFIG_FILE_SUPPORTED
/* This parameter is used to find the dpy override conf file */
@@ -132,7 +178,7 @@ module_param_named(debug, debug, int, 0600);
MODULE_PARM_DESC(config_file,
"Path to the nvidia-modeset configuration file (default: disabled)");
static char *nvkms_conf = NULL;
module_param_named(config_file, nvkms_conf, charp, 0400);
module_param_named(config_file, nvkms_conf, charp, 0444);
#endif
static atomic_t nvkms_alloc_called_count;
@@ -204,6 +250,22 @@ NvBool nvkms_hdmi_deepcolor(void)
return hdmi_deepcolor;
}
NvU32 nvkms_max_output_color_bpc(void)
{
switch (max_output_color_depth) {
case 6:
return 6;
case 8:
return 8;
case 12:
return 12;
case 10:
case 0:
default:
return 10;
}
}
NvBool nvkms_opportunistic_display_sync(void)
{
return opportunistic_display_sync;
@@ -474,20 +536,6 @@ NvBool nvkms_syncpt_op(
#define NVKMS_MAJOR_DEVICE_NUMBER 195
#define NVKMS_MINOR_DEVICE_NUMBER 254
/*
* Convert from microseconds to jiffies. The conversion is:
* ((usec) * HZ / 1000000)
*
* Use do_div() to avoid gcc-generated references to __udivdi3().
* Note that the do_div() macro divides the first argument in place.
*/
static inline unsigned long NVKMS_USECS_TO_JIFFIES(NvU64 usec)
{
unsigned long result = usec * HZ;
do_div(result, 1000000);
return result;
}
/*************************************************************************
* NVKMS uses a global lock, nvkms_lock. The lock is taken in the
@@ -1116,7 +1164,7 @@ nvkms_init_timer(struct nvkms_timer_t *timer, nvkms_timer_proc_t *proc,
timer_setup(&timer->kernel_timer, nvkms_timer_callback_typed_data, 0);
timer->kernel_timer_created = NV_TRUE;
mod_timer(&timer->kernel_timer, jiffies + NVKMS_USECS_TO_JIFFIES(usec));
mod_timer(&timer->kernel_timer, jiffies + usecs_to_jiffies(usec));
}
spin_unlock_irqrestore(&nvkms_timers.lock, flags);
}
@@ -2043,9 +2091,7 @@ static struct file_operations nvkms_fops = {
.owner = THIS_MODULE,
.poll = nvkms_poll,
.unlocked_ioctl = nvkms_unlocked_ioctl,
#if NVCPU_IS_X86_64 || NVCPU_IS_AARCH64
.compat_ioctl = nvkms_unlocked_ioctl,
#endif
.mmap = nvkms_mmap,
.open = nvkms_open,
.release = nvkms_close,
kernel-open/nvidia-modeset/nvidia-modeset-os-interface.h
+3
View File
@@ -124,10 +124,13 @@ NvBool nvkms_output_rounding_fix(void);
NvBool nvkms_disable_hdmi_frl(void);
NvBool nvkms_disable_vrr_memclk_switch(void);
NvBool nvkms_hdmi_deepcolor(void);
NvU32 nvkms_max_output_color_bpc(void);
NvBool nvkms_opportunistic_display_sync(void);
enum NvKmsDebugForceColorSpace nvkms_debug_force_color_space(void);
NvBool nvkms_enable_overlay_layers(void);
NvBool nvkms_debug_logging(void);
void nvkms_extcon_report_hdmi(NvBool state);
void nvkms_extcon_report_hdmi_audio(NvBool state);
void nvkms_call_rm (void *ops);
void* nvkms_alloc (size_t size,
kernel-open/nvidia-peermem/nvidia-peermem.Kbuild
+6 -10
View File
@@ -37,21 +37,17 @@ ifeq ($(ARCH), arm64)
OFA_ARCH := aarch64
else ifeq ($(ARCH), powerpc)
OFA_ARCH := ppc64le
else ifeq ($(ARCH), x86)
OFA_ARCH := x86_64
else
OFA_ARCH := $(ARCH)
endif
OFA_DIR := /usr/src/ofa_kernel
OFA_CANDIDATES = $(OFA_DIR)/$(OFA_ARCH)/$(KERNELRELEASE) $(OFA_DIR)/$(KERNELRELEASE) $(OFA_DIR)/default /var/lib/dkms/mlnx-ofed-kernel
MLNX_OFED_KERNEL := $(shell for d in $(OFA_CANDIDATES); do \
if [ -d "$$d" ]; then \
echo "$$d"; \
exit 0; \
fi; \
done; \
echo $(OFA_DIR) \
)
OFA_DKMS_DIR := $(OFA_DIR)-dkms
OFA_CANDIDATES = $(OFA_DKMS_DIR)/$(OFA_ARCH)/$(KERNELRELEASE) $(OFA_DIR)/$(OFA_ARCH)/$(KERNELRELEASE) $(OFA_DIR)/$(KERNELRELEASE) $(OFA_DIR)/default
MLNX_OFED_KERNEL := $(firstword $(wildcard $(OFA_CANDIDATES)))
ifneq ($(shell test -d $(MLNX_OFED_KERNEL) && echo "true" || echo "" ),)
ifneq ($(MLNX_OFED_KERNEL),)
NVIDIA_PEERMEM_CFLAGS += -I$(MLNX_OFED_KERNEL)/include -I$(MLNX_OFED_KERNEL)/include/rdma
KBUILD_EXTRA_SYMBOLS := $(MLNX_OFED_KERNEL)/Module.symvers
endif
kernel-open/nvidia-uvm/ctrl2080mc.h
+1 -4
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2013-2024 NVIDIA Corporation
Copyright (c) 2013-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -47,8 +47,5 @@
#define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10B (0x0000000B)
#define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB20B (0x0000000B)
/* valid ARCHITECTURE_GB20x implementation values */
#define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB206 (0x00000006)
#define NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB207 (0x00000007)
#endif /* _ctrl2080mc_h_ */
kernel-open/nvidia-uvm/nvidia-uvm-sources.Kbuild
+2
View File
@@ -79,6 +79,7 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_migrate.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_populate_pageable.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_migrate_pageable.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_map_external.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_va_range_dmabuf.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_user_channel.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_hmm.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_perf_heuristics.c
@@ -116,3 +117,4 @@ NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_thread_context_test.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_rb_tree_test.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_sec2_test.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_test_file.c
NVIDIA_UVM_SOURCES += nvidia-uvm/uvm_dmabuf_test.c
kernel-open/nvidia-uvm/nvidia-uvm.Kbuild
+3 -1
View File
@@ -52,10 +52,10 @@ NV_CONFTEST_FUNCTION_COMPILE_TESTS += list_is_first
NV_CONFTEST_FUNCTION_COMPILE_TESTS += ioasid_get
NV_CONFTEST_FUNCTION_COMPILE_TESTS += mm_pasid_drop
NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_sva_bind_device_has_drvdata_arg
NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_is_dma_domain
NV_CONFTEST_FUNCTION_COMPILE_TESTS += for_each_sgtable_dma_page
NV_CONFTEST_FUNCTION_COMPILE_TESTS += folio_test_swapcache
NV_CONFTEST_FUNCTION_COMPILE_TESTS += page_pgmap
NV_CONFTEST_FUNCTION_COMPILE_TESTS += iommu_is_dma_domain
NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_notifier_ops_arch_invalidate_secondary_tlbs
NV_CONFTEST_TYPE_COMPILE_TESTS += migrate_vma_added_flags
@@ -70,8 +70,10 @@ NV_CONFTEST_TYPE_COMPILE_TESTS += mmu_interval_notifier
NV_CONFTEST_TYPE_COMPILE_TESTS += sg_dma_page_iter
NV_CONFTEST_TYPE_COMPILE_TESTS += struct_page_has_zone_device_data
NV_CONFTEST_TYPE_COMPILE_TESTS += memory_device_coherent_present
NV_CONFTEST_TYPE_COMPILE_TESTS += dma_buf_supports_dynamic_importer
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_int_active_memcg
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_migrate_vma_setup
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present___iowrite64_lo_hi
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_present_make_device_exclusive
NV_CONFTEST_SYMBOL_COMPILE_TESTS += is_export_symbol_gpl_dma_iova_try_alloc
kernel-open/nvidia-uvm/uvm.c
+4
View File
@@ -1013,6 +1013,10 @@ static long uvm_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_CREATE_EXTERNAL_RANGE, uvm_api_create_external_range);
UVM_ROUTE_CMD_ALLOC_INIT_CHECK(UVM_MAP_EXTERNAL_ALLOCATION, uvm_api_map_external_allocation);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_MAP_EXTERNAL_SPARSE, uvm_api_map_external_sparse);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_IS_DMA_BUF_IMPORT_SUPPORTED, uvm_api_is_dma_buf_import_supported);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_IMPORT_DMA_BUF, uvm_api_import_dma_buf);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_MAP_DMA_BUF, uvm_api_map_dma_buf);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_UNMAP_DMA_BUF, uvm_api_unmap_dma_buf);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_FREE, uvm_api_free);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_DISCARD, uvm_api_discard);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_SET_PREFERRED_LOCATION, uvm_api_set_preferred_location);
kernel-open/nvidia-uvm/uvm.h
+240 -60
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2013-2025 NVIDIA Corporation
Copyright (c) 2013-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -233,7 +233,7 @@ NV_STATUS UvmDeinitialize(void);
//
// Arguments:
// flags: (INPUT)
// Must be zero. UVM will be reinitialized with the
// Must be zero. UVM will be reinitialized with the
// Same flags that were passed to UvmInitialize() originally.
//
// Error codes:
@@ -475,17 +475,12 @@ NV_STATUS UvmUnregisterGpu(const NvProcessorUuid *gpuUuid);
// on this GPU with the mapping and caching attributes as specified during that
// call, or with default attributes if none were specified.
//
// Any VA ranges that had a preferred location set to this GPU will be mapped on
// this GPU only if this GPU is not fault-capable and the VA range belongs to a
// non-migratable range group. If such a mapping cannot be established, an error
// is returned.
//
// Any VA ranges which have accessed-by set for this GPU will be mapped on this
// GPU. If that VA range resides in a PCIe peer GPU's memory and P2P support
// between the two GPUs has not been enabled via UvmEnablePeerAccess, then a
// mapping won't be established. Also, if read duplication is enabled for this
// VA range, or its preferred location is set to this GPU, and this GPU is a
// fault-capable GPU, then a mapping will not be established. If this is a
// fault-capable GPU, then a mapping will not be established. If this is a
// non-fault-capable GPU and a mapping cannot be established, then an error is
// returned.
//
@@ -926,9 +921,9 @@ NV_STATUS UvmReleaseVa(void *base,
//
// Creates a new mapping in the virtual address space of the process, populates
// it at the specified preferred location, maps it on the provided list of
// processors if feasible and associates the range with the given range group.
// If the preferredLocationUuid is the UUID of the CPU, preferred location is
// set to all CPU nodes allowed by the global and thread memory policies.
// processors if feasible. If the preferredLocationUuid is the UUID of the CPU,
// preferred location is set to all CPU nodes allowed by the global and thread
// memory policies.
//
// This API is equivalent to the following code sequence:
// UvmMemMap(base, length);
@@ -964,9 +959,6 @@ NV_STATUS UvmReleaseVa(void *base,
// accessedByCount: (INPUT)
// Number of elements in the accessedByUuids array.
//
// rangeGroupId: (INPUT)
// ID of the range group to associate this VA range with.
//
// Errors:
// NV_ERR_UVM_ADDRESS_IN_USE:
// The requested address range overlaps with an existing allocation.
@@ -980,9 +972,6 @@ NV_STATUS UvmReleaseVa(void *base,
// accessedByUuids array was not registered or the UUID represents a GPU
// that has no VA space registered for it.
//
// NV_ERR_OBJECT_NOT_FOUND:
// rangeGroupId was not found.
//
// NV_ERR_NO_MEMORY:
// Internal memory allocation failed.
//
@@ -1092,9 +1081,8 @@ NV_STATUS UvmCleanUpZombieResources(void);
// a GPU VA space is registered in the future for a GPU which is unable to map
// this allocation, that GPU VA space registration will fail.
//
// The pages in this VA range cannot be associated with range groups, cannot be
// the target for read duplication, cannot have a preferred location set, and
// cannot have any accessed-by processors.
// The pages in this VA range cannot be the target for read duplication, cannot
// have a preferred location set, and cannot have any accessed-by processors.
//
// The VA range can be unmapped and freed via a call to UvmFree.
//
@@ -1356,10 +1344,6 @@ NV_STATUS UvmAllocDeviceP2P(NvProcessorUuid gpuUuid,
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
// NV_WARN_MORE_PROCESSING_REQUIRED:
// Fewer than the number of requested pages were migrated because some
// pages were associated with a non-migratable range group.
//
//------------------------------------------------------------------------------
NV_STATUS UvmMigrate(void *base,
NvLength length,
@@ -1444,10 +1428,6 @@ NV_STATUS UvmMigrate(void *base,
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
// NV_WARN_MORE_PROCESSING_REQUIRED:
// Fewer than the number of requested pages were migrated because some
// pages were associated with a non-migratable range group.
//
//------------------------------------------------------------------------------
NV_STATUS UvmMigrateAsync(void *base,
NvLength length,
@@ -1657,10 +1637,9 @@ NV_STATUS UvmMemMap(void *base,
// previously reserved via UvmReserveVa.
//
// Any mappings created within this VA range are considered non-migratable.
// Consequently, pages cannot be associated with range groups, cannot be
// the target for read duplication, cannot have a preferred location set,
// cannot have any accessed-by processors, and any GPU faults within this range
// are fatal.
// Consequently, pages cannot be the target for read duplication, cannot have a
// preferred location set, cannot have any accessed-by processors, and any GPU
// faults within this range are fatal.
//
// Mappings within this range neither create nor modify any CPU mappings, even
// if the mappings came from a region previously reserved via UvmReserveVa.
@@ -1879,7 +1858,6 @@ NV_STATUS UvmMapExternalAllocation(void *base,
// length: (INPUT)
// Length, in bytes, of the range. The length must be 64K aligned.
//
//
// gpuUuid: (INPUT)
// UUID of the physical GPU if the GPU is not SMC capable or SMC
// enabled, or the GPU instance UUID of the partition to map the sparse
@@ -1968,6 +1946,225 @@ NV_STATUS UvmUnmapExternal(void *base,
NvLength length,
const NvProcessorUuid *gpuUuid);
//------------------------------------------------------------------------------
// UvmIsDmaBufImportSupported
//
// Returns true only if the system features support for the Linux kernel
// DMA-BUF interface.
//
// Arguments:
// dmaBufImportSupported: (OUTPUT)
// Returns true (non-zero) if the system has sufficient support for the
// Linux kernel DMA-BUF interface and false (zero) otherwise.
//
// Error codes:
// NV_ERR_INVALID_STATE:
// UVM was not initialized.
//
// NV_ERR_GENERIC:
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
//------------------------------------------------------------------------------
NV_STATUS UvmIsDmaBufImportSupported(NvBool *dmaBufImportSupported);
//------------------------------------------------------------------------------
// UvmImportDmaBuf
//
// Create a VA range within the process's address space reserved for mapping
// memory exposed through the kernel DMA-BUF interface through a handle file
// descriptor. The VA range is not mapped to any physical allocation at the
// time of creation. Once the DMA-BUF VA range has been created using this API,
// the user is free to map any number of gpus to the memory referenced by the
// DMA-BUF (see UvmMapDmaBuf for more details) provided that they are
// actually capable of mapping that memory. Often the DMA-BUF memories are
// expected to be PCIe peers or otherwise on self-hosted systems, the CPU must
// be able to map the memory.
//
// The virtual address range, itself, does not impose any restrictions on the
// alignment of the physical allocations mapped within it. However, both base
// and length must be aligned to the system page size.
//
// The VA range must not overlap with an existing VA range, irrespective of
// whether the existing range corresponds to a UVM allocation or an external
// allocation.
//
// It is allowed (but not required) for the VA range to come from a region
// previously reserved via UvmReserveVa.
//
// Any mappings created within this VA range are considered non-migratable.
// Consequently, pages cannot be the target for read duplication, cannot have a
// preferred location set, cannot have any accessed-by processors, and any GPU
// faults within this range are fatal.
//
// Mappings within this range neither create nor modify any CPU mappings, even
// if the mappings came from a region previously reserved via UvmReserveVa.
// This implies that CPU accesses to any mappings within this range will cause
// a fatal fault if it's not mapped.
//
// The VA range is not reclaimed until UvmFree is called on it even if it is
// fully unmapped from all GPUs either explicitly via UvmUnmapDmaBuf or
// implicitly via APIs such as UvmUnregisterGpu, UvmUnregisterGpuVaSpace,
// UvmDisablePeerAccess, etc.
//
// Arguments:
// base: (INPUT)
// Base address of the virtual address range.
//
// length: (INPUT)
// Length, in bytes, of the range.
//
// Errors:
// NV_ERR_INVALID_ADDRESS:
// base is NULL or length is zero or at least one of base and length is
// not aligned to the system page size, or the length is not equal to
// the reported size of the DMA-BUF.
//
// NV_ERR_UVM_ADDRESS_IN_USE:
// The requested address range overlaps with an existing allocation.
//
// NV_ERR_NO_MEMORY:
// Internal memory allocation failed.
//
//------------------------------------------------------------------------------
NV_STATUS UvmImportDmaBuf(UvmDmaBufFileDescriptor handleFd,
void *base,
NvLength length);
//------------------------------------------------------------------------------
// UvmMapDmaBuf
//
// Maps a buffer exposed by file descriptor through the kernel DMA-BUF interface
// on the specified GPU. The DMA-BUF can be unmapped from a specific GPU using
// UvmUnmapDmaBuf or from all GPUs using UvmFree.
//
// The virtual address range specified by (base, length) must fall within a VA
// range previously created with UvmImportDmaBuf. A GPU VA space must
// have been registered for each GPU in the list. The (base, length) range must
// lie within the largest possible virtual address supported by the specified
// GPUs.
//
// The page size used for the mapping is the system page size, as the DMA-BUF
// interface requires exporters to ensure page size alignment for the DMA
// mapped chunks returned through the dma_buf_map_attachment() call.
//
// The page size used for the mapping is the largest supported page size less
// than or equal to the alignments of base, length, offset, and the allocation
// page size.
//
// If the range specified by (base, length) falls within any existing mappings,
// the behavior is the same as if UvmUnmapDmaBuf with the range specified by
// (base, length) had been called first.
//
// The underlying buffer must not reside in GPU memory.
//
// Arguments:
// base: (INPUT)
// Base address of the virtual address range.
//
// length: (INPUT)
// Length, in bytes, of the range.
//
// gpuUuid: (INPUT)
// UUID of the physical GPU if the GPU is not SMC capable or SMC
// enabled, or the GPU instance UUID of the partition to map the VA
// range.
//
// Errors:
// NV_ERR_INVALID_ADDRESS:
// One of the following occurred:
// - base is NULL.
// - length is zero.
// - either base and length are not aligned to the system page size.
// - The requested address range does not fall entirely within an
// existing external VA range created with a single call to
// UvmCreateExternalRange.
// - The mapping page size allowed by the alignments of base, length,
// and offset is smaller than the minimum supported page size on the
// GPU.
// - base or base + length fall within an existing mapping but are not
// aligned to that mapping's page size.
// - The DMA mappings to the memory backing this DMA-BUF have been
// revoked by the exporter, and so no GPU mapping is possible.
//
// NV_ERR_OUT_OF_RANGE:
// The range specified by (base, length) exceeds the largest virtual
// address supported by one or more of the specified GPUs.
//
// NV_ERR_INVALID_DEVICE:
// One of the following occurred:
// - The gpu UUID to be mapped was either not registered or has no GPU
// VA space registered for it.
// - The specified GPU is unable to DMA map the buffer (for example,
// the exporter rejected the attachment, the backing store is not
// reachable from this GPU, or the requested direction is not
// supported).
//
// NV_ERR_NO_MEMORY:
// Internal memory allocation failed.
//
// NV_ERR_GENERIC:
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
//------------------------------------------------------------------------------
NV_STATUS UvmMapDmaBuf(void *base,
NvLength length,
const NvProcessorUuid *gpuUuid);
//------------------------------------------------------------------------------
// UvmUnmapDmaBuf
//
// Unmaps a virtual address range that was mapped using UvmMapDmaBuf from the
// specified GPU. The range specified by (base, length) must be fully contained
// within a single DMA-BUF VA range created with UvmImportDmaBuf.
//
// If the range specified by (base, length) range partially overlaps existing
// mappings, the overlapping portion of the existing mappings will be unmapped
// provided that the split points are aligned to the mappings' respective page
// sizes. Otherwise, the overlapping portions of the existing mappings will be
// left in an undefined state.
//
// Note that the VA range is not reclaimed until UvmFree is called on it even if
// all mappings in the created range have been unmapped from all GPUs via this
// API.
//
// Arguments:
// base: (INPUT)
// Base address of the virtual address range.
//
// length: (INPUT)
// The length of the virtual address range.
//
// gpuUuid: (INPUT)
// UUID of the physical GPU if the GPU is not SMC capable or SMC
// enabled, or the GPU instance UUID of the partition to unmap the VA
// range from.
//
// Errors:
// NV_ERR_INVALID_ADDRESS:
// One of the following has occurred:
// - base is NULL.
// - The requested address range does not fall entirely within an
// existing external VA range created with a single call to
// UvmImportDmaBuf.
// - base or base + length fall within an existing mapping but are not
// aligned to that mapping's page size.
//
// NV_ERR_INVALID_DEVICE:
// Either gpuUuid does not represent a valid registered GPU or the VA
// range corresponding to the given base address is not mapped on the
// specified GPU.
//
// NV_ERR_GENERIC:
// Unexpected error. We try hard to avoid returning this error code,
// because it is not very informative.
//
//------------------------------------------------------------------------------
NV_STATUS UvmUnmapDmaBuf(void *base,
NvLength length,
const NvProcessorUuid *gpuUuid);
//------------------------------------------------------------------------------
// UvmMapDynamicParallelismRegion
//
@@ -1980,11 +2177,11 @@ NV_STATUS UvmUnmapExternal(void *base,
// mapping for the GPU. A GPU VA space must have been registered for the GPU and
// the GPU must support dynamic parallelism.
//
// The mapping is created immediately and not modified until a call to UvmFree
// Calling UvmFree frees the GPU page table mapping. The range cannot be
// associated with range groups and any GPU faults within this range are fatal.
// Also, the pages cannot be the target for read duplication, cannot have a
// preferred location set, and cannot have any accessed-by processors.
// The mapping is created immediately and not modified until a call to UvmFree.
// Calling UvmFree frees the GPU page table mapping. Any GPU faults within this
// range are fatal. Also, the pages cannot be the target for read duplication,
// cannot have a preferred location set, and cannot have any accessed-by
// processors.
//
// Note that calling UvmUnregisterGpuVaSpace will also unmap all mappings
// created via this API on the GPU that the GPU VA space is associated with.
@@ -2253,16 +2450,8 @@ NV_STATUS UvmDisableReadDuplication(void *base,
// specified processor.
//
// NV_ERR_INVALID_DEVICE:
// One of the following occurred:
// - preferredLocationUuid is neither the UUID of the CPU nor the UUID
// of a GPU that was registered by this process.
// - At least one page in VA range belongs to a non-migratable range
// group and the specified UUID represents a fault-capable GPU.
// - preferredLocationUuid is the UUID of a non-fault-capable GPU and at
// least one page in the VA range belongs to a non-migratable range
// group and another non-fault-capable GPU is in the accessed-by list
// of the same page but P2P support between both GPUs has not been
// enabled.
// preferredLocationUuid is neither the UUID of the CPU nor the UUID of
// a GPU that was registered by this process.
//
// NV_ERR_INVALID_ARGUMENT:
// One of the following occured:
@@ -2359,9 +2548,7 @@ NV_STATUS UvmUnsetPreferredLocation(void *base,
// permit a mapping to be established, then no mapping is created for that page.
// If a page in the VA range migrates to a new location, then the mapping is
// updated to point to the new location if establishing such a mapping is
// possible. If a page in the VA range is associated with a non-migratable range
// group and the specified processor is a non-fault-capable GPU, then an error
// is returned if the mapping cannot be established.
// possible.
//
// If the specified processor is a GPU and no GPU VA space has been registered
// for it or if the registered GPU VA space gets unregistered, then the policies
@@ -2420,11 +2607,7 @@ NV_STATUS UvmUnsetPreferredLocation(void *base,
//
// NV_ERR_INVALID_DEVICE:
// accessedByUuid is neither the UUID of the CPU nor the UUID of a GPU
// that was registered by this process. Or accessedByUuid is the UUID of
// a non-fault-capable GPU and the VA range is associated with a
// non-migratable range group with a preferred location set to another
// non-fault-capable GPU that doesn't have P2P support enabled with this
// GPU.
// that was registered by this process.
//
// NV_ERR_NO_MEMORY:
// accessedByUuid is a non-fault-capable GPU and there was insufficient
@@ -2691,7 +2874,6 @@ NV_STATUS UvmIs8Supported(NvU32 *is8Supported);
// least that you should expect is that all your session related objects will
// become useless once target process closes Uvm file handle.
//
//
// There are security requirements for this call to be successful. Fortunately,
// after validating a file descriptor, one of the following conditions must
// hold:
@@ -2864,7 +3046,6 @@ NV_STATUS UvmToolsCreateEventQueue_V2(UvmToolsSessionHandle session,
UvmToolsEventQueueDescriptor UvmToolsGetEventQueueDescriptor(UvmToolsEventQueueHandle queue);
//------------------------------------------------------------------------------
// UvmToolsSetNotificationThreshold
//
@@ -2981,7 +3162,6 @@ NV_STATUS UvmToolsEventQueueEnableEvents(UvmToolsEventQueueHandle queue,
NV_STATUS UvmToolsEventQueueDisableEvents(UvmToolsEventQueueHandle queue,
NvU64 eventTypeFlags);
//------------------------------------------------------------------------------
// UvmToolsCreateProcessAggregateCounters
//
@@ -3348,7 +3528,7 @@ NV_STATUS UvmToolsGetProcessorUuidTable_V2(UvmToolsSessionHandle session,
// UvmToolsFlushEvents
//
// Some events, like migrations, which have end timestamps are not immediately
// submitted to queues when they are completed. This call enqueues any
// submitted to queues when they are completed. This call enqueues any
// completed but unenqueued events associated with the session.
//
// Arguments:
kernel-open/nvidia-uvm/uvm_ada.c
+1 -1
View File
@@ -63,7 +63,7 @@ void uvm_hal_ada_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->access_bits_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
kernel-open/nvidia-uvm/uvm_ampere.c
+1 -1
View File
@@ -83,7 +83,7 @@ void uvm_hal_ampere_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->map_remap_larger_page_promotion = false;
// Note that access bits are only supported GA10x onwards
parent_gpu->access_bits_supported = false;
parent_gpu->access_bits_supported = true;
}
parent_gpu->ats.no_ats_range_required = false;
kernel-open/nvidia-uvm/uvm_api.h
+9 -5
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2025 NVIDIA Corporation
Copyright (c) 2015-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -34,10 +34,10 @@
#include "uvm_va_space.h"
#include "nv_uvm_user_types.h"
// This weird number comes from UVM_PREVENT_MIGRATION_RANGE_GROUPS_PARAMS. That
// ioctl is called frequently so we don't want to allocate a copy every time.
// It's a little over 256 bytes in size.
#define UVM_MAX_IOCTL_PARAM_STACK_SIZE 288
// This number comes from UVM_TEST_RANGE_TREE_RANDOM_PARAMS. It cover almost all
// driver and test ioctls. Above this limit, ioctls require two orders of
// magnitude more space. In these cases, the UVM_ROUTE_CMD_ALLOC_* is preferred.
#define UVM_MAX_IOCTL_PARAM_STACK_SIZE 176
// The UVM_ROUTE_CMD_* macros are only intended for use in the ioctl routines
@@ -238,6 +238,10 @@ NV_STATUS uvm_api_disable_peer_access(UVM_DISABLE_PEER_ACCESS_PARAMS *params, st
NV_STATUS uvm_api_create_external_range(UVM_CREATE_EXTERNAL_RANGE_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_map_external_allocation(UVM_MAP_EXTERNAL_ALLOCATION_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_map_external_sparse(UVM_MAP_EXTERNAL_SPARSE_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_is_dma_buf_import_supported(UVM_IS_DMA_BUF_IMPORT_SUPPORTED_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_import_dma_buf(UVM_IMPORT_DMA_BUF_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_map_dma_buf(UVM_MAP_DMA_BUF_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_unmap_dma_buf(UVM_UNMAP_DMA_BUF_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_free(UVM_FREE_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_discard(UVM_DISCARD_PARAMS *params, struct file *filp);
NV_STATUS uvm_api_set_preferred_location(const UVM_SET_PREFERRED_LOCATION_PARAMS *params, struct file *filp);
kernel-open/nvidia-uvm/uvm_blackwell.c
+9 -20
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2022-2025 NVIDIA Corporation
Copyright (c) 2022-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -79,7 +79,7 @@ void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->ats.gmmu_pt_depth0_init_required = parent_gpu->ats.non_pasid_ats_enabled;
parent_gpu->access_bits_supported = false;
parent_gpu->access_bits_supported = true;
// Blackwell has a physical translation prefetcher, meaning SW must assume
// that any physical ATS translation can be fetched at any time. The
@@ -121,29 +121,18 @@ void uvm_hal_blackwell_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->conf_computing.per_channel_key_rotation = true;
// TODO: Bug 5023085: this should be queried from RM instead of determined
// by UVM.
if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100 &&
parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10B) {
// If the parent GPU has no vidmem, initialise as an iGPU. This is the
// same property that is queried by NV0000_CTRL_GPU_GET_ID_INFO_V2.
if (parent_gpu->rm_info.gpuArchIsZeroFb) {
parent_gpu->is_integrated_gpu = true;
parent_gpu->access_bits_supported = false;
}
if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100 &&
parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB10B) {
// GB10B has sticky L2 coherent cache lines.
// For details, refer to the comments in uvm_gpu.h
// where this field is declared.
parent_gpu->sticky_l2_coherent_cache_lines = true;
}
if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB200 &&
parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB20B) {
parent_gpu->is_integrated_gpu = true;
parent_gpu->access_bits_supported = false;
}
if (parent_gpu->rm_info.gpuArch == NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB200 &&
(parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB206 ||
parent_gpu->rm_info.gpuImplementation == NV2080_CTRL_MC_ARCH_INFO_IMPLEMENTATION_GB207)) {
// TODO: Bug 3186788 : As reported in Bug 5309034, GB206
// and GB207 experience a GSP crash with VAB. Depending
// on whether RM fixes it or marks it as cannot fix, the
// below checks can be removed or retained.
parent_gpu->access_bits_supported = false;
}
}
kernel-open/nvidia-uvm/uvm_devmem.c
+12 -5
View File
@@ -48,10 +48,12 @@ uvm_va_space_t *uvm_devmem_page_to_va_space(struct page *page)
// uvm_hmm_unregister_gpu() needs to do a racy check here so
// page->zone_device_data might be NULL.
if (!gpu_chunk || !gpu_chunk->va_block)
if (!gpu_chunk || !gpu_chunk->va_space)
return NULL;
return gpu_chunk->va_block->hmm.va_space;
// Use the cached va_space rather than gpu_chunk->va_block->hmm.va_space
// since the va_block may have been freed while the va_space is still valid.
return gpu_chunk->va_space;
}
// Check there are no orphan pages. This should be only called as part of
@@ -113,10 +115,14 @@ static void devmem_page_free_gpu_chunk(uvm_gpu_chunk_t *chunk)
{
uvm_gpu_t *gpu = uvm_gpu_chunk_get_gpu(chunk);
if (chunk->va_block) {
uvm_va_space_t *va_space = chunk->va_block->hmm.va_space;
UVM_ASSERT(chunk->va_space);
// Use the cached va_space rather than chunk->va_block->hmm.va_space. The
// va_block may already have been freed by the time this page_free() callback
// runs, but the va_space is guaranteed to outlive the device page.
if (chunk->va_space) {
uvm_va_space_t *va_space = chunk->va_space;
UVM_ASSERT(va_space);
atomic64_dec(&va_space->hmm.allocated_page_count);
UVM_ASSERT(atomic64_read(&va_space->hmm.allocated_page_count) >= 0);
}
@@ -130,6 +136,7 @@ static void devmem_page_free_gpu_chunk(uvm_gpu_chunk_t *chunk)
UVM_ASSERT(chunk->is_referenced);
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk->is_referenced = false;
if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) {
kernel-open/nvidia-uvm/uvm_dmabuf_test.c
+313
View File
@@ -0,0 +1,313 @@
/*******************************************************************************
Copyright (c) 2015-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_linux.h"
#include "nv-kref.h"
#include "uvm_test.h"
#include "uvm_test_ioctl.h"
#include "uvm_kvmalloc.h"
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
#include <linux/dma-buf.h>
#include <linux/dma-mapping.h>
#include <linux/dma-resv.h>
#include <linux/err.h>
#include <linux/mm.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
// The struct dma_buf doesn't tracking the backing store.
typedef struct {
struct page **pages;
unsigned long num_pages;
size_t size;
} dma_buf_priv_t;
// The dma_buf_attach() and dma_buf_map_attachment() return but don't
// store the scatter gather table of DMA mappings.
typedef struct {
struct sg_table *sg_table;
enum dma_data_direction direction;
} attachment_priv_t;
static int sysmem_attach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
attachment_priv_t *attch_priv;
attch_priv = uvm_kvmalloc_zero(sizeof(*attch_priv));
if (!attch_priv)
return -ENOMEM;
attach->priv = attch_priv;
return 0;
}
static void sysmem_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
uvm_kvfree(attach->priv);
attach->priv = NULL;
}
static struct sg_table *sysmem_map(struct dma_buf_attachment *attach,
enum dma_data_direction direction)
{
dma_buf_priv_t *dmabuf_priv = attach->dmabuf->priv;
attachment_priv_t *attach_priv = attach->priv;
struct sg_table *sg_table;
int ret;
sg_table = uvm_kvmalloc_zero(sizeof(*sg_table));
if (!sg_table)
return ERR_PTR(-ENOMEM);
// Merges adjacent pages into fewer sg entries where possible.
ret = sg_alloc_table_from_pages(sg_table,
dmabuf_priv->pages,
dmabuf_priv->num_pages,
0,
dmabuf_priv->size,
GFP_KERNEL);
if (ret) {
uvm_kvfree(sg_table);
return ERR_PTR(ret);
}
ret = dma_map_sgtable(attach->dev,
sg_table,
direction,
0);
if (ret) {
sg_free_table(sg_table);
uvm_kvfree(sg_table);
return ERR_PTR(ret);
}
attach_priv->sg_table = sg_table;
attach_priv->direction = direction;
return sg_table;
}
static void sysmem_unmap(struct dma_buf_attachment *attach,
struct sg_table *sg_table,
enum dma_data_direction direction)
{
attachment_priv_t *attach_priv = attach->priv;
attach_priv->sg_table = NULL;
dma_unmap_sgtable(attach->dev, sg_table, direction, 0);
sg_free_table(sg_table);
uvm_kvfree(sg_table);
}
static int sysmem_mmap(struct dma_buf *dmabuf,
struct vm_area_struct *vma)
{
dma_buf_priv_t *dmabuf_priv = dmabuf->priv;
int i, ret;
if (vma_pages(vma) > dmabuf_priv->num_pages)
return -EINVAL;
for (i = 0; i < vma_pages(vma); i++) {
ret = vm_insert_page(vma, vma->vm_start + i * PAGE_SIZE,
dmabuf_priv->pages[i]);
if (ret)
return ret;
}
return 0;
}
static int sysmem_begin_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
struct dma_buf_attachment *attach;
dma_resv_lock(dmabuf->resv, NULL);
list_for_each_entry(attach, &dmabuf->attachments, node) {
attachment_priv_t *attach_priv = attach->priv;
if (attach_priv->sg_table)
dma_sync_sgtable_for_cpu(attach->dev, attach_priv->sg_table, direction);
}
dma_resv_unlock(dmabuf->resv);
return 0;
}
static int sysmem_end_cpu_access(struct dma_buf *dmabuf,
enum dma_data_direction direction)
{
struct dma_buf_attachment *attach;
dma_resv_lock(dmabuf->resv, NULL);
list_for_each_entry(attach, &dmabuf->attachments, node) {
attachment_priv_t *attach_priv = attach->priv;
if (attach_priv->sg_table)
dma_sync_sgtable_for_device(attach->dev, attach_priv->sg_table, direction);
}
dma_resv_unlock(dmabuf->resv);
return 0;
}
static void sysmem_release(struct dma_buf *dmabuf)
{
int i;
dma_buf_priv_t *dmabuf_priv = dmabuf->priv;
if (!dmabuf_priv->pages) {
uvm_kvfree(dmabuf_priv);
return;
}
for (i = 0; i < dmabuf_priv->num_pages; i++)
__free_page(dmabuf_priv->pages[i]);
uvm_kvfree(dmabuf_priv->pages);
dmabuf_priv->pages = NULL;
dmabuf_priv->num_pages = 0;
uvm_kvfree(dmabuf_priv);
}
static const struct dma_buf_ops uvm_sysmem_ops = {
.attach = sysmem_attach,
.detach = sysmem_detach,
.map_dma_buf = sysmem_map,
.unmap_dma_buf = sysmem_unmap,
.mmap = sysmem_mmap,
.begin_cpu_access = sysmem_begin_cpu_access,
.end_cpu_access = sysmem_end_cpu_access,
.release = sysmem_release,
};
NV_STATUS uvm_test_export_dma_buf(UVM_TEST_EXPORT_DMA_BUF_PARAMS *params,
struct file *filp)
{
DEFINE_DMA_BUF_EXPORT_INFO(exp_info);
dma_buf_priv_t *dmabuf_priv;
struct dma_buf *dmabuf;
unsigned long i;
int fd;
if (!params->size || !IS_ALIGNED(params->size, PAGE_SIZE))
return NV_ERR_INVALID_ARGUMENT;
dmabuf_priv = uvm_kvmalloc_zero(sizeof(*dmabuf_priv));
if (!dmabuf_priv)
return NV_ERR_NO_MEMORY;
dmabuf_priv->size = params->size;
dmabuf_priv->num_pages = params->size >> PAGE_SHIFT;
dmabuf_priv->pages = uvm_kvmalloc_zero(dmabuf_priv->num_pages * sizeof(*dmabuf_priv->pages));
if (!dmabuf_priv->pages) {
uvm_kvfree(dmabuf_priv);
return NV_ERR_NO_MEMORY;
}
for (i = 0; i < dmabuf_priv->num_pages; i++) {
dmabuf_priv->pages[i] = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!dmabuf_priv->pages[i]) {
while (i--)
__free_page(dmabuf_priv->pages[i]);
uvm_kvfree(dmabuf_priv->pages);
uvm_kvfree(dmabuf_priv);
return NV_ERR_NO_MEMORY;
}
}
exp_info.ops = &uvm_sysmem_ops;
exp_info.size = dmabuf_priv->size;
exp_info.flags = O_RDWR | O_CLOEXEC;
exp_info.priv = dmabuf_priv;
dmabuf = dma_buf_export(&exp_info);
if (IS_ERR(dmabuf)) {
UVM_ERR_PRINT("dma_buf_export failed: %ld\n", PTR_ERR(dmabuf));
for (i = 0; i < dmabuf_priv->num_pages; i++)
__free_page(dmabuf_priv->pages[i]);
uvm_kvfree(dmabuf_priv->pages);
uvm_kvfree(dmabuf_priv);
return NV_ERR_OPERATING_SYSTEM;
}
fd = dma_buf_fd(dmabuf, O_CLOEXEC);
if (fd < 0) {
UVM_ERR_PRINT("dma_buf_fd failed: %d\n", fd);
dma_buf_put(dmabuf);
return NV_ERR_OPERATING_SYSTEM;
}
params->fd = fd;
return NV_OK;
}
NV_STATUS uvm_test_revoke_dma_buf(UVM_TEST_REVOKE_DMA_BUF_PARAMS *params,
struct file *filp)
{
struct dma_buf *dmabuf;
dmabuf = dma_buf_get(params->fd);
if (IS_ERR(dmabuf))
return NV_ERR_INVALID_ARGUMENT;
if (dmabuf->ops != &uvm_sysmem_ops) {
dma_buf_put(dmabuf);
return NV_ERR_INVALID_ARGUMENT;
}
// Notify importers that the DMA-BUF is being revoked. The callback,
// invalidate_mappings() is expected to synchronously provide a new mapping,
// but in UVM's use case all GMMU mappings will be withdrawn and any
// further access should produce a fatal fault.
dma_resv_lock(dmabuf->resv, NULL);
#if defined(NV_DMA_BUF_ATTACH_OPS_HAS_INVALIDATE_MAPPINGS)
dma_buf_invalidate_mappings(dmabuf);
#else
dma_buf_move_notify(dmabuf);
#endif
dma_resv_unlock(dmabuf->resv);
dma_buf_put(dmabuf);
return NV_OK;
}
#else
NV_STATUS uvm_test_export_dma_buf(UVM_TEST_EXPORT_DMA_BUF_PARAMS *params,
struct file *filp)
{
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS uvm_test_revoke_dma_buf(UVM_TEST_REVOKE_DMA_BUF_PARAMS *params,
struct file *filp)
{
return NV_ERR_NOT_SUPPORTED;
}
#endif // UVM_PROVIDES_DMA_BUF_IMPORTER()
kernel-open/nvidia-uvm/uvm_forward_decl.h
+1
View File
@@ -58,6 +58,7 @@ typedef struct uvm_va_policy_struct uvm_va_policy_t;
typedef struct uvm_va_range_struct uvm_va_range_t;
typedef struct uvm_va_range_managed_struct uvm_va_range_managed_t;
typedef struct uvm_va_range_external_struct uvm_va_range_external_t;
typedef struct uvm_va_range_dma_buf_struct uvm_va_range_dma_buf_t;
typedef struct uvm_va_range_channel_struct uvm_va_range_channel_t;
typedef struct uvm_va_range_sked_reflected_struct uvm_va_range_sked_reflected_t;
typedef struct uvm_va_range_semaphore_pool_struct uvm_va_range_semaphore_pool_t;
kernel-open/nvidia-uvm/uvm_global.c
+12 -8
View File
@@ -43,7 +43,16 @@
#include "uvm_devmem.h"
uvm_global_t g_uvm_global;
static struct UvmEventsLinux g_exported_uvm_events;
static struct UvmEventsLinux g_exported_uvm_events =
{
.isrTopHalf = uvm_isr_top_half_entry,
.suspend = uvm_suspend_entry,
.resume = uvm_resume_entry,
.drainP2P = uvm_suspend_and_drainP2P_entry,
.resumeP2P = uvm_resumeP2P_entry,
};
static bool g_ops_registered = false;
unsigned uvm_force_conf_computing = 0;
@@ -53,12 +62,6 @@ static NV_STATUS uvm_register_callbacks(void)
{
NV_STATUS status = NV_OK;
g_exported_uvm_events.isrTopHalf = uvm_isr_top_half_entry;
g_exported_uvm_events.suspend = uvm_suspend_entry;
g_exported_uvm_events.resume = uvm_resume_entry;
g_exported_uvm_events.drainP2P = uvm_suspend_and_drainP2P_entry;
g_exported_uvm_events.resumeP2P = uvm_resumeP2P_entry;
// Register the UVM callbacks with the main GPU driver:
status = uvm_rm_locked_call(nvUvmInterfaceRegisterUvmEvents(&g_exported_uvm_events));
if (status != NV_OK)
@@ -117,7 +120,7 @@ NV_STATUS uvm_global_init(void)
status = uvm_rm_locked_call(nvUvmInterfaceSessionCreate(&g_uvm_global.rm_session_handle, &platform_info));
if (status != NV_OK) {
UVM_ERR_PRINT("nvUvmInterfaceSessionCreate() failed: %s\n", nvstatusToString(status));
return status;
goto error;
}
uvm_ats_init(&platform_info);
@@ -129,6 +132,7 @@ NV_STATUS uvm_global_init(void)
g_uvm_global.hw_conf_computing_enabled = platform_info.confComputingEnabled;
g_uvm_global.conf_computing_enabled = g_uvm_global.hw_conf_computing_enabled ||
uvm_force_conf_computing;
g_uvm_global.have_non_ats_devices = platform_info.nonAtsDevicePresent;
status = uvm_processor_mask_cache_init();
if (status != NV_OK) {
kernel-open/nvidia-uvm/uvm_global.h
+3
View File
@@ -125,6 +125,9 @@ struct uvm_global_struct
bool cdmm_enabled;
// Indicates whether there is a non-ATS GPU in the system.
bool have_non_ats_devices;
// List of all active VA spaces
struct
{
kernel-open/nvidia-uvm/uvm_gpu.c
+155 -17
View File
@@ -46,6 +46,9 @@
#include "uvm_linux.h"
#include "uvm_mmu.h"
#include "uvm_kvmalloc.h"
#if UVM_USE_DMA_IOVA_API()
#include <linux/dma-mapping.h>
#endif
#define UVM_PROC_GPUS_PEER_DIR_NAME "peers"
@@ -123,7 +126,13 @@ static void fill_parent_gpu_info(uvm_parent_gpu_t *parent_gpu, const UvmGpuInfo
// 47-bit address space holds the routing information for each peer.
// Currently, this is limited to a 16GB framebuffer window size.
parent_gpu->nvswitch_info.fabric_memory_window_start = gpu_info->nvswitchMemoryWindowStart;
parent_gpu->nvswitch_info.egm_fabric_memory_window_start = gpu_info->nvswitchEgmMemoryWindowStart;
if (gpu_info->flaWindowForCpuMemNode.bSupported) {
parent_gpu->egm.fabric_memory_window_start = gpu_info->flaWindowForCpuMemNode.flaStart;
parent_gpu->egm.window_size = gpu_info->flaWindowForCpuMemNode.flaSize;
}
else {
parent_gpu->egm.fabric_memory_window_start = gpu_info->nvswitchEgmMemoryWindowStart;
}
}
parent_gpu->ats.non_pasid_ats_enabled = gpu_info->nonPasidAtsSupport;
@@ -1470,9 +1479,23 @@ static NV_STATUS init_parent_gpu(uvm_parent_gpu_t *parent_gpu,
parent_gpu->closest_cpu_numa_node = dev_to_node(&parent_gpu->pci_dev->dev);
parent_gpu->dma_addressable_start = gpu_platform_info->dma_addressable_start;
parent_gpu->dma_addressable_limit = gpu_platform_info->dma_addressable_limit;
parent_gpu->egm.enabled = gpu_info->egmEnabled;
// RM considers gpu_info->egmEnabled as "legacy" EGM and FLA EGM as
// EGM over FLA. Therefore, the two settings are mutually exclusive.
// However, it would make it easier for us to gate EGM (both styles)
// using a single variable.
UVM_ASSERT(!(gpu_info->egmEnabled && gpu_info->flaWindowForCpuMemNode.bSupported));
parent_gpu->egm.enabled = gpu_info->egmEnabled || gpu_info->flaWindowForCpuMemNode.bSupported;
parent_gpu->egm.local_peer_id = gpu_info->egmPeerId;
parent_gpu->egm.base_address = gpu_info->egmBaseAddr;
parent_gpu->egm.legacy = !gpu_info->flaWindowForCpuMemNode.bSupported;
if (!parent_gpu->egm.legacy) {
UVM_ASSERT(parent_gpu->rm_info.gpuArch >= NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GR100);
parent_gpu->egm.base_address = gpu_info->flaWindowForCpuMemNode.dmaAddrBase;
parent_gpu->egm.identity_iommu = gpu_info->flaWindowForCpuMemNode.bIdentityDmaMap;
}
else {
parent_gpu->egm.base_address = gpu_info->egmBaseAddr;
}
parent_gpu->access_counters_supported = (gpu_info->accessCntrBufferCount != 0);
status = uvm_rm_locked_call(nvUvmInterfaceGetFbInfo(parent_gpu->rm_device, &fb_info));
@@ -2464,17 +2487,24 @@ static NV_STATUS parent_peers_init(uvm_parent_gpu_t *parent_gpu0,
parent_peer_caps->link_type = link_type;
parent_peer_caps->total_link_line_rate_mbyte_per_s = p2p_caps_params.totalLinkLineRateMBps;
if (parent_gpu0->egm.enabled || parent_gpu1->egm.enabled)
if (uvm_parent_gpu_egm_enabled(parent_gpu0) || uvm_parent_gpu_egm_enabled(parent_gpu1))
UVM_ASSERT(parent_peer_caps->link_type >= UVM_GPU_LINK_NVLINK_2);
// Initialize peer ids and establish peer mappings
// Peer id from min(gpu_id0, gpu_id1) -> max(gpu_id0, gpu_id1)
parent_peer_caps->peer_ids[0] = p2p_caps_params.peerIds[0];
parent_peer_caps->egm_peer_ids[0] = p2p_caps_params.egmPeerIds[0];
// Peer id from max(gpu_id0, gpu_id1) -> min(gpu_id0, gpu_id1)
parent_peer_caps->peer_ids[1] = p2p_caps_params.peerIds[1];
parent_peer_caps->egm_peer_ids[1] = p2p_caps_params.egmPeerIds[1];
// RM does not set egmPeerIds for Rubin+ where the EGM carveout is no
// longer supported. Intead, it uses the normal peer IDs for FLA EGM.
if (!uvm_parent_gpu_egm_is_legacy(parent_gpu0) || !uvm_parent_gpu_egm_is_legacy(parent_gpu1)) {
parent_peer_caps->egm_peer_ids[0] = parent_peer_caps->peer_ids[0];
parent_peer_caps->egm_peer_ids[1] = parent_peer_caps->peer_ids[1];
}
else {
parent_peer_caps->egm_peer_ids[0] = p2p_caps_params.egmPeerIds[0];
parent_peer_caps->egm_peer_ids[1] = p2p_caps_params.egmPeerIds[1];
}
parent_peer_caps->optimalNvlinkWriteCEs[0] = p2p_caps_params.optimalNvlinkWriteCEs[0];
parent_peer_caps->optimalNvlinkWriteCEs[1] = p2p_caps_params.optimalNvlinkWriteCEs[1];
@@ -3286,12 +3316,15 @@ uvm_aperture_t uvm_gpu_egm_peer_aperture(uvm_parent_gpu_t *local_gpu, uvm_parent
uvm_parent_gpu_peer_t *peer_caps;
NvU8 peer_id;
if (local_gpu == remote_gpu) {
UVM_ASSERT(local_gpu->egm.enabled);
if (uvm_parent_id_equal(local_gpu->id, remote_gpu->id)) {
if (!uvm_parent_gpu_egm_is_legacy(local_gpu))
return UVM_APERTURE_SYS;
UVM_ASSERT(uvm_parent_gpu_egm_enabled(local_gpu));
peer_id = local_gpu->egm.local_peer_id;
}
else {
UVM_ASSERT(remote_gpu->egm.enabled);
UVM_ASSERT(uvm_parent_gpu_egm_enabled(remote_gpu));
peer_caps = parent_gpu_peer_caps(local_gpu, remote_gpu);
@@ -3302,7 +3335,22 @@ uvm_aperture_t uvm_gpu_egm_peer_aperture(uvm_parent_gpu_t *local_gpu, uvm_parent
peer_id = peer_caps->egm_peer_ids[1];
}
return UVM_APERTURE_PEER(peer_id);
return UVM_APERTURE_EGM_PEER(peer_id);
}
NV_STATUS uvm_test_query_egm_state(UVM_TEST_QUERY_EGM_STATE_PARAMS *params, struct file *filp)
{
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_gpu_t *gpu;
gpu = uvm_va_space_retain_gpu_by_uuid(va_space, &params->gpu_uuid);
if (!gpu)
return NV_ERR_INVALID_DEVICE;
params->enabled = uvm_parent_gpu_egm_enabled(gpu->parent);
uvm_gpu_release(gpu);
return NV_OK;
}
NvU64 uvm_gpu_peer_ref_count(const uvm_gpu_t *gpu0, const uvm_gpu_t *gpu1)
@@ -3387,18 +3435,22 @@ bool uvm_gpu_address_is_peer(uvm_gpu_t *gpu, uvm_gpu_address_t address)
}
}
else {
if (uvm_aperture_is_peer(address.aperture)) {
if (uvm_aperture_is_peer(address.aperture))
return true;
if (uvm_aperture_is_egm(address.aperture)) {
uvm_parent_processor_mask_t peer_parent_gpus;
uvm_parent_gpu_t *peer_parent_gpu;
// Local EGM accesses don't go over NVLINK
if (gpu->parent->egm.enabled && address.aperture == gpu->parent->egm.local_peer_id)
if (uvm_parent_gpu_egm_enabled(gpu->parent) && uvm_parent_gpu_egm_is_legacy(gpu->parent) &&
UVM_APERTURE_EGM_PEER_ID(address.aperture) == gpu->parent->egm.local_peer_id)
return false;
uvm_spin_lock(&gpu->peer_info.peer_gpu_lock);
uvm_parent_gpus_from_processor_mask(&peer_parent_gpus, &gpu->peer_info.peer_gpu_mask);
for_each_parent_gpu_in_mask(peer_parent_gpu, &peer_parent_gpus) {
if (!peer_parent_gpu->egm.enabled)
if (!uvm_parent_gpu_egm_enabled(peer_parent_gpu))
continue;
// EGM uses peer IDs but they are different from VIDMEM peer
@@ -3976,12 +4028,13 @@ NV_STATUS uvm_gpu_map_cpu_pages_no_invalidate(uvm_gpu_t *gpu, struct page *page,
UVM_ASSERT(PAGE_ALIGNED(size));
// This is also the path we take when EGM is enabled but the
// IOMMU is OFF or in IDENTITY domain.
dma_addr = dma_map_page(&parent_gpu->pci_dev->dev, page, 0, size, DMA_BIDIRECTIONAL);
if (dma_mapping_error(&parent_gpu->pci_dev->dev, dma_addr))
return NV_ERR_OPERATING_SYSTEM;
if (dma_addr < parent_gpu->dma_addressable_start ||
dma_addr + size - 1 > parent_gpu->dma_addressable_limit) {
if (dma_addr < parent_gpu->dma_addressable_start || dma_addr + size - 1 > parent_gpu->dma_addressable_limit) {
dma_unmap_page(&parent_gpu->pci_dev->dev, dma_addr, size, DMA_BIDIRECTIONAL);
UVM_ERR_PRINT_RL("PCI mapped range [0x%llx, 0x%llx) not in the addressable range [0x%llx, 0x%llx), GPU %s\n",
dma_addr,
@@ -3994,7 +4047,73 @@ NV_STATUS uvm_gpu_map_cpu_pages_no_invalidate(uvm_gpu_t *gpu, struct page *page,
atomic64_add(size, &parent_gpu->mapped_cpu_pages_size);
*dma_address_out = uvm_parent_gpu_dma_addr_to_gpu_addr(parent_gpu, dma_addr);
return NV_OK;
}
NV_STATUS uvm_gpu_map_cpu_pages_for_egm(uvm_parent_gpu_t *parent_gpu,
struct page *page,
size_t size,
NvU64 *dma_address_out)
{
unsigned long node_start = node_start_pfn(page_to_nid(page)) << PAGE_SHIFT;
dma_addr_t dma_addr = 0;
NV_STATUS status = NV_OK;
// This function only actually needs the parent GPU, but it takes in the
// sub GPU for API symmetry with uvm_gpu_map_cpu_pages().
UVM_ASSERT(PAGE_ALIGNED(size));
*dma_address_out = dma_addr;
// If EGM is not enabled, we don't create any mappings. They shouldn't
// be needed in such a case anyway.
// If EGM is enabled but it's legacy EGM, we also don't need a mapping
// because the EGM PA is based on the page's physical address in
// SYSMEM.
if (!uvm_parent_gpu_egm_enabled(parent_gpu) || uvm_parent_gpu_egm_is_legacy(parent_gpu))
return NV_OK;
if (uvm_parent_gpu_egm_iommu_is_identity(parent_gpu)) {
phys_addr_t phys_addr = page_to_phys(page);
dma_addr = phys_addr - node_start;
}
else {
#if UVM_USE_DMA_IOVA_API()
// Note that the check below also implies kernels >= 6.17+, where
// the DMA IOVA API is available. RM will only enable EGM with FLA
// mappings and create DMA IOVA windows only on those kernel.
struct dma_iova_state state = {.addr = uvm_parent_gpu_egm_iova_address(parent_gpu),
.__size = uvm_parent_gpu_egm_window_size(parent_gpu)};
phys_addr_t phys_addr = page_to_phys(page);
NvU64 offset = phys_addr - node_start;
int ret;
UVM_ASSERT(page_to_nid(page) == parent_gpu->closest_cpu_numa_node);
UVM_ASSERT(size <= uvm_parent_gpu_egm_window_size(parent_gpu));
ret = dma_iova_link(&parent_gpu->pci_dev->dev, &state, phys_addr, offset, size, DMA_BIDIRECTIONAL, 0);
if (ret != 0) {
status = errno_to_nv_status(ret);
goto done;
}
ret = dma_iova_sync(&parent_gpu->pci_dev->dev, &state, offset, size);
if (ret != 0) {
dma_iova_unlink(&parent_gpu->pci_dev->dev, &state, offset, size, DMA_BIDIRECTIONAL, 0);
status = errno_to_nv_status(ret);
goto done;
}
dma_addr = offset;
atomic64_add(size, &parent_gpu->mapped_cpu_pages_size);
done:
#else
status = NV_ERR_NOT_SUPPORTED;
#endif
}
*dma_address_out = dma_addr;
return NV_OK;
}
@@ -4018,10 +4137,29 @@ void uvm_parent_gpu_unmap_cpu_pages(uvm_parent_gpu_t *parent_gpu, NvU64 dma_addr
UVM_ASSERT(PAGE_ALIGNED(size));
dma_address = gpu_addr_to_dma_addr(parent_gpu, dma_address);
dma_unmap_page(&parent_gpu->pci_dev->dev, dma_address, size, DMA_BIDIRECTIONAL);
atomic64_sub(size, &parent_gpu->mapped_cpu_pages_size);
}
void uvm_parent_gpu_unmap_cpu_pages_for_egm(uvm_parent_gpu_t *parent_gpu, NvU64 dma_address, size_t size)
{
if (!uvm_parent_gpu_egm_enabled(parent_gpu) || uvm_parent_gpu_egm_is_legacy(parent_gpu))
return;
#if UVM_USE_DMA_IOVA_API()
if (!uvm_parent_gpu_egm_iommu_is_identity(parent_gpu)) {
// See comment in uvm_gpu_map_cpu_pages_no_invalidate.
struct dma_iova_state state = {.addr = uvm_parent_gpu_egm_iova_address(parent_gpu),
.__size = uvm_parent_gpu_egm_window_size(parent_gpu)};
dma_iova_unlink(&parent_gpu->pci_dev->dev, &state, dma_address, size, DMA_BIDIRECTIONAL, 0);
atomic64_sub(size, &parent_gpu->mapped_cpu_pages_size);
}
#endif
return;
}
// This function implements the UvmRegisterGpu API call, as described in uvm.h.
// Notes:
//
kernel-open/nvidia-uvm/uvm_gpu.h
+79 -10
View File
@@ -1288,10 +1288,6 @@ struct uvm_parent_gpu_struct
// 47-bit fabric memory physical offset that peer gpus need to access
// to read a peer's memory
NvU64 fabric_memory_window_start;
// 47-bit fabric memory physical offset that peer gpus need to access
// to read remote EGM memory.
NvU64 egm_fabric_memory_window_start;
} nvswitch_info;
struct
@@ -1351,15 +1347,34 @@ struct uvm_parent_gpu_struct
// Is EGM support enabled on this GPU.
bool enabled;
// Legacy EGM uses GPA GMMU mappings while non-legacy uses FLA.
bool legacy;
// Local EGM peer ID. This ID is used to route EGM memory accesses to
// the local CPU socket.
// the local CPU socket. Only, valid/used for legacy EGM.
NvU8 local_peer_id;
// EGM base address of the EGM carveout for remote EGM accesses.
// The base address is used when computing PTE PA address values for
// accesses to the local CPU socket's EGM memory from other peer
// GPUs.
NvU64 base_address;
// For non-legacy EGM, this is the base of the DMA mapping created
// by RM for the SYSMEM window of the NUMA node to which the GPU
// is attached.
dma_addr_t base_address;
// Size of EGM memory window. Effectively, this is the size of the
// DMA mapping created by RM.
size_t window_size;
// True if IOMMU is off or IOMMU_DOMAIN_IDENTITY type.
bool identity_iommu;
// 47-bit fabric memory physical offset that peer gpus need to access
// to read remote EGM memory.
// This field is also used for FLA mappings. It holds the base FLA
// address for each peer GPU.
NvU64 fabric_memory_window_start;
} egm;
// Peer VIDMEM base offset used when creating GPA PTEs for
@@ -1617,14 +1632,16 @@ static bool uvm_parent_gpu_supports_full_coherence(uvm_parent_gpu_t *parent_gpu)
// so pageable memory which uses ZONE_DEVICE can only be migrated to GPUs which
// have neither property.
//
// CDMM GPUs use a separate devmem mechanism that does not rely on
// DEVICE_PRIVATE pages, so UVM_CAN_USE_DEVICE_PRIVATE_MEMREMAP_PAGES() does
// not apply to them — uvm_devmem_init() handles this distinction internally.
// CDMM GPUs use a separate devmem mechanism that does not rely on HMM or
// DEVICE_PRIVATE pages. Other GPUs only support devmem when HMM is enabled
// system-wide and DEVICE_PRIVATE pages can be used.
static bool uvm_gpu_supports_devmem(uvm_gpu_t *gpu)
{
if (gpu->mem_info.numa.enabled || gpu->parent->is_integrated_gpu)
return false;
return gpu->parent->cdmm_enabled || UVM_CAN_USE_DEVICE_PRIVATE_MEMREMAP_PAGES();
return gpu->parent->cdmm_enabled ||
(uvm_hmm_is_enabled_system_wide() && UVM_CAN_USE_DEVICE_PRIVATE_MEMREMAP_PAGES());
}
// Returns a GPU peer pair index in the range [0 .. UVM_MAX_UNIQUE_GPU_PAIRS).
@@ -1788,9 +1805,59 @@ NV_STATUS uvm_gpu_map_cpu_pages(uvm_gpu_t *gpu, struct page *page, size_t size,
// uvm_hal_tlb_invalidate_phys().
NV_STATUS uvm_gpu_map_cpu_pages_no_invalidate(uvm_gpu_t *gpu, struct page *page, size_t size, NvU64 *dma_address_out);
// Create IOVA mappings for EGM accesses.
// Please note that the GPU passed here is not the accessing GPU
// but the routing GPU.
// Note that this function take in a parent GPU unlike other mapping
// functions.
//
// Locking: The needs to have retained the parent GPU. The block
// lock for the black that is being mapped should be held.
NV_STATUS uvm_gpu_map_cpu_pages_for_egm(uvm_parent_gpu_t *parent_gpu,
struct page *page,
size_t size,
NvU64 *dma_address_out);
// Unmap num_pages pages previously mapped with uvm_gpu_map_cpu_pages().
void uvm_parent_gpu_unmap_cpu_pages(uvm_parent_gpu_t *parent_gpu, NvU64 dma_address, size_t size);
static bool uvm_parent_gpu_egm_enabled(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.enabled;
}
static bool uvm_parent_gpu_egm_iommu_is_identity(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.identity_iommu;
}
static bool uvm_parent_gpu_egm_is_legacy(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.legacy;
}
static dma_addr_t uvm_parent_gpu_egm_iova_address(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.base_address;
}
static NvU64 uvm_parent_gpu_egm_fabric_address(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.fabric_memory_window_start;
}
static size_t uvm_parent_gpu_egm_window_size(uvm_parent_gpu_t *parent_gpu)
{
return parent_gpu->egm.window_size;
}
// Destroy IOVA mappings for EGM.
// Just like with uvm_vapu_map_cpu_pages_for_egm(), the parent GPU
// passed here is the routing GPU.
//
// Locking: Same as uvm_parent_gpu_map_cpu_pages_for_egm()
void uvm_parent_gpu_unmap_cpu_pages_for_egm(uvm_parent_gpu_t *parent_gpu, NvU64 dma_address, size_t size);
static NV_STATUS uvm_gpu_map_cpu_page(uvm_gpu_t *gpu, struct page *page, NvU64 *dma_address_out)
{
return uvm_gpu_map_cpu_pages(gpu, page, PAGE_SIZE, dma_address_out);
@@ -1918,4 +1985,6 @@ typedef enum
// PCIe BAR containing static framebuffer memory mappings for PCIe P2P
int uvm_device_p2p_static_bar(uvm_parent_gpu_t *gpu);
NV_STATUS uvm_test_query_egm_state(UVM_TEST_QUERY_EGM_STATE_PARAMS *params, struct file *filp);
#endif // __UVM_GPU_H__
kernel-open/nvidia-uvm/uvm_gpu_access_counters.c
+135 -69
View File
@@ -27,6 +27,7 @@
#include "uvm_api.h"
#include "uvm_gpu.h"
#include "uvm_hal.h"
#include "uvm_hmm.h"
#include "uvm_kvmalloc.h"
#include "uvm_tools.h"
#include "uvm_va_block.h"
@@ -1126,7 +1127,8 @@ static NV_STATUS service_va_block_locked(uvm_gpu_t *gpu,
uvm_va_block_t *va_block,
uvm_va_block_retry_t *va_block_retry,
uvm_service_block_context_t *service_context,
uvm_page_mask_t *accessed_pages)
uvm_page_mask_t *accessed_pages,
bool *did_migrate)
{
NV_STATUS status = NV_OK;
uvm_page_index_t page_index;
@@ -1139,17 +1141,24 @@ static NV_STATUS service_va_block_locked(uvm_gpu_t *gpu,
uvm_assert_mutex_locked(&va_block->lock);
// GPU VA space could be gone since we received the notification. We handle
// this case by skipping service if processor is not in the mapped mask.
// Using this approach we also filter out notifications for pages that
// moved since they were reported by the GPU. This is fine because:
// Filter out notifications for pages that moved since they were reported by
// the GPU. This is fine because:
// - If the GPU is still accessing them, it should have faulted
// - If the GPU gets remote mappings in the future, we will get new
// notifications and we will act accordingly
// - If the GPU does not access the pages again, we do not want to migrate
// them
if (!uvm_processor_mask_test(&va_block->mapped, processor))
return NV_OK;
//
// Skip this check for coherent GPUs since the coherent GPU would never be
// in the mapped list of the va_block.
//
// gpu_va_space is obtained under va_space lock with uvm_gpu_va_space_get().
// gpu_va_space can't be destroyed from under us.This code is only reached
// iff gpu_va_space is active, valid and non-null.
if (!(uvm_parent_gpu_is_coherent(gpu->parent) && uvm_va_block_is_hmm(va_block)) &&
!uvm_processor_mask_test(&va_block->mapped, processor)) {
return NV_OK;
}
if (uvm_processor_mask_test(&va_block->resident, processor))
residency_mask = uvm_va_block_resident_mask_get(va_block, processor, NUMA_NO_NODE);
@@ -1264,8 +1273,21 @@ static NV_STATUS service_va_block_locked(uvm_gpu_t *gpu,
&policy,
&outer);
}
if (status != NV_OK)
break;
if (status != NV_OK) {
if (uvm_va_block_is_hmm(va_block)) {
// HMM va_blocks may be backed by multiple VMAs with
// different protections.
// uvm_hmm_find_policy_vma_and_outer() can return
// NV_ERR_INVALID_ADDRESS if one of the VMAs doesn't
// even have VM_READ or if there's no backing VMA.
status = NV_OK;
first_page_index++;
continue;
}
else {
break;
}
}
}
service_context->region = uvm_va_block_region(first_page_index, outer);
@@ -1274,6 +1296,11 @@ static NV_STATUS service_va_block_locked(uvm_gpu_t *gpu,
status = uvm_va_block_service_locked(gpu, va_block, va_block_retry, service_context);
if (status != NV_OK)
break;
if (did_migrate &&
!uvm_page_mask_empty(&service_context->block_context->make_resident.pages_changed_residency)) {
*did_migrate = true;
}
}
}
}
@@ -1294,12 +1321,15 @@ static NV_STATUS service_va_block_locked(uvm_gpu_t *gpu,
static NV_STATUS service_notification_va_block_helper(struct mm_struct *mm,
uvm_va_block_t *va_block,
uvm_gpu_t *gpu,
uvm_access_counter_service_batch_context_t *batch_context)
uvm_access_counter_service_batch_context_t *batch_context,
bool *did_migrate)
{
uvm_va_block_retry_t va_block_retry;
uvm_page_mask_t *accessed_pages = &batch_context->accessed_pages;
uvm_service_block_context_t *service_context = &batch_context->block_service_context;
*did_migrate = false;
if (uvm_page_mask_empty(accessed_pages))
return NV_OK;
@@ -1314,10 +1344,12 @@ static NV_STATUS service_notification_va_block_helper(struct mm_struct *mm,
va_block,
&va_block_retry,
service_context,
accessed_pages));
accessed_pages,
did_migrate));
}
static void expand_notification_block(uvm_gpu_va_space_t *gpu_va_space,
struct mm_struct *mm,
uvm_va_block_t *va_block,
const uvm_access_counter_buffer_t *access_counters,
uvm_va_block_context_t *va_block_context,
@@ -1326,10 +1358,10 @@ static void expand_notification_block(uvm_gpu_va_space_t *gpu_va_space,
{
NvU64 addr;
NvU64 granularity = 0;
uvm_gpu_t *resident_gpu = NULL;
uvm_processor_id_t resident_id;
uvm_page_index_t page_index;
uvm_gpu_t *gpu = gpu_va_space->gpu;
const bool is_hmm = uvm_va_block_is_hmm(va_block);
config_granularity_to_bytes(access_counters->current_config.rm.granularity, &granularity);
@@ -1347,17 +1379,29 @@ static void expand_notification_block(uvm_gpu_va_space_t *gpu_va_space,
resident_id = uvm_va_block_page_get_closest_resident(va_block, va_block_context, page_index, gpu->id);
// resident_id might be invalid or might already be the same as the GPU
// which received the notification if the memory was already migrated before
// acquiring the locks either during the servicing of previous notifications
// or during faults or because of explicit migrations or if the VA range was
// freed after receiving the notification. Return NV_OK in such cases.
// For managed memory, resident_id might be invalid or might already be the
// same as the GPU which received the notification if the memory was already
// migrated before acquiring the locks either during the servicing of
// previous notifications or during faults or because of explicit migrations
// or if the VA range was freed after receiving the notification. Return
// NV_OK in such cases.
//
// For HMM/CDMM system memory, resident_id might be invalid if there were no
// associated va_blocks before the access counter notifications were
// triggered. This implies resident_id was CPU.
if (is_hmm) {
if (!UVM_ID_IS_VALID(resident_id)) {
NV_STATUS status = uvm_hmm_va_block_update_residency_info(va_block, mm, addr, false);
if (status != NV_OK)
return;
resident_id = uvm_va_block_page_get_closest_resident(va_block, va_block_context, page_index, gpu->id);
}
}
if (!UVM_ID_IS_VALID(resident_id) || uvm_id_equal(resident_id, gpu->id))
return;
if (UVM_ID_IS_GPU(resident_id))
resident_gpu = uvm_gpu_get(resident_id);
if (uvm_va_block_get_physical_size(va_block, resident_id, page_index) != granularity) {
uvm_page_mask_set(accessed_pages, page_index);
}
@@ -1390,6 +1434,7 @@ static NV_STATUS service_notifications_in_block(uvm_gpu_va_space_t *gpu_va_space
struct mm_struct *mm,
uvm_access_counter_buffer_t *access_counters,
uvm_va_block_t *va_block,
struct vm_area_struct *vma,
NvU32 index,
NvU32 *out_index)
{
@@ -1397,12 +1442,16 @@ static NV_STATUS service_notifications_in_block(uvm_gpu_va_space_t *gpu_va_space
NvU32 flags = 0;
NV_STATUS status = NV_OK;
NV_STATUS flags_status;
bool did_migrate = false;
bool has_accessed_pages;
NvU64 end;
uvm_gpu_t *gpu = gpu_va_space->gpu;
uvm_va_space_t *va_space = gpu_va_space->va_space;
uvm_access_counter_service_batch_context_t *batch_context = &access_counters->batch_service_context;
uvm_page_mask_t *accessed_pages = &batch_context->accessed_pages;
uvm_access_counter_buffer_entry_t **notifications = batch_context->notifications;
uvm_service_block_context_t *service_context = &batch_context->block_service_context;
const bool is_hmm = uvm_va_block_is_hmm(va_block);
UVM_ASSERT(va_block);
UVM_ASSERT(index < batch_context->num_notifications);
@@ -1413,6 +1462,19 @@ static NV_STATUS service_notifications_in_block(uvm_gpu_va_space_t *gpu_va_space
uvm_va_block_context_init(service_context->block_context, mm);
if (is_hmm) {
NvU64 base;
uvm_access_counter_buffer_entry_t *current_entry = notifications[index];
NvU64 address = current_entry->address;
uvm_hmm_migrate_begin_wait(va_block);
UVM_ASSERT(vma);
base = UVM_VA_BLOCK_ALIGN_DOWN(address);
end = min(base + UVM_VA_BLOCK_SIZE, (NvU64)vma->vm_end);
}
uvm_mutex_lock(&va_block->lock);
for (i = index; i < batch_context->num_notifications; i++) {
@@ -1422,7 +1484,11 @@ static NV_STATUS service_notifications_in_block(uvm_gpu_va_space_t *gpu_va_space
if (current_entry->va_space != va_space || current_entry->gpu != gpu || address > va_block->end)
break;
if (is_hmm && (address >= end))
break;
expand_notification_block(gpu_va_space,
mm,
va_block,
access_counters,
batch_context->block_service_context.block_context,
@@ -1437,11 +1503,16 @@ static NV_STATUS service_notifications_in_block(uvm_gpu_va_space_t *gpu_va_space
batch_context->block_service_context.access_counters_buffer_index = access_counters->index;
status = service_notification_va_block_helper(mm, va_block, gpu, batch_context);
has_accessed_pages = !uvm_page_mask_empty(accessed_pages);
status = service_notification_va_block_helper(mm, va_block, gpu, batch_context, &did_migrate);
uvm_mutex_unlock(&va_block->lock);
if (status == NV_OK)
if (is_hmm)
uvm_hmm_migrate_finish(va_block);
if (status == NV_OK && (!is_hmm || !has_accessed_pages || did_migrate))
flags |= UVM_ACCESS_COUNTER_ACTION_BATCH_CLEAR;
flags_status = notify_tools_and_process_flags(va_space,
@@ -1589,12 +1660,7 @@ static NV_STATUS service_notifications_batch(uvm_gpu_va_space_t *gpu_va_space,
}
if (va_block) {
status = service_notifications_in_block(gpu_va_space,
mm,
access_counters,
va_block,
index,
out_index);
status = service_notifications_in_block(gpu_va_space, mm, access_counters, va_block, NULL, index, out_index);
}
else {
status = notify_tools_and_process_flags(va_space,
@@ -1608,56 +1674,56 @@ static NV_STATUS service_notifications_batch(uvm_gpu_va_space_t *gpu_va_space,
*out_index = index + 1;
}
}
else if (uvm_ats_can_service_faults(gpu_va_space, mm)) {
if (!gpu_va_space->gpu->parent->cdmm_enabled) {
status = service_notification_ats(gpu_va_space, mm, access_counters, index, out_index);
}
else {
status = notify_tools_and_process_flags(va_space,
gpu_va_space->gpu,
access_counters,
0,
batch_context->notifications,
1,
0,
NULL);
*out_index = index + 1;
}
}
else {
NvU32 flags;
uvm_va_block_t *va_block = NULL;
status = uvm_hmm_va_block_find(va_space, address, &va_block);
// TODO: Bug 4309292: [UVM][HMM] Re-enable access counter HMM block
// migrations for virtual notifications
//
// - If the va_block is HMM, don't clear the notification since HMM
// migrations are currently disabled.
//
// - If the va_block isn't HMM, the notification belongs to a recently
// freed va_range. Clear the notification entry to continue receiving
// notifications when a new va_range is allocated in this region.
flags = va_block ? 0 : UVM_ACCESS_COUNTER_ACTION_BATCH_CLEAR;
UVM_ASSERT((status == NV_ERR_OBJECT_NOT_FOUND) ||
(status == NV_ERR_INVALID_ADDRESS) ||
uvm_va_block_is_hmm(va_block));
// Clobber status to continue processing the rest of the notifications
// in the batch.
else if (!gpu_va_space->va_space->pageable.migrations_enabled) {
status = notify_tools_and_process_flags(va_space,
gpu_va_space->gpu,
access_counters,
0,
batch_context->notifications,
&batch_context->notifications[index],
1,
flags,
0,
NULL);
*out_index = index + 1;
}
else if (uvm_ats_can_service_faults(gpu_va_space, mm)) {
status = service_notification_ats(gpu_va_space, mm, access_counters, index, out_index);
}
else {
NvU32 flags;
uvm_va_block_t *va_block = NULL;
struct vm_area_struct *vma = NULL;
// HMM and CDMM pageable memory are serviced through HMM VA blocks.
status = uvm_hmm_va_block_find_create(va_space, address, &vma, &va_block);
if (va_block) {
UVM_ASSERT(uvm_va_block_is_hmm(va_block));
status = service_notifications_in_block(gpu_va_space, mm, access_counters, va_block, vma, index, out_index);
}
else {
// If no va_block was found, the notifications was for a recently
// freed memory region. Clear the notification entry to continue
// receiving notifications when a new region is allocated.
UVM_ASSERT(status == NV_ERR_INVALID_ADDRESS);
flags = UVM_ACCESS_COUNTER_ACTION_BATCH_CLEAR;
// Clobber status to continue processing the rest of the
// notifications in the batch.
status = notify_tools_and_process_flags(va_space,
gpu_va_space->gpu,
access_counters,
0,
&batch_context->notifications[index],
1,
flags,
NULL);
*out_index = index + 1;
}
}
return status;
}
kernel-open/nvidia-uvm/uvm_gpu_non_replayable_faults.c
+6 -3
View File
@@ -441,15 +441,18 @@ static NV_STATUS service_managed_fault_in_block_locked(uvm_va_block_t *va_block,
// If this is a ATS processor and the page is already resident in the correct location
// then it should already be mapped on the CPU so handle this as a minor fault.
if (uvm_va_block_is_hmm(va_block) && gpu->parent->ats_supported) {
uvm_page_mask_t *resident_pages = uvm_va_block_resident_mask_get(va_block, gpu->id, NUMA_NO_NODE);
if (resident_pages && uvm_page_mask_test(resident_pages, page_index)) {
uvm_va_block_page_resident_processors(va_block, page_index,
&service_context->block_context->scratch_processor_mask);
if (uvm_processor_mask_test(&service_context->block_context->scratch_processor_mask, gpu->id)) {
unsigned int flags = FAULT_FLAG_REMOTE;
if (fault_entry->fault_access_type >= UVM_FAULT_ACCESS_TYPE_WRITE)
flags |= FAULT_FLAG_WRITE;
// handle_mm_fault() may call hmm_invalidate() so we can't call it with the block lock held
uvm_mutex_unlock(&va_block->lock);
UVM_HANDLE_MM_FAULT(service_context->block_context->hmm.vma,
uvm_va_block_cpu_page_address(va_block, page_index), flags);
uvm_mutex_lock(&va_block->lock);
return NV_OK;
}
kernel-open/nvidia-uvm/uvm_gpu_replayable_faults.c
+9 -6
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2025 NVIDIA Corporation
Copyright (c) 2015-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -1259,12 +1259,11 @@ static void fault_entry_duplicate_flags(uvm_fault_service_batch_context_t *batch
// need to be cancelled since they disappear on replay.
//
// The UVM driver considers two scenarios for logical permissions violation:
// - All access types are invalid. For example, when faulting from a processor
// that doesn't have access to the preferred location of a range group when it
// is not migratable. In this case all accesses to the page must be cancelled.
// - Access types are invalid, see:
// uvm_va_block.h:uvm_va_block_check_logical_permissions()
// - Write/atomic accesses are invalid. Basically, when trying to modify a
// read-only VA range. In this case we restrict fault cancelling to those types
// of accesses.
// read-only VA range. In this case we restrict fault cancelling to those
// types of accesses.
//
// Return values:
// - service_access_type: highest access type that can be serviced.
@@ -1534,8 +1533,12 @@ static NV_STATUS service_fault_batch_block_locked(uvm_gpu_va_space_t *gpu_va_spa
if (service_access_type >= UVM_FAULT_ACCESS_TYPE_WRITE)
flags |= FAULT_FLAG_WRITE;
// handle_mm_fault() may call hmm_invalidate() so we can't call it with the block lock held
uvm_mutex_unlock(&va_block->lock);
UVM_HANDLE_MM_FAULT(block_context->block_context->hmm.vma,
uvm_va_block_cpu_page_address(va_block, page_index), flags);
uvm_mutex_lock(&va_block->lock);
continue;
}
}
kernel-open/nvidia-uvm/uvm_hal.c
+10 -1
View File
@@ -806,7 +806,7 @@ void uvm_hal_tlb_invalidate_phys(uvm_push_t *push, uvm_dma_map_invalidation_t in
const char *uvm_aperture_string(uvm_aperture_t aperture)
{
BUILD_BUG_ON(UVM_APERTURE_MAX != 13);
BUILD_BUG_ON(UVM_APERTURE_MAX != 22);
switch (aperture) {
UVM_ENUM_STRING_CASE(UVM_APERTURE_PEER_0);
@@ -818,6 +818,15 @@ const char *uvm_aperture_string(uvm_aperture_t aperture)
UVM_ENUM_STRING_CASE(UVM_APERTURE_PEER_6);
UVM_ENUM_STRING_CASE(UVM_APERTURE_PEER_7);
UVM_ENUM_STRING_CASE(UVM_APERTURE_PEER_MAX);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_0);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_1);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_2);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_3);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_4);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_5);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_6);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_7);
UVM_ENUM_STRING_CASE(UVM_APERTURE_EGM_PEER_MAX);
UVM_ENUM_STRING_CASE(UVM_APERTURE_SYS);
UVM_ENUM_STRING_CASE(UVM_APERTURE_SYS_NON_COHERENT);
UVM_ENUM_STRING_CASE(UVM_APERTURE_VID);
kernel-open/nvidia-uvm/uvm_hal_types.h
+31
View File
@@ -42,6 +42,16 @@ typedef enum
UVM_APERTURE_PEER_7,
UVM_APERTURE_PEER_MAX,
UVM_APERTURE_EGM_PEER_0,
UVM_APERTURE_EGM_PEER_1,
UVM_APERTURE_EGM_PEER_2,
UVM_APERTURE_EGM_PEER_3,
UVM_APERTURE_EGM_PEER_4,
UVM_APERTURE_EGM_PEER_5,
UVM_APERTURE_EGM_PEER_6,
UVM_APERTURE_EGM_PEER_7,
UVM_APERTURE_EGM_PEER_MAX,
// SYS aperture is used for memory addresses in system physical address
// space, or device DMA space. This is currently used for system memory
// and certain types of peer memory.
@@ -123,6 +133,27 @@ static inline uvm_aperture_t UVM_APERTURE_PEER(NvU32 id)
return aperture;
}
static bool uvm_aperture_is_egm(uvm_aperture_t aperture)
{
return (aperture >= UVM_APERTURE_EGM_PEER_0) && (aperture < UVM_APERTURE_EGM_PEER_MAX);
}
static inline NvU32 UVM_APERTURE_EGM_PEER_ID(uvm_aperture_t aperture)
{
UVM_ASSERT(uvm_aperture_is_egm(aperture));
return (NvU32)aperture - (NvU32)UVM_APERTURE_EGM_PEER_0;
}
static inline uvm_aperture_t UVM_APERTURE_EGM_PEER(NvU32 id)
{
uvm_aperture_t aperture = (uvm_aperture_t)(UVM_APERTURE_EGM_PEER_0 + id);
UVM_ASSERT(UVM_APERTURE_EGM_PEER_ID(aperture) == id);
return aperture;
}
// A physical GPU address
typedef struct
{
kernel-open/nvidia-uvm/uvm_hmm.c
+18 -2
View File
@@ -1908,6 +1908,7 @@ static NV_STATUS gpu_chunk_add(uvm_va_block_t *va_block,
uvm_page_mask_set(&gpu_state->resident, page_index);
gpu_chunk->va_block = va_block;
gpu_chunk->va_space = va_block->hmm.va_space;
gpu_state->chunks[page_index] = gpu_chunk;
@@ -3160,7 +3161,11 @@ NV_STATUS uvm_hmm_va_block_service_locked(uvm_gpu_t *gpu,
args->fault_page = NULL;
ret = migrate_vma_setup_locked(args, va_block);
UVM_ASSERT(!ret);
if (ret < 0) {
UVM_ASSERT(0);
gpu_chunk_free_preallocated(va_block, va_block_retry);
return errno_to_nv_status(ret);
}
status = uvm_hmm_gpu_fault_alloc_and_copy(&uvm_hmm_gpu_fault_event);
if (status == NV_WARN_MORE_PROCESSING_REQUIRED) {
@@ -3540,6 +3545,13 @@ NV_STATUS uvm_hmm_va_block_evict_chunk_prep(uvm_va_block_t *va_block,
return NV_OK;
}
void uvm_hmm_va_block_evict_chunk_cancel(uvm_va_block_context_t *block_context)
{
migrate_device_finalize(block_context->hmm.src_pfns,
block_context->hmm.dst_pfns,
ARRAY_SIZE(block_context->hmm.src_pfns));
}
// Note that the caller must initialize va_block_context->hmm.src_pfns by
// calling uvm_hmm_va_block_evict_chunk_prep() before calling this.
static NV_STATUS hmm_va_block_evict_chunks(uvm_va_block_t *va_block,
@@ -3660,6 +3672,7 @@ NV_STATUS uvm_hmm_va_block_evict_pages_from_gpu(uvm_va_block_t *va_block,
{
uvm_va_block_context_t *block_context = service_context->block_context;
unsigned long *src_pfns = block_context->hmm.src_pfns;
unsigned long *dst_pfns = block_context->hmm.dst_pfns;
uvm_va_block_gpu_state_t *gpu_state;
uvm_page_index_t page_index;
uvm_gpu_chunk_t *gpu_chunk;
@@ -3673,6 +3686,7 @@ NV_STATUS uvm_hmm_va_block_evict_pages_from_gpu(uvm_va_block_t *va_block,
// Fill in the src_pfns[] with the ZONE_DEVICE private PFNs of the GPU.
memset(src_pfns, 0, sizeof(block_context->hmm.src_pfns));
memset(dst_pfns, 0, sizeof(block_context->hmm.dst_pfns));
// TODO: Bug 3368756: add support for large GPU pages.
for_each_va_block_page_in_region_mask(page_index, pages_to_evict, region) {
@@ -3683,8 +3697,10 @@ NV_STATUS uvm_hmm_va_block_evict_pages_from_gpu(uvm_va_block_t *va_block,
block_context,
gpu_chunk,
uvm_va_block_region_for_page(page_index));
if (status != NV_OK)
if (status != NV_OK) {
uvm_hmm_va_block_evict_chunk_cancel(block_context);
return status;
}
}
return hmm_va_block_evict_chunks(va_block,
kernel-open/nvidia-uvm/uvm_hmm.h
+16
View File
@@ -340,6 +340,9 @@ typedef struct
uvm_gpu_chunk_t *gpu_chunk,
uvm_va_block_region_t chunk_region);
// clears out any migration PTEs setup by uvm_hmm_va_block_evict_chunk_prep() above
void uvm_hmm_va_block_evict_chunk_cancel(uvm_va_block_context_t *block_context);
// Migrate pages to system memory for the given page mask. Note that the
// mmap lock is not held and there is no MM retained. This must be called
// after uvm_hmm_va_block_evict_chunk_prep() has initialized
@@ -632,6 +635,11 @@ typedef struct
return NV_OK;
}
static void uvm_hmm_va_block_evict_chunk_cancel(uvm_va_block_context_t *block_context)
{
return;
}
static NV_STATUS uvm_hmm_va_block_evict_chunks(uvm_va_block_t *va_block,
uvm_service_block_context_t *service_context,
const uvm_page_mask_t *pages_to_evict,
@@ -660,6 +668,14 @@ typedef struct
return NV_ERR_INVALID_ADDRESS;
}
static NV_STATUS uvm_hmm_va_block_update_residency_info(uvm_va_block_t *va_block,
struct mm_struct *mm,
NvU64 lookup_address,
bool populate)
{
return NV_ERR_INVALID_ADDRESS;
}
static NV_STATUS uvm_hmm_va_block_update_residency_info_unlocked(uvm_va_block_t *va_block,
struct mm_struct *mm,
NvU64 lookup_address,
kernel-open/nvidia-uvm/uvm_hopper.c
+1 -1
View File
@@ -80,7 +80,7 @@ void uvm_hal_hopper_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->access_bits_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
kernel-open/nvidia-uvm/uvm_hopper_ce.c
+12 -4
View File
@@ -60,10 +60,18 @@ static NvU32 ce_aperture(uvm_aperture_t aperture)
return HWCONST(C8B5, SET_SRC_PHYS_MODE, TARGET, LOCAL_FB);
}
else {
UVM_ASSERT(uvm_aperture_is_peer(aperture));
return HWCONST(C8B5, SET_SRC_PHYS_MODE, TARGET, PEERMEM) |
HWVALUE(C8B5, SET_SRC_PHYS_MODE, FLA, 0) |
HWVALUE(C8B5, SET_SRC_PHYS_MODE, PEER_ID, UVM_APERTURE_PEER_ID(aperture));
NvU32 aperture_bits;
UVM_ASSERT(uvm_aperture_is_peer(aperture) || uvm_aperture_is_egm(aperture));
aperture_bits = HWCONST(C8B5, SET_SRC_PHYS_MODE, TARGET, PEERMEM) | HWVALUE(C8B5, SET_SRC_PHYS_MODE, FLA, 0);
if (uvm_aperture_is_egm(aperture))
aperture_bits |= HWVALUE(C8B5, SET_SRC_PHYS_MODE, PEER_ID, UVM_APERTURE_EGM_PEER_ID(aperture));
else
aperture_bits |= HWVALUE(C8B5, SET_SRC_PHYS_MODE, PEER_ID, UVM_APERTURE_PEER_ID(aperture));
return aperture_bits;
}
}
kernel-open/nvidia-uvm/uvm_hopper_mmu.c
+4 -2
View File
@@ -189,7 +189,7 @@ static NvU64 make_pte_hopper(uvm_aperture_t aperture, NvU64 address, uvm_prot_t
aperture_bits = NV_MMU_VER3_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_VID)
aperture_bits = NV_MMU_VER3_PTE_APERTURE_VIDEO_MEMORY;
else if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7)
else if (uvm_aperture_is_peer(aperture) || uvm_aperture_is_egm(aperture))
aperture_bits = NV_MMU_VER3_PTE_APERTURE_PEER_MEMORY;
else
UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", aperture);
@@ -213,8 +213,10 @@ static NvU64 make_pte_hopper(uvm_aperture_t aperture, NvU64 address, uvm_prot_t
pte_bits |= HWVALUE64(_MMU_VER3, PTE, ADDRESS, address);
// peer id 63:61
if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7)
if (uvm_aperture_is_peer(aperture))
pte_bits |= HWVALUE64(_MMU_VER3, PTE, PEER_ID, UVM_APERTURE_PEER_ID(aperture));
else if (uvm_aperture_is_egm(aperture))
pte_bits |= HWVALUE64(_MMU_VER3, PTE, PEER_ID, UVM_APERTURE_EGM_PEER_ID(aperture));
}
return pte_bits;
kernel-open/nvidia-uvm/uvm_ioctl.h
+46
View File
@@ -955,6 +955,52 @@ typedef struct
NV_STATUS rmStatus; // OUT
} UVM_QUERY_RESIDENCY_PARAMS;
//
// UvmIsDmaBufImportSupported
//
#define UVM_IS_DMA_BUF_IMPORT_SUPPORTED UVM_IOCTL_BASE(82)
typedef struct
{
NvBool dmaBufImportSupported; // OUT
NV_STATUS rmStatus; // OUT
} UVM_IS_DMA_BUF_IMPORT_SUPPORTED_PARAMS;
//
// UvmImportDmaBuf
//
#define UVM_IMPORT_DMA_BUF UVM_IOCTL_BASE(83)
typedef struct
{
NvU64 base NV_ALIGN_BYTES(8); // IN
NvU64 length NV_ALIGN_BYTES(8); // IN
NvS32 handleFd; // IN
NV_STATUS rmStatus; // OUT
} UVM_IMPORT_DMA_BUF_PARAMS;
//
// UvmMapDmaBuf
//
#define UVM_MAP_DMA_BUF UVM_IOCTL_BASE(84)
typedef struct
{
NvU64 base NV_ALIGN_BYTES(8); // IN
NvU64 length NV_ALIGN_BYTES(8); // IN
NvProcessorUuid gpuUuid; // IN
NV_STATUS rmStatus; // OUT
} UVM_MAP_DMA_BUF_PARAMS;
//
// UvmUnmapDmaBuf
//
#define UVM_UNMAP_DMA_BUF UVM_IOCTL_BASE(85)
typedef struct
{
NvU64 base NV_ALIGN_BYTES(8); // IN
NvU64 length NV_ALIGN_BYTES(8); // IN
NvProcessorUuid gpuUuid; // IN
NV_STATUS rmStatus; // OUT
} UVM_UNMAP_DMA_BUF_PARAMS;
//
// Temporary ioctls which should be removed before UVM 8 release
// Number backwards from 2047 - highest custom ioctl function number
kernel-open/nvidia-uvm/uvm_linux.h
+18
View File
@@ -124,6 +124,18 @@
#define UVM_CAN_USE_MMU_NOTIFIERS() 0
#endif
// Check that importers can provide an invalidate_mappings() callback to
// determine that DMA-BUF API stability has been reached and whether to provide
// importer support through UVM.
//
// This callback was previously called move_notify().
#if (defined(NV_DMA_BUF_ATTACH_OPS_HAS_INVALIDATE_MAPPINGS) || defined(NV_DMA_BUF_ATTACH_OPS_HAS_MOVE_NOTIFY)) && \
defined(NV_DMA_BUF_ATTACHMENT_HAS_PEER2PEER)
#define UVM_PROVIDES_DMA_BUF_IMPORTER() 1
#else
#define UVM_PROVIDES_DMA_BUF_IMPORTER() 0
#endif
//
// printk.h already defined pr_fmt, so we have to redefine it so the pr_*
// routines pick up our version
@@ -195,6 +207,12 @@ typedef struct
#include <asm/pgtable_types.h>
#endif
#if NV_IS_EXPORT_SYMBOL_GPL_dma_iova_try_alloc
#define UVM_USE_DMA_IOVA_API() 1
#else
#define UVM_USE_DMA_IOVA_API() 0
#endif // NV_IS_EXPORT_SYMBOL_GPL_dma_iova_try_alloc
// PAGE_KERNEL_NOENC is only defined on x86. Define it for all architectures so
// we don't have to wrap uses in #ifdefs.
#if !defined(PAGE_KERNEL_NOENC)
kernel-open/nvidia-uvm/uvm_lock.c
+3 -1
View File
@@ -27,7 +27,7 @@
const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
{
BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 38);
BUILD_BUG_ON(UVM_LOCK_ORDER_COUNT != 40);
switch (lock_order) {
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_INVALID);
@@ -37,10 +37,12 @@ const char *uvm_lock_order_to_string(uvm_lock_order_t lock_order)
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_ISR);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_MMAP_LOCK);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_VA_SPACES_LIST);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_VA_SPACE_SERIALIZE_WRITERS);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_VA_SPACE_READ_ACQUIRE_WRITE_RELEASE_LOCK);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_VA_SPACE);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_EXT_RANGE_TREE);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_DMA_BUF_RANGE_TREE);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_GPU_SEMAPHORE_POOL);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_RM_API);
UVM_ENUM_STRING_CASE(UVM_LOCK_ORDER_RM_GPUS);
kernel-open/nvidia-uvm/uvm_lock.h
+46
View File
@@ -139,6 +139,20 @@
// UVM driver calls into RM. In other words, mmap_lock and the RM GPUs lock
// are mutually exclusive.
//
// - DMA-BUF resv lock (dma_resv)
// Order: UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK
// Wound-wait mutex (ww_mutex)
// Exclusive lock per exported DMA-BUF.
//
// Imported DMA-BUF resources by UVM are tracked by VA ranges. Multiple
// such ranges may refer to a single resource obtained from an exporter.
//
// Exporters may issue the invalidate_mappings() callback to any attached
// GPU. The callback holds the dma_resv lock, though the callback must
// first clear all GMMU mappings before it is safe to tear down dma all
// mappings with dma_buf_unmap_attachment under this lock. Thus this lock
// should be taken before va_space or subsequent locks.
//
// - Global VA spaces list lock
// Order: UVM_LOCK_ORDER_VA_SPACES_LIST
// Mutex which protects g_uvm_global.va_spaces state.
@@ -251,6 +265,12 @@
//
// Protects the per-GPU sub-range tree mappings in each external VA range.
//
// - DMA-BUF Allocation Tree lock
// Order: UVM_LOCK_ORDER_DMA_BUF_RANGE_TREE
// Exclusive lock (mutex) per DMA-BUF VA range, per GPU.
//
// Protects the per-GPU sub-range tree mappings in each DMA-BUF VA range.
//
// - GPU semaphore pool lock (semaphore_pool->mutex)
// Order: UVM_LOCK_ORDER_GPU_SEMAPHORE_POOL
// Exclusive lock (mutex) per uvm_gpu_semaphore_pool
@@ -507,10 +527,12 @@ typedef enum
UVM_LOCK_ORDER_ISR,
UVM_LOCK_ORDER_MMAP_LOCK,
UVM_LOCK_ORDER_VA_SPACES_LIST,
UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK,
UVM_LOCK_ORDER_VA_SPACE_SERIALIZE_WRITERS,
UVM_LOCK_ORDER_VA_SPACE_READ_ACQUIRE_WRITE_RELEASE_LOCK,
UVM_LOCK_ORDER_VA_SPACE,
UVM_LOCK_ORDER_EXT_RANGE_TREE,
UVM_LOCK_ORDER_DMA_BUF_RANGE_TREE,
UVM_LOCK_ORDER_GPU_SEMAPHORE_POOL,
UVM_LOCK_ORDER_RM_API,
UVM_LOCK_ORDER_RM_GPUS,
@@ -643,6 +665,14 @@ bool __uvm_locking_initialized(void);
#define uvm_check_locked_mmap_lock(mm, flags) \
__uvm_check_locked(nv_mmap_get_lock(mm), UVM_LOCK_ORDER_MMAP_LOCK, (flags))
// Helpers for recording DMA-BUF lock usage.
#define uvm_record_lock_dma_resv() \
uvm_record_lock_raw((void*)UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK, UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK, \
UVM_LOCK_FLAGS_MODE_EXCLUSIVE)
#define uvm_record_unlock_dma_resv() \
uvm_record_unlock_raw((void*)UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK, UVM_LOCK_ORDER_DMA_BUF_RESV_LOCK, \
UVM_LOCK_FLAGS_MODE_EXCLUSIVE)
// Helpers for recording RM API lock usage around UVM-RM interfaces
#define uvm_record_lock_rm_api() \
uvm_record_lock_raw((void*)UVM_LOCK_ORDER_RM_API, UVM_LOCK_ORDER_RM_API, \
@@ -683,6 +713,9 @@ bool __uvm_locking_initialized(void);
#define uvm_check_locked_mmap_lock uvm_check_locked
#define uvm_record_lock_dma_resv()
#define uvm_record_unlock_dma_resv()
#define uvm_record_lock_rm_api()
#define uvm_record_unlock_rm_api()
@@ -751,6 +784,19 @@ bool __uvm_locking_initialized(void);
uvm_record_unlock_mmap_lock_write(_mm); \
})
// Helpers for locking dma_resv lock and recording its usage.
#define uvm_dma_resv_lock(resv) ({ \
typeof(resv) _resv = (resv); \
uvm_record_lock_dma_resv(); \
dma_resv_lock(_resv, NULL); \
})
#define uvm_dma_resv_unlock(resv) ({ \
typeof(resv) _resv = (resv); \
dma_resv_unlock(_resv); \
uvm_record_unlock_dma_resv(); \
})
// Helper for calling a UVM-RM interface function with lock recording
#define uvm_rm_locked_call(call) ({ \
typeof(call) ret; \
kernel-open/nvidia-uvm/uvm_map_external.c
+38 -36
View File
@@ -370,10 +370,10 @@ static uvm_membar_t va_range_downgrade_membar(uvm_va_range_t *va_range, uvm_ext_
return UVM_MEMBAR_GPU;
// EGM uses the same barriers as sysmem.
return uvm_hal_downgrade_membar_type(ext_gpu_map->gpu,
return uvm_hal_downgrade_membar_type(ext_gpu_map->base.gpu,
!ext_gpu_map->is_sysmem &&
!ext_gpu_map->is_fabricmem &&
ext_gpu_map->gpu == ext_gpu_map->owning_gpu);
ext_gpu_map->base.gpu == ext_gpu_map->owning_gpu);
}
NV_STATUS uvm_va_range_map_rm_allocation(uvm_va_range_t *va_range,
@@ -423,8 +423,8 @@ NV_STATUS uvm_va_range_map_rm_allocation(uvm_va_range_t *va_range,
// We should be never called with ext_gpu_map == NULL
// and UVM_VA_RANGE_TYPE_EXTERNAL
UVM_ASSERT(ext_gpu_map != NULL);
node = &ext_gpu_map->node;
pt_range_vec = &ext_gpu_map->pt_range_vec;
node = &ext_gpu_map->base.node;
pt_range_vec = &ext_gpu_map->base.pt_range_vec;
}
else {
node = &va_range->node;
@@ -700,7 +700,7 @@ static uvm_ext_gpu_map_t *uvm_va_range_ext_gpu_map(uvm_va_range_external_t *exte
node = uvm_range_tree_find(&range_tree->tree, addr);
if (node) {
ext_gpu_map = uvm_ext_gpu_map_container(node);
UVM_ASSERT(ext_gpu_map->gpu == mapping_gpu);
UVM_ASSERT(ext_gpu_map->base.gpu == mapping_gpu);
}
}
else {
@@ -719,18 +719,18 @@ static NV_STATUS uvm_ext_gpu_map_split(uvm_range_tree_t *tree,
NV_STATUS status;
NvU64 new_start = new_end + 1;
if (!IS_ALIGNED(new_start, existing_map->pt_range_vec.page_size))
if (!IS_ALIGNED(new_start, existing_map->base.pt_range_vec.page_size))
return NV_ERR_INVALID_ADDRESS;
UVM_ASSERT(new_start >= existing_map->node.start && new_start < existing_map->node.end);
UVM_ASSERT(new_start >= existing_map->base.node.start && new_start < existing_map->base.node.end);
new = uvm_kvmalloc_zero(sizeof(*new));
if (!new)
return NV_ERR_NO_MEMORY;
RB_CLEAR_NODE(&new->node.rb_node);
RB_CLEAR_NODE(&new->base.node.rb_node);
new->mem_handle = existing_map->mem_handle;
new->gpu = existing_map->gpu;
new->base.gpu = existing_map->base.gpu;
new->owning_gpu = existing_map->owning_gpu;
new->is_sysmem = existing_map->is_sysmem;
new->is_egm = existing_map->is_egm;
@@ -745,20 +745,22 @@ static NV_STATUS uvm_ext_gpu_map_split(uvm_range_tree_t *tree,
return status;
}
status = uvm_page_table_range_vec_split_upper(&existing_map->pt_range_vec, new_start - 1, &new->pt_range_vec);
status = uvm_page_table_range_vec_split_upper(&existing_map->base.pt_range_vec,
new_start - 1,
&new->base.pt_range_vec);
if (status != NV_OK) {
uvm_tracker_deinit(&new->tracker);
uvm_kvfree(new);
return status;
}
new->node.start = new_start;
new->base.node.start = new_start;
// Sparse mappings don't have actual allocations.
if (new->mem_handle)
nv_kref_get(&new->mem_handle->ref_count);
uvm_range_tree_split(tree, &existing_map->node, &new->node);
uvm_range_tree_split(tree, &existing_map->base.node, &new->base.node);
if (new_map)
*new_map = new;
@@ -822,13 +824,13 @@ static NV_STATUS uvm_unmap_external_in_range(uvm_va_range_external_t *external_r
// newly created uvm_ext_gpu_map_t.
ext_map = uvm_ext_gpu_map_iter_first(external_range, gpu, start, end);
while (ext_map) {
if (start > ext_map->node.start) {
if (start > ext_map->base.node.start) {
status = uvm_ext_gpu_map_split(&range_tree->tree, ext_map, start - 1, &ext_map_next);
if (status != NV_OK)
break;
}
else {
if (end < ext_map->node.end) {
if (end < ext_map->base.node.end) {
status = uvm_ext_gpu_map_split(&range_tree->tree, ext_map, end, NULL);
if (status != NV_OK)
break;
@@ -886,9 +888,9 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_external_t *ext
// Insert the ext_gpu_map into the external range immediately since some of
// the below calls require it to be there.
ext_gpu_map->node.start = base;
ext_gpu_map->node.end = base + length - 1;
RB_CLEAR_NODE(&ext_gpu_map->node.rb_node);
ext_gpu_map->base.node.start = base;
ext_gpu_map->base.node.end = base + length - 1;
RB_CLEAR_NODE(&ext_gpu_map->base.node.rb_node);
uvm_tracker_init(&ext_gpu_map->tracker);
ext_gpu_map->mem_handle = uvm_kvmalloc_zero(sizeof(*ext_gpu_map->mem_handle));
if (!ext_gpu_map->mem_handle) {
@@ -898,11 +900,11 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_external_t *ext
// Due to the fact that any overlapping mappings were already unmapped,
// adding the new mapping to the tree cannot fail.
status = uvm_range_tree_add(&range_tree->tree, &ext_gpu_map->node);
status = uvm_range_tree_add(&range_tree->tree, &ext_gpu_map->base.node);
UVM_ASSERT(status == NV_OK);
uvm_processor_mask_set_atomic(&external_range->mapped_gpus, mapping_gpu->id);
ext_gpu_map->gpu = mapping_gpu;
ext_gpu_map->base.gpu = mapping_gpu;
ext_gpu_map->mem_handle->gpu = mapping_gpu;
nv_kref_init(&ext_gpu_map->mem_handle->ref_count);
@@ -952,7 +954,7 @@ static NV_STATUS uvm_map_external_allocation_on_gpu(uvm_va_range_external_t *ext
// allocations, the vMMU segment size may limit the range of page sizes.
biggest_mapping_page_size = uvm_mmu_biggest_page_size_up_to(&gpu_va_space->page_tables,
mapping_gpu->mem_info.max_vidmem_page_size);
if (!ext_gpu_map->is_sysmem && (ext_gpu_map->gpu == ext_gpu_map->owning_gpu) &&
if (!ext_gpu_map->is_sysmem && (ext_gpu_map->base.gpu == ext_gpu_map->owning_gpu) &&
(mapping_page_size > biggest_mapping_page_size))
mapping_page_size = biggest_mapping_page_size;
}
@@ -1135,31 +1137,31 @@ static NV_STATUS uvm_map_external_sparse_on_gpu(uvm_va_range_external_t *externa
goto error;
}
ext_gpu_map->node.start = base;
ext_gpu_map->node.end = base + length - 1;
RB_CLEAR_NODE(&ext_gpu_map->node.rb_node);
ext_gpu_map->base.node.start = base;
ext_gpu_map->base.node.end = base + length - 1;
RB_CLEAR_NODE(&ext_gpu_map->base.node.rb_node);
uvm_tracker_init(&ext_gpu_map->tracker);
// Due to the fact that any overlapping mappings were already unmapped,
// adding the new mapping to the tree cannot fail.
status = uvm_range_tree_add(&range_tree->tree, &ext_gpu_map->node);
status = uvm_range_tree_add(&range_tree->tree, &ext_gpu_map->base.node);
UVM_ASSERT(status == NV_OK);
uvm_processor_mask_set_atomic(&external_range->mapped_gpus, mapping_gpu->id);
ext_gpu_map->gpu = mapping_gpu;
ext_gpu_map->base.gpu = mapping_gpu;
UVM_ASSERT(uvm_va_range_ext_gpu_map(external_range, mapping_gpu, base) == ext_gpu_map);
status = uvm_page_table_range_vec_init(page_tree,
ext_gpu_map->node.start,
uvm_range_tree_node_size(&ext_gpu_map->node),
ext_gpu_map->base.node.start,
uvm_range_tree_node_size(&ext_gpu_map->base.node),
UVM_PAGE_SIZE_64K,
UVM_PMM_ALLOC_FLAGS_EVICT,
&ext_gpu_map->pt_range_vec);
&ext_gpu_map->base.pt_range_vec);
if (status != NV_OK)
goto error;
status = uvm_page_table_range_vec_write_ptes(&ext_gpu_map->pt_range_vec,
status = uvm_page_table_range_vec_write_ptes(&ext_gpu_map->base.pt_range_vec,
UVM_MEMBAR_NONE,
external_sparse_pte_maker,
NULL);
@@ -1233,7 +1235,7 @@ static uvm_gpu_t *uvm_ext_gpu_map_free_internal(uvm_ext_gpu_map_t *ext_gpu_map)
if (!ext_gpu_map)
return NULL;
UVM_ASSERT(!ext_gpu_map->pt_range_vec.ranges);
UVM_ASSERT(!ext_gpu_map->base.pt_range_vec.ranges);
if (ext_gpu_map->mem_handle)
nv_kref_put(&ext_gpu_map->mem_handle->ref_count, uvm_release_rm_handle);
@@ -1267,13 +1269,13 @@ void uvm_ext_gpu_map_destroy(uvm_va_range_external_t *external_range,
// The external map is inserted into the tree prior to the rest of the mapping
// steps. So, if it has not been inserted yet, there is nothing to clean up. Just
// free the memory.
if (RB_EMPTY_NODE(&ext_gpu_map->node.rb_node)) {
if (RB_EMPTY_NODE(&ext_gpu_map->base.node.rb_node)) {
uvm_kvfree(ext_gpu_map->mem_handle);
uvm_kvfree(ext_gpu_map);
return;
}
mapped_gpu = ext_gpu_map->gpu;
mapped_gpu = ext_gpu_map->base.gpu;
range_tree = uvm_ext_gpu_range_tree(external_range, mapped_gpu);
@@ -1297,13 +1299,13 @@ void uvm_ext_gpu_map_destroy(uvm_va_range_external_t *external_range,
uvm_assert_mutex_locked(&range_tree->lock);
UVM_ASSERT(uvm_gpu_va_space_get(external_range->va_range.va_space, mapped_gpu));
uvm_range_tree_remove(&range_tree->tree, &ext_gpu_map->node);
uvm_range_tree_remove(&range_tree->tree, &ext_gpu_map->base.node);
// Unmap the PTEs
if (ext_gpu_map->pt_range_vec.ranges) {
if (ext_gpu_map->base.pt_range_vec.ranges) {
membar = va_range_downgrade_membar(&external_range->va_range, ext_gpu_map);
uvm_page_table_range_vec_clear_ptes(&ext_gpu_map->pt_range_vec, membar);
uvm_page_table_range_vec_deinit(&ext_gpu_map->pt_range_vec);
uvm_page_table_range_vec_clear_ptes(&ext_gpu_map->base.pt_range_vec, membar);
uvm_page_table_range_vec_deinit(&ext_gpu_map->base.pt_range_vec);
}
if (deferred_free_list && ext_gpu_map->mem_handle) {
kernel-open/nvidia-uvm/uvm_map_external.h
+25 -29
View File
@@ -40,7 +40,7 @@ typedef struct
UvmGpuCompressionType compression_type;
} uvm_map_rm_params_t;
static uvm_ext_gpu_range_tree_t *uvm_ext_gpu_range_tree(uvm_va_range_external_t *external_range, uvm_gpu_t *gpu)
static uvm_gpu_range_tree_t *uvm_ext_gpu_range_tree(uvm_va_range_external_t *external_range, uvm_gpu_t *gpu)
{
return &external_range->gpu_ranges[uvm_id_gpu_index(gpu->id)];
}
@@ -52,50 +52,46 @@ static uvm_ext_gpu_map_t *uvm_ext_gpu_map_iter_first(uvm_va_range_external_t *ex
NvU64 start,
NvU64 end)
{
uvm_ext_gpu_range_tree_t *range_tree;
uvm_range_tree_node_t *node;
UVM_ASSERT(start >= external_range->va_range.node.start);
UVM_ASSERT(end <= external_range->va_range.node.end);
range_tree = uvm_ext_gpu_range_tree(external_range, gpu);
node = uvm_range_tree_iter_first(&range_tree->tree, start, end);
return uvm_ext_gpu_map_container(node);
return uvm_ext_gpu_map_container(
uvm_gpu_range_tree_iter_first(uvm_ext_gpu_range_tree(external_range, gpu), start, end));
}
// Returns the external map following the provided map (if any) in address order from
// the gpu's range tree.
// The caller must hold the range tree lock.
// Returns the external map following the provided map (if any) in address order
// from the gpu's range tree. The caller must hold the range tree lock.
static uvm_ext_gpu_map_t *uvm_ext_gpu_map_iter_next(uvm_va_range_external_t *external_range,
uvm_ext_gpu_map_t *ext_gpu_map,
NvU64 end)
{
uvm_ext_gpu_range_tree_t *range_tree;
uvm_range_tree_node_t *node;
if (!ext_gpu_map)
return NULL;
UVM_ASSERT(end <= external_range->va_range.node.end);
range_tree = uvm_ext_gpu_range_tree(external_range, ext_gpu_map->gpu);
node = uvm_range_tree_iter_next(&range_tree->tree, &ext_gpu_map->node, end);
return uvm_ext_gpu_map_container(node);
return uvm_ext_gpu_map_container(
uvm_gpu_range_tree_iter_next(uvm_ext_gpu_range_tree(external_range, ext_gpu_map->base.gpu),
&ext_gpu_map->base, end));
}
// The four iterators below require that the caller hold the gpu's range tree
// lock.
#define uvm_ext_gpu_map_for_each_in(ext_gpu_map, external_range, gpu, start, end) \
for ((ext_gpu_map) = uvm_ext_gpu_map_iter_first((external_range), (gpu), (start), (end)); \
(ext_gpu_map); \
(ext_gpu_map) = uvm_ext_gpu_map_iter_next((external_range), (ext_gpu_map), (end)))
#define uvm_ext_gpu_map_for_each_in(ext_gpu_map, external_range, gpu, start, end) \
uvm_gpu_map_for_each_in(ext_gpu_map, \
external_range, \
gpu, \
start, \
end, \
uvm_ext_gpu_map_iter_first, \
uvm_ext_gpu_map_iter_next)
#define uvm_ext_gpu_map_for_each_in_safe(ext_gpu_map, ext_gpu_map_next, external_range, gpu, start, end) \
for ((ext_gpu_map) = uvm_ext_gpu_map_iter_first((external_range), (gpu), (start), (end)), \
(ext_gpu_map_next) = uvm_ext_gpu_map_iter_next((external_range), (ext_gpu_map), (end)); \
(ext_gpu_map); \
(ext_gpu_map) = (ext_gpu_map_next), \
(ext_gpu_map_next) = uvm_ext_gpu_map_iter_next((external_range), (ext_gpu_map), (end)))
#define uvm_ext_gpu_map_for_each_in_safe(ext_gpu_map, ext_gpu_map_next, external_range, gpu, start, end) \
uvm_gpu_map_for_each_in_safe(ext_gpu_map, \
ext_gpu_map_next, \
external_range, \
gpu, \
start, \
end, \
uvm_ext_gpu_map_iter_first, \
uvm_ext_gpu_map_iter_next)
#define uvm_ext_gpu_map_for_each(ext_gpu_map, external_range, gpu) \
uvm_ext_gpu_map_for_each_in(ext_gpu_map, \
kernel-open/nvidia-uvm/uvm_migrate_pageable.c
+60 -26
View File
@@ -303,7 +303,7 @@ static void uvm_migrate_vma_state_compute_masks(struct vm_area_struct *vma,
}
}
static struct page *uvm_migrate_vma_alloc_page(migrate_vma_state_t *state)
static struct page *uvm_migrate_vma_alloc_dst_page(migrate_vma_state_t *state)
{
struct page *dst_page;
uvm_migrate_args_t *uvm_migrate_args = state->uvm_migrate_args;
@@ -461,7 +461,7 @@ static NV_STATUS uvm_migrate_vma_populate_anon_pages(struct vm_area_struct *vma,
{
uvm_push_t push;
unsigned long i;
uvm_gpu_t *copying_gpu;
uvm_gpu_t *scrubbing_gpu;
NV_STATUS status = NV_OK;
uvm_migrate_args_t *uvm_migrate_args = state->uvm_migrate_args;
uvm_processor_id_t dst_id = uvm_migrate_args->dst_id;
@@ -475,25 +475,12 @@ static NV_STATUS uvm_migrate_vma_populate_anon_pages(struct vm_area_struct *vma,
UVM_ASSERT(state->num_populate_anon_pages == bitmap_weight(page_mask, state->num_pages));
// Try to get a GPU attached to the node being populated. If there
// is none, use any of the GPUs registered in the VA space.
if (UVM_ID_IS_CPU(dst_id)) {
copying_gpu = uvm_va_space_find_first_gpu_attached_to_cpu_node(va_space, uvm_migrate_args->dst_node_id);
if (!copying_gpu)
copying_gpu = uvm_va_space_find_first_gpu(va_space);
}
else {
copying_gpu = uvm_gpu_get(dst_id);
}
UVM_ASSERT(copying_gpu);
// Pre-allocate the dst pages and mark the ones that failed
for_each_set_bit(i, page_mask, state->num_pages) {
struct page *dst_page = NULL;
if (!state->out_of_memory)
dst_page = uvm_migrate_vma_alloc_page(state);
dst_page = uvm_migrate_vma_alloc_dst_page(state);
if (!dst_page) {
__set_bit(i, state->allocation_failed_mask.page_mask);
@@ -504,13 +491,25 @@ static NV_STATUS uvm_migrate_vma_populate_anon_pages(struct vm_area_struct *vma,
dst[i] = migrate_pfn(page_to_pfn(dst_page));
}
if (uvm_dma_mapping_required_on_copying_gpu(va_space, dst_id, copying_gpu)) {
status = dma_map_non_failed_pages_in_mask(copying_gpu, uvm_sgt, dst, page_mask, state);
// Try to get a GPU attached to the node being populated. If there
// is none, use any of the GPUs registered in the VA space.
if (UVM_ID_IS_CPU(dst_id)) {
scrubbing_gpu = uvm_va_space_find_first_gpu_attached_to_cpu_node(va_space, uvm_migrate_args->dst_node_id);
if (!scrubbing_gpu)
scrubbing_gpu = uvm_va_space_find_first_gpu(va_space);
UVM_ASSERT(uvm_dma_mapping_required_on_copying_gpu(va_space, dst_id, scrubbing_gpu));
status = dma_map_non_failed_pages_in_mask(scrubbing_gpu, uvm_sgt, dst, page_mask, state);
if (status != NV_OK)
return status;
}
else {
scrubbing_gpu = uvm_gpu_get(dst_id);
}
status = migrate_vma_zero_begin_push(va_space, dst_id, copying_gpu, start, outer - 1, &push);
UVM_ASSERT(scrubbing_gpu);
status = migrate_vma_zero_begin_push(va_space, dst_id, scrubbing_gpu, start, outer - 1, &push);
if (status != NV_OK)
return status;
@@ -591,15 +590,20 @@ static void copy_dma_mapped_pages(uvm_va_space_t *va_space,
if (src_has_dma_mappings) {
uvm_processor_id_t dst_id = uvm_migrate_args->dst_id;
// Copying from sysmem should always be to local vidmem
UVM_ASSERT(UVM_ID_IS_GPU(dst_id));
UVM_ASSERT(uvm_id_equal(dst_id, gpu->id));
UVM_ASSERT(dst[i] & MIGRATE_PFN_VALID);
page = migrate_pfn_to_page(dst[i]);
gpu_addr = uvm_migrate_vma_page_copy_address(page, i, dst_id, gpu, state);
gpu->parent->ce_hal->memcopy(push, gpu_addr, gpu_dma_addr, PAGE_SIZE);
}
else {
UVM_ASSERT(UVM_ID_IS_GPU(src_id));
// Copying to sysmem should always be from local vidmem
UVM_ASSERT(uvm_id_equal(src_id, gpu->id));
UVM_ASSERT(src[i] & MIGRATE_PFN_VALID);
page = migrate_pfn_to_page(src[i]);
gpu_addr = uvm_migrate_vma_page_copy_address(page, i, src_id, gpu, state);
gpu->parent->ce_hal->memcopy(push, gpu_dma_addr, gpu_addr, PAGE_SIZE);
@@ -679,7 +683,8 @@ static void copy_pages_in_mask(uvm_va_space_t *va_space,
// selection of copying_gpu changes based on the source nid, scatter gather
// table management might need to be changed since a source processor could
// potentially be accessed by multiple GPUs.
static uvm_gpu_t *select_gpu_for_vma_copy_push(uvm_processor_id_t dst_id,
static uvm_gpu_t *select_gpu_for_vma_copy_push(uvm_va_space_t *va_space,
uvm_processor_id_t dst_id,
uvm_processor_id_t src_id,
uvm_channel_type_t *out_channel_type)
{
@@ -695,7 +700,14 @@ static uvm_gpu_t *select_gpu_for_vma_copy_push(uvm_processor_id_t dst_id,
} else {
// Prefer to "push" the data from the source for GPU to GPU copies
gpu = uvm_gpu_get(src_id);
channel_type = UVM_CHANNEL_TYPE_GPU_TO_GPU;
if (uvm_processor_mask_test(&va_space->can_copy_from[uvm_id_value(src_id)], dst_id)) {
// Use GPU_TO_GPU if we can do peer copies
channel_type = UVM_CHANNEL_TYPE_GPU_TO_GPU;
}
else {
// Use GPU_TO_CPU if we need to use cLinks
channel_type = UVM_CHANNEL_TYPE_GPU_TO_CPU;
}
}
if (out_channel_type)
@@ -725,6 +737,7 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma,
uvm_va_space_t *va_space = uvm_migrate_args->va_space;
unsigned long *page_mask;
uvm_sgt_t *uvm_sgt = uvm_select_sgt(src_id, src_nid, state);
bool use_clinks_for_p2p = false;
uvm_tracker_t zero_tracker = UVM_TRACKER_INIT();
@@ -735,14 +748,14 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma,
UVM_ASSERT(!bitmap_empty(page_mask, state->num_pages));
copying_gpu = select_gpu_for_vma_copy_push(dst_id, src_id, &channel_type);
copying_gpu = select_gpu_for_vma_copy_push(va_space, dst_id, src_id, &channel_type);
// Pre-allocate the dst pages and mark the ones that failed
for_each_set_bit(i, page_mask, state->num_pages) {
struct page *dst_page = NULL;
if (!state->out_of_memory)
dst_page = uvm_migrate_vma_alloc_page(state);
dst_page = uvm_migrate_vma_alloc_dst_page(state);
if (!dst_page) {
__set_bit(i, state->allocation_failed_mask.page_mask);
@@ -786,8 +799,11 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma,
if (bitmap_equal(page_mask, state->allocation_failed_mask.page_mask, state->num_pages))
return uvm_tracker_wait_deinit(&zero_tracker);
retry_p2p:
// We don't have a case where both src and dst use the SYS aperture.
// In other word, only one mapping for page index i is allowed.
// If one location is CPU the other location is always local vidmem on
// the copying GPU.
// In other words, only one mapping for page index i is allowed.
// In both cases, we're using the source processor scatterlist to host
// the pages because we cannot reuse the destination scatterlist among
// the different source processors.
@@ -796,7 +812,9 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma,
src_has_dma_mappings = (uvm_sgt->dma_count != 0);
}
else if (uvm_dma_mapping_required_on_copying_gpu(va_space, dst_id, copying_gpu)) {
// Create destination DMA mapping if we're using cLinks even if the
// destination is peer GPU.
else if (uvm_dma_mapping_required_on_copying_gpu(va_space, dst_id, copying_gpu) || use_clinks_for_p2p) {
status = dma_map_non_failed_pages_in_mask(copying_gpu, uvm_sgt, dst, page_mask, state);
dst_has_dma_mappings = (uvm_sgt->dma_count != 0);
}
@@ -807,6 +825,22 @@ static NV_STATUS uvm_migrate_vma_copy_pages_from(struct vm_area_struct *vma,
}
status = migrate_vma_copy_begin_push(va_space, copying_gpu, channel_type, dst_id, src_id, start, outer - 1, &push);
// If reserving a p2p channel slot failed with NV_ERR_BUSY_RETRY, nvlink
// is currently suspended. Retry the copy using cLinks instead of gLinks.
if (status == NV_ERR_BUSY_RETRY && UVM_ID_IS_GPU(src_id) && UVM_ID_IS_GPU(dst_id) && !use_clinks_for_p2p) {
UVM_ASSERT_MSG(!src_has_dma_mappings, "Copy %d -> %d should not have src dma mappings\n", src_id, dst_id);
UVM_ASSERT_MSG(!dst_has_dma_mappings, "Copy %d -> %d should not have dst dma mappings\n", src_id, dst_id);
UVM_ASSERT(uvm_id_equal(copying_gpu->id, src_id));
UVM_ASSERT(channel_type == UVM_CHANNEL_TYPE_GPU_TO_GPU);
// Using clinks treats peer vidmem as sysmem
channel_type = UVM_CHANNEL_TYPE_GPU_TO_CPU;
use_clinks_for_p2p = true;
goto retry_p2p;
}
if (status != NV_OK) {
uvm_tracker_wait_deinit(&zero_tracker);
return status;
kernel-open/nvidia-uvm/uvm_pmm_gpu.c
+99 -15
View File
@@ -773,7 +773,8 @@ static bool assert_chunk_mergeable(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
// uniform state. This only merges leaves, so none of the children can be in the
// split state themselves.
//
// The children need to be removed from any lists before the merge.
// Removes all children from any lists they are on under list_lock before
// freeing them.
//
// The merged chunk inherits the former state of its children.
static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
@@ -791,6 +792,15 @@ static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
// the subchunk state.
uvm_spin_lock(&pmm->list_lock);
// A concurrent devmem_page_free_gpu_chunk() may have added a sub-chunk to
// va_block_lazy_free The lazy-free kthread acquires pmm->lock before
// dequeuing, so it is blocked for now. Remove every sub-chunk from any list
// here, under list_lock, so that when the kthread runs after pmm->lock is
// released it finds nothing to dequeue and avoids any potential
// use-after-free.
for (i = 0; i < num_sub; i++)
list_del_init(&chunk->suballoc->subchunks[i]->list);
subchunk = chunk->suballoc->subchunks[0];
child_state = subchunk->state;
@@ -798,12 +808,14 @@ static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
UVM_ASSERT(subchunk->va_block);
chunk->va_block = subchunk->va_block;
chunk->va_space = subchunk->va_space;
chunk->is_referenced = subchunk->is_referenced;
}
else if (child_state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) {
UVM_ASSERT(root_chunk->chunk.suballoc->pinned_leaf_chunks >= num_sub);
root_chunk->chunk.suballoc->pinned_leaf_chunks += 1 - num_sub;
chunk->va_block = subchunk->va_block;
chunk->va_space = subchunk->va_space;
if (chunk_is_root_chunk(chunk))
++pmm->root_chunks.pinned_count;
}
@@ -824,10 +836,6 @@ static void merge_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
for (i = 0; i < num_sub; i++) {
subchunk = suballoc->subchunks[i];
// The subchunks should have been removed from their lists prior to the
// merge.
UVM_ASSERT(list_empty(&subchunk->list));
if ((child_state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) && uvm_gpu_chunk_is_user(subchunk))
UVM_ASSERT(subchunk->va_block);
@@ -1199,6 +1207,7 @@ void uvm_pmm_gpu_mark_chunk_evicted(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
UVM_ASSERT(chunk->va_block);
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk_pin(pmm, chunk);
uvm_spin_unlock(&pmm->list_lock);
@@ -2154,10 +2163,12 @@ NV_STATUS split_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
uvm_assert_mutex_locked(&chunk->va_block->lock);
subchunk->va_block = chunk->va_block;
subchunk->va_space = chunk->va_space;
subchunk->is_referenced = chunk->is_referenced;
}
else if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) {
subchunk->va_block = chunk->va_block;
subchunk->va_space = chunk->va_space;
}
}
@@ -2172,6 +2183,7 @@ NV_STATUS split_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_ALLOCATED) {
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk->is_referenced = false;
}
else if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED) {
@@ -2187,6 +2199,7 @@ NV_STATUS split_gpu_chunk(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
}
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk->is_referenced = false;
}
@@ -2275,6 +2288,7 @@ static void chunk_free_locked(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
UVM_ASSERT(chunk->va_block);
UVM_ASSERT(list_empty(&chunk->list));
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk->is_zero = false;
chunk_pin(pmm, chunk);
}
@@ -2292,6 +2306,7 @@ static void chunk_free_locked(uvm_pmm_gpu_t *pmm, uvm_gpu_chunk_t *chunk)
}
chunk->va_block = NULL;
chunk->va_space = NULL;
chunk->is_zero = false;
if (chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED)
@@ -2947,25 +2962,43 @@ cleanup:
void uvm_pmm_gpu_process_lazy_free(uvm_pmm_gpu_t *pmm)
{
uvm_gpu_chunk_t *chunk;
// Process entries one at a time, holding pmm->lock across each dequeue and
// free. This closes the race with every path that can call
// merge_gpu_chunk() on this chunk's parent while we hold a pointer to it.
while (1) {
uvm_gpu_chunk_t *chunk;
uvm_pmm_gpu_memory_type_t type;
bool free_root;
NvU32 test_delay_us = pmm->root_chunks.test_lazy_free_delay_us;
uvm_spin_lock(&pmm->list_lock);
uvm_mutex_lock(&pmm->lock);
uvm_spin_lock(&pmm->list_lock);
if (list_empty(&pmm->root_chunks.va_block_lazy_free)) {
uvm_spin_unlock(&pmm->list_lock);
uvm_mutex_unlock(&pmm->lock);
break;
}
// Note: We can't use list_for_each_safe_entry() because we drop the lock
// in the loop. Instead, just keep removing the first entry until the list
// is empty.
while (!list_empty(&pmm->root_chunks.va_block_lazy_free)) {
chunk = list_first_entry(&pmm->root_chunks.va_block_lazy_free, uvm_gpu_chunk_t, list);
UVM_ASSERT(chunk->state == UVM_PMM_GPU_CHUNK_STATE_TEMP_PINNED);
type = chunk->type;
free_root = chunk_is_root_chunk(chunk) || chunk_is_last_allocated_child(pmm, chunk);
list_del_init(&chunk->list);
uvm_spin_unlock(&pmm->list_lock);
free_chunk(pmm, chunk);
if (unlikely(test_delay_us))
usleep_range(test_delay_us, test_delay_us + 1000);
uvm_spin_lock(&pmm->list_lock);
free_chunk_with_merges(pmm, chunk);
uvm_mutex_unlock(&pmm->lock);
if (free_root)
free_next_available_root_chunk(pmm, type);
}
uvm_spin_unlock(&pmm->list_lock);
}
static void process_lazy_free_entry(void *args)
@@ -3564,3 +3597,54 @@ out:
uvm_va_space_up_read(va_space);
return status;
}
NV_STATUS uvm_test_pmm_lazy_free_set_delay(UVM_TEST_PMM_LAZY_FREE_SET_DELAY_PARAMS *params, struct file *filp)
{
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_gpu_t *gpu;
gpu = uvm_va_space_retain_gpu_by_uuid(va_space, &params->gpu_uuid);
if (!gpu)
return NV_ERR_INVALID_DEVICE;
gpu->pmm.root_chunks.test_lazy_free_delay_us = params->delay_us;
uvm_gpu_release(gpu);
return NV_OK;
}
NV_STATUS uvm_test_pmm_evict_temp_pinned(UVM_TEST_PMM_EVICT_TEMP_PINNED_PARAMS *params, struct file *filp)
{
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_gpu_t *gpu;
uvm_pmm_gpu_t *pmm;
size_t i;
gpu = uvm_va_space_retain_gpu_by_uuid(va_space, &params->gpu_uuid);
if (!gpu)
return NV_ERR_INVALID_DEVICE;
pmm = &gpu->pmm;
uvm_mutex_lock(&pmm->lock);
for (i = 0; i < pmm->root_chunks.count; i++) {
uvm_gpu_chunk_t *root = &pmm->root_chunks.array[i].chunk;
NV_STATUS walk_status;
if (root->state != UVM_PMM_GPU_CHUNK_STATE_IS_SPLIT)
continue;
do {
walk_status = chunk_walk_pre_order(pmm, root, free_first_pinned_chunk_func, NULL);
} while (walk_status == NV_ERR_MORE_DATA_AVAILABLE);
UVM_ASSERT(walk_status == NV_OK);
}
uvm_mutex_unlock(&pmm->lock);
uvm_gpu_release(gpu);
return NV_OK;
}
kernel-open/nvidia-uvm/uvm_pmm_gpu.h
+15
View File
@@ -292,6 +292,19 @@ struct uvm_gpu_chunk_struct
// This field is always NULL in kernel chunks.
uvm_va_block_t *va_block;
// The VA space owning the VA block using the chunk, if any. This caches
// uvm_va_block_get_va_space(va_block) (i.e. va_block->hmm.va_space for HMM
// blocks) and is kept in sync whenever va_block is assigned or cleared.
//
// It is needed because a device private (or device coherent) struct page
// can outlive its va_block: the Linux page_free() callback
// (devmem_page_free_gpu_chunk()) may run after the va_block has already been
// freed, so it must not dereference va_block to find the va_space. The
// va_space is guaranteed to outlive any device page referencing this chunk.
//
// This field is always NULL in kernel chunks.
uvm_va_space_t *va_space;
// If this is subchunk it points to the parent - in other words
// chunk of bigger size which contains this chunk.
uvm_gpu_chunk_t *parent;
@@ -344,6 +357,8 @@ typedef struct uvm_pmm_gpu_struct
struct list_head va_block_lazy_free;
nv_kthread_q_item_t va_block_lazy_free_q_item;
NvU32 test_lazy_free_delay_us;
// Count of the number of root chunks "in_eviction". Incremented for
// each root chunks that starts the eviction process, and decremented
// on eviction success or failure.
kernel-open/nvidia-uvm/uvm_pmm_sysmem.c
+117 -5
View File
@@ -27,6 +27,8 @@
#include "uvm_va_block.h"
#include "uvm_va_space.h"
#define DMA_ADDR_INVALID (~(dma_addr_t)0)
static int uvm_cpu_chunk_allocation_sizes = UVM_CPU_CHUNK_SIZES;
module_param(uvm_cpu_chunk_allocation_sizes, uint, S_IRUGO | S_IWUSR);
MODULE_PARM_DESC(uvm_cpu_chunk_allocation_sizes, "OR'ed value of all CPU chunk allocation sizes.");
@@ -164,6 +166,17 @@ static void chunk_inc_gpu_mapping(uvm_cpu_physical_chunk_t *chunk, uvm_parent_gp
mapping->map_count++;
}
static void cpu_chunk_unmap_physical_egm(uvm_cpu_physical_chunk_t *chunk, uvm_parent_gpu_t *gpu)
{
if (chunk->common.egm_dma_addr != DMA_ADDR_INVALID) {
UVM_ASSERT(uvm_parent_id_equal(chunk->egm_parent_id, gpu->id));
UVM_ASSERT(page_to_nid(chunk->common.page) == gpu->closest_cpu_numa_node);
uvm_parent_gpu_unmap_cpu_pages_for_egm(gpu, chunk->common.egm_dma_addr, uvm_cpu_chunk_get_size(&chunk->common));
chunk->common.egm_dma_addr = DMA_ADDR_INVALID;
chunk->egm_parent_id = UVM_PARENT_ID_INVALID;
}
}
static void chunk_dec_gpu_mapping(uvm_cpu_physical_chunk_t *chunk, uvm_parent_gpu_id_t id)
{
uvm_cpu_phys_mapping_t *mapping;
@@ -179,13 +192,20 @@ static void chunk_dec_gpu_mapping(uvm_cpu_physical_chunk_t *chunk, uvm_parent_gp
uvm_parent_gpu_unmap_cpu_pages(parent_gpu, mapping->dma_addr, uvm_cpu_chunk_get_size(&chunk->common));
mapping->dma_addr = 0;
if (UVM_PARENT_ID_IS_VALID(chunk->egm_parent_id)) {
parent_gpu = uvm_parent_gpu_get(chunk->egm_parent_id);
UVM_ASSERT(parent_gpu);
cpu_chunk_unmap_physical_egm(chunk, parent_gpu);
}
if (chunk->gpu_mappings.max_entries > 1) {
NvU32 num_active_entries = uvm_parent_processor_mask_get_gpu_count(&chunk->gpu_mappings.dma_addrs_mask);
NvU32 array_index = compute_gpu_mappings_entry_index(&chunk->gpu_mappings.dma_addrs_mask, id);
// Shift any GPU mappings above this one down in the mappings array.
for (; array_index < num_active_entries - 1; array_index++)
chunk->gpu_mappings.dynamic_entries[array_index] = chunk->gpu_mappings.dynamic_entries[array_index+1];
chunk->gpu_mappings.dynamic_entries[array_index] = chunk->gpu_mappings.dynamic_entries[array_index + 1];
}
uvm_parent_processor_mask_clear(&chunk->gpu_mappings.dma_addrs_mask, id);
@@ -202,6 +222,8 @@ NvU64 uvm_cpu_chunk_get_gpu_phys_addr(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu)
if (uvm_cpu_chunk_is_logical(chunk)) {
uvm_cpu_logical_chunk_t *logical_chunk = uvm_cpu_chunk_to_logical(chunk);
// For EGM, we don't care if the GPU is mapped. We know that the entire
// physical chunk is mapped, so we only need the offset.
if (!uvm_processor_mask_test(&logical_chunk->mapped_gpus, gpu->id))
return 0;
@@ -210,10 +232,38 @@ NvU64 uvm_cpu_chunk_get_gpu_phys_addr(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu)
uvm_mutex_lock(&phys_chunk->lock);
mapping = chunk_phys_mapping_get(phys_chunk, gpu->parent->id);
if (mapping &&
(uvm_cpu_chunk_is_logical(chunk) ||
uvm_sub_processor_mask_test(&mapping->sub_processors, uvm_id_sub_processor_index(gpu->id))))
if (mapping && (uvm_cpu_chunk_is_logical(chunk) ||
uvm_sub_processor_mask_test(&mapping->sub_processors, uvm_id_sub_processor_index(gpu->id)))) {
dma_addr = mapping->dma_addr + (parent_offset * PAGE_SIZE);
}
uvm_mutex_unlock(&phys_chunk->lock);
return dma_addr;
}
NvU64 uvm_cpu_chunk_get_gpu_egm_phys_addr(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu)
{
uvm_cpu_physical_chunk_t *phys_chunk = get_physical_parent(chunk);
uvm_page_index_t parent_offset = 0;
uvm_cpu_logical_chunk_t *logical;
uvm_cpu_chunk_t *track_chunk;
NvU64 dma_addr = 0;
track_chunk = chunk;
uvm_mutex_lock(&phys_chunk->lock);
while (track_chunk->egm_dma_addr == DMA_ADDR_INVALID && uvm_cpu_chunk_is_logical(track_chunk)) {
logical = uvm_cpu_chunk_to_logical(track_chunk);
track_chunk = logical->parent;
}
if (track_chunk->egm_dma_addr == DMA_ADDR_INVALID) {
uvm_mutex_unlock(&phys_chunk->lock);
return 0;
}
parent_offset = (uvm_page_index_t)(chunk->page - track_chunk->page);
dma_addr = track_chunk->egm_dma_addr + (parent_offset * PAGE_SIZE);
uvm_mutex_unlock(&phys_chunk->lock);
return dma_addr;
@@ -262,11 +312,12 @@ static NV_STATUS cpu_chunk_map_gpu_phys(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu)
mapping->dma_addr = dma_addr;
mapping->map_count = 1;
uvm_sub_processor_mask_zero(&mapping->sub_processors);
if (!logical_chunk)
uvm_sub_processor_mask_set(&mapping->sub_processors, uvm_id_sub_processor_index(gpu->id));
uvm_parent_processor_mask_set(&phys_chunk->gpu_mappings.dma_addrs_mask, parent_gpu->id);
uvm_parent_processor_mask_set(&phys_chunk->gpu_mappings.dma_addrs_mask, gpu->parent->id);
}
else {
mapping = chunk_phys_mapping_get(phys_chunk, parent_gpu->id);
@@ -335,10 +386,68 @@ NV_STATUS uvm_cpu_chunk_map_gpu(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu)
return status;
}
NV_STATUS uvm_cpu_chunk_map_gpu_egm(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu)
{
uvm_cpu_physical_chunk_t *phys_chunk = get_physical_parent(chunk);
uvm_cpu_logical_chunk_t *logical;
NV_STATUS status = NV_OK;
dma_addr_t dma_addr;
UVM_ASSERT(page_to_nid(chunk->page) == routing_gpu->closest_cpu_numa_node);
uvm_mutex_lock(&phys_chunk->lock);
while (chunk->egm_dma_addr == DMA_ADDR_INVALID && uvm_cpu_chunk_is_logical(chunk)) {
logical = uvm_cpu_chunk_to_logical(chunk);
chunk = logical->parent;
}
if (chunk->egm_dma_addr == DMA_ADDR_INVALID) {
status = uvm_gpu_map_cpu_pages_for_egm(routing_gpu, chunk->page, uvm_cpu_chunk_get_size(chunk), &dma_addr);
// NV_ERR_NOT_SUPPORTED is returned if the kernel is
// < 6.17. In this cases, there will be no mappings
// mappings created, which will result in non-EGM accesses.
if (status == NV_ERR_NOT_SUPPORTED) {
status = NV_OK;
goto done;
}
if (status != NV_OK)
goto done;
// We already hold a map count on the chunk. All of the
// EGM mappings will be destroyed when the chunk is freed.
chunk->egm_dma_addr = dma_addr;
phys_chunk->egm_parent_id = routing_gpu->id;
}
done:
uvm_mutex_unlock(&phys_chunk->lock);
return status;
}
void uvm_cpu_chunk_unmap_gpu(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu)
{
cpu_chunk_unmap_gpu_phys(chunk, gpu->id);
}
void uvm_cpu_chunk_unmap_gpu_egm(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu)
{
uvm_cpu_physical_chunk_t *phys_chunk = get_physical_parent(chunk);
uvm_cpu_logical_chunk_t *logical;
uvm_mutex_lock(&phys_chunk->lock);
while (chunk->egm_dma_addr == DMA_ADDR_INVALID && uvm_cpu_chunk_is_logical(chunk)) {
logical = uvm_cpu_chunk_to_logical(chunk);
chunk = logical->parent;
}
if (uvm_cpu_chunk_is_physical(chunk))
cpu_chunk_unmap_physical_egm(phys_chunk, routing_gpu);
else
uvm_parent_gpu_unmap_cpu_pages_for_egm(routing_gpu, chunk->egm_dma_addr, uvm_cpu_chunk_get_size(chunk));
uvm_mutex_unlock(&phys_chunk->lock);
// Note: there is no corresponding uvm_mmu_sysmem_unmap() for
// uvm_mmu_sysmem_map().
}
@@ -486,6 +595,8 @@ static uvm_cpu_physical_chunk_t *uvm_cpu_chunk_create(uvm_chunk_size_t alloc_siz
nv_kref_init(&chunk->common.refcount);
uvm_mutex_init(&chunk->lock, UVM_LOCK_ORDER_LEAF);
chunk->gpu_mappings.max_entries = 1;
chunk->egm_parent_id = UVM_PARENT_ID_INVALID;
chunk->common.egm_dma_addr = DMA_ADDR_INVALID;
return chunk;
}
@@ -613,6 +724,7 @@ NV_STATUS uvm_cpu_chunk_split(uvm_cpu_chunk_t *chunk, uvm_cpu_chunk_t **new_chun
new_chunk->common.type = UVM_CPU_CHUNK_TYPE_LOGICAL;
new_chunk->common.page = chunk->page + (i * num_subchunk_pages);
new_chunk->common.egm_dma_addr = DMA_ADDR_INVALID;
uvm_cpu_chunk_set_size(&new_chunk->common, new_size);
nv_kref_init(&new_chunk->common.refcount);
new_chunk->parent = chunk;
kernel-open/nvidia-uvm/uvm_pmm_sysmem.h
+25 -1
View File
@@ -29,6 +29,7 @@
#include "uvm_forward_decl.h"
#include "uvm_lock.h"
#include "uvm_pmm_gpu.h"
#include "uvm_processors.h"
// Global initialization/exit functions, that need to be called during driver
// initialization/tear-down. These are needed to allocate/free global internal
@@ -101,12 +102,21 @@ struct uvm_cpu_chunk_struct
// reference counted, there is no need to take separate references
// to the struct page for logical chunks.
struct page *page;
// This is the IOVA address used for EGM accesses. Each chunk
// holds it's own mapping address. The mapping size is the size
// of the chunk.
// EGM IOVA mappings are created on the parent GPU attached to
// the chunk's page NUMA node.
// This address is property of the common chunk in order to
// correctly handle splitting chunks and overlapping chunks.
dma_addr_t egm_dma_addr;
};
typedef struct
{
// Physical GPU DMA address of the CPU chunk.
NvU64 dma_addr;
dma_addr_t dma_addr;
// Reference count of all sub_processors using this mapping across logical
// and physical chunks.
@@ -153,6 +163,9 @@ typedef struct
uvm_parent_processor_mask_t dma_addrs_mask;
} gpu_mappings;
// ID of the parent GPU holding all of the EGM mappings. This value
// is mostly used for sanity checking.
uvm_parent_gpu_id_t egm_parent_id;
} uvm_cpu_physical_chunk_t;
typedef struct
@@ -282,16 +295,27 @@ void uvm_cpu_chunk_free(uvm_cpu_chunk_t *chunk);
// the operation is issued.
NV_STATUS uvm_cpu_chunk_map_gpu(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu);
// Create EGM mappings for the chunk chunk.
// Mappings are created on the mapping GPU but they are with respect to IOVA
// ranges on the routing GPU.
NV_STATUS uvm_cpu_chunk_map_gpu_egm(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu);
// Destroy a CPU chunk's DMA mapping for the given GPU.
// If chunk is a logical chunk, this call may not necessarily destroy the DMA
// mapping of the parent physical chunk since all logical chunks and MIG
// partitions share the parent's DMA mapping.
void uvm_cpu_chunk_unmap_gpu(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu);
// Destroy EGM mappings for the CPU chunk.
void uvm_cpu_chunk_unmap_gpu_egm(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu);
// Get the CPU chunk's DMA mapping address for the specified GPU ID.
// If there is no mapping for the GPU, 0 is returned.
NvU64 uvm_cpu_chunk_get_gpu_phys_addr(uvm_cpu_chunk_t *chunk, uvm_gpu_t *gpu);
// Get the EGM mapping for the CPU chunk with respect to the routing GPU.
NvU64 uvm_cpu_chunk_get_gpu_egm_phys_addr(uvm_cpu_chunk_t *chunk, uvm_parent_gpu_t *routing_gpu);
// Split a CPU chunk into a set of CPU chunks of the next size down from the set
// of enabled CPU chunk sizes.
//
kernel-open/nvidia-uvm/uvm_rubin.c
+1 -1
View File
@@ -66,7 +66,7 @@ void uvm_hal_rubin_arch_init_properties(uvm_parent_gpu_t *parent_gpu)
parent_gpu->access_counters_serialize_clear_ops_by_type = false;
parent_gpu->access_bits_supported = false;
parent_gpu->access_bits_supported = true;
parent_gpu->has_clear_faulted_channel_sw_method = true;
kernel-open/nvidia-uvm/uvm_rubin_mmu.c
+111
View File
@@ -28,6 +28,11 @@
#include "hwref/rubin/gr100/dev_fault.h"
#include "hwref/rubin/gr100/dev_mmu.h"
// Field defines which are unmaintained by HW
#define NV_MMU_VER3_PTE_ADDRESS_SYS 51 : 12
#define NV_MMU_VER3_PTE_ADDRESS_PEER 51 : 12
#define NV_MMU_VER3_PTE_ADDRESS_VID 39 : 12
static uvm_mmu_mode_hal_t rubin_mmu_mode_hal;
NvU16 uvm_hal_rubin_mmu_client_id_to_utlb_id(NvU16 client_id)
@@ -153,6 +158,111 @@ static NvU64 poisoned_pte_rubin(uvm_page_tree_t *tree)
return WRITE_HWCONST64(pte_bits, _MMU_VER3, PTE, PCF, PRIVILEGE_RO_NO_ATOMIC_UNCACHED_ACD);
}
// PTE Permission Control Flags
static NvU64 pte_pcf(uvm_prot_t prot, NvU64 flags)
{
bool ac = !(flags & UVM_MMU_PTE_FLAGS_ACCESS_COUNTERS_DISABLED);
bool cached = flags & UVM_MMU_PTE_FLAGS_CACHED;
UVM_ASSERT(prot != UVM_PROT_NONE);
UVM_ASSERT((flags & ~UVM_MMU_PTE_FLAGS_MASK) == 0);
if (ac) {
switch (prot) {
case UVM_PROT_READ_ONLY:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_CACHED_ACE :
NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_UNCACHED_ACE;
case UVM_PROT_READ_WRITE:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_CACHED_ACE :
NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_UNCACHED_ACE;
case UVM_PROT_READ_WRITE_ATOMIC:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACE :
NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACE;
default:
break;
}
}
else {
switch (prot) {
case UVM_PROT_READ_ONLY:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_CACHED_ACD :
NV_MMU_VER3_PTE_PCF_REGULAR_RO_NO_ATOMIC_UNCACHED_ACD;
case UVM_PROT_READ_WRITE:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_CACHED_ACD :
NV_MMU_VER3_PTE_PCF_REGULAR_RW_NO_ATOMIC_UNCACHED_ACD;
case UVM_PROT_READ_WRITE_ATOMIC:
return cached ? NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_CACHED_ACD :
NV_MMU_VER3_PTE_PCF_REGULAR_RW_ATOMIC_UNCACHED_ACD;
default:
break;
}
}
// Unsupported PCF
UVM_ASSERT_MSG(0, "Unsupported PTE PCF: prot: %s, ac: %d, cached: %d\n", uvm_prot_string(prot), ac, cached);
return NV_MMU_VER3_PTE_PCF_INVALID;
}
static NvU64 make_pte_rubin(uvm_aperture_t aperture, NvU64 address, uvm_prot_t prot, NvU64 flags)
{
NvU8 aperture_bits = 0;
NvU64 pte_bits = 0;
UVM_ASSERT(prot != UVM_PROT_NONE);
UVM_ASSERT((flags & ~UVM_MMU_PTE_FLAGS_MASK) == 0);
// valid 0:0
pte_bits |= HWCONST64(_MMU_VER3, PTE, VALID, TRUE);
// aperture 2:1
if (aperture == UVM_APERTURE_SYS)
aperture_bits = NV_MMU_VER3_PTE_APERTURE_SYSTEM_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_SYS_NON_COHERENT)
aperture_bits = NV_MMU_VER3_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_VID)
aperture_bits = NV_MMU_VER3_PTE_APERTURE_VIDEO_MEMORY;
else if (uvm_aperture_is_peer(aperture) || uvm_aperture_is_egm(aperture))
aperture_bits = NV_MMU_VER3_PTE_APERTURE_PEER_MEMORY;
else
UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", aperture);
pte_bits |= HWVALUE64(_MMU_VER3, PTE, APERTURE, aperture_bits);
// PCF (permission control flags) 7:3
if (uvm_aperture_is_egm(aperture)) {
// FLA Unicast mappings require *_ACD PCF flags.
flags |= UVM_MMU_PTE_FLAGS_ACCESS_COUNTERS_DISABLED;
}
pte_bits |= HWVALUE64(_MMU_VER3, PTE, PCF, pte_pcf(prot, flags));
// kind 11:8
if (uvm_aperture_is_egm(aperture))
pte_bits |= HWVALUE64(_MMU_VER3, PTE, KIND, NV_MMU_PTE_KIND_SMSKED_MESSAGE);
else
pte_bits |= HWVALUE64(_MMU_VER3, PTE, KIND, NV_MMU_PTE_KIND_GENERIC_MEMORY);
address >>= NV_MMU_VER3_PTE_ADDRESS_SHIFT;
if (aperture == UVM_APERTURE_VID) {
// vid address 39:12
pte_bits |= HWVALUE64(_MMU_VER3, PTE, ADDRESS_VID, address);
}
else {
// sys/peer address 51:12
pte_bits |= HWVALUE64(_MMU_VER3, PTE, ADDRESS, address);
// peer id 63:61
if (uvm_aperture_is_peer(aperture))
pte_bits |= HWVALUE64(_MMU_VER3, PTE, PEER_ID, UVM_APERTURE_PEER_ID(aperture));
else if (uvm_aperture_is_egm(aperture))
pte_bits |= HWVALUE64(_MMU_VER3, PTE, PEER_ID, UVM_APERTURE_EGM_PEER_ID(aperture));
}
return pte_bits;
}
uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_rubin(void)
{
static bool initialized = false;
@@ -168,6 +278,7 @@ uvm_mmu_mode_hal_t *uvm_hal_mmu_mode_rubin(void)
rubin_mmu_mode_hal = *blackwell_mmu_mode_hal;
rubin_mmu_mode_hal.poisoned_pte = poisoned_pte_rubin;
rubin_mmu_mode_hal.make_pte = make_pte_rubin;
initialized = true;
}
kernel-open/nvidia-uvm/uvm_test.c
+6
View File
@@ -42,6 +42,7 @@
#include "uvm_test_file.h"
#include "uvm_user_channel.h"
#include "uvm_gpu_non_replayable_faults.h"
#include "uvm_gpu.h"
static NV_STATUS uvm_test_get_gpu_ref_count(UVM_TEST_GET_GPU_REF_COUNT_PARAMS *params, struct file *filp)
{
@@ -375,6 +376,11 @@ long uvm_test_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_DEAD_CHANNEL, uvm_test_dead_channel);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_SET_NON_REPLAYABLE_DELAY, uvm_test_set_non_replayable_delay);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_QUERY_CDMM_DEVMEM, uvm_test_query_cdmm_devmem);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_EXPORT_DMA_BUF, uvm_test_export_dma_buf);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_REVOKE_DMA_BUF, uvm_test_revoke_dma_buf);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_QUERY_EGM_STATE, uvm_test_query_egm_state);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_PMM_LAZY_FREE_SET_DELAY, uvm_test_pmm_lazy_free_set_delay);
UVM_ROUTE_CMD_STACK_INIT_CHECK(UVM_TEST_PMM_EVICT_TEMP_PINNED, uvm_test_pmm_evict_temp_pinned);
}
return -EINVAL;
kernel-open/nvidia-uvm/uvm_test.h
+6
View File
@@ -131,6 +131,8 @@ NV_STATUS uvm_test_pma_get_batch_size(UVM_TEST_PMA_GET_BATCH_SIZE_PARAMS *params
NV_STATUS uvm_test_pmm_alloc_free_root(UVM_TEST_PMM_ALLOC_FREE_ROOT_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_pmm_inject_pma_evict_error(UVM_TEST_PMM_INJECT_PMA_EVICT_ERROR_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_pmm_query_pma_stats(UVM_TEST_PMM_QUERY_PMA_STATS_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_pmm_lazy_free_set_delay(UVM_TEST_PMM_LAZY_FREE_SET_DELAY_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_pmm_evict_temp_pinned(UVM_TEST_PMM_EVICT_TEMP_PINNED_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_perf_events_sanity(UVM_TEST_PERF_EVENTS_SANITY_PARAMS *params, struct file *filp);
@@ -185,4 +187,8 @@ NV_STATUS uvm_test_dump_access_bits(UVM_TEST_DUMP_ACCESS_BITS_PARAMS *params, st
NV_STATUS uvm_test_sec2_sanity(UVM_TEST_SEC2_SANITY_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_sec2_cpu_gpu_roundtrip(UVM_TEST_SEC2_CPU_GPU_ROUNDTRIP_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_cpu_chunk_api(UVM_TEST_CPU_CHUNK_API_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_export_dma_buf(UVM_TEST_EXPORT_DMA_BUF_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_revoke_dma_buf(UVM_TEST_REVOKE_DMA_BUF_PARAMS *params, struct file *filp);
NV_STATUS uvm_test_query_egm_state(UVM_TEST_QUERY_EGM_STATE_PARAMS *params, struct file *filp);
#endif
kernel-open/nvidia-uvm/uvm_test_ioctl.h
+37
View File
@@ -142,6 +142,7 @@ typedef enum
UVM_TEST_VA_RANGE_TYPE_INVALID = 0,
UVM_TEST_VA_RANGE_TYPE_MANAGED,
UVM_TEST_VA_RANGE_TYPE_EXTERNAL,
UVM_TEST_VA_RANGE_TYPE_DMA_BUF,
UVM_TEST_VA_RANGE_TYPE_CHANNEL,
UVM_TEST_VA_RANGE_TYPE_SKED_REFLECTED,
UVM_TEST_VA_RANGE_TYPE_SEMAPHORE_POOL,
@@ -1624,6 +1625,42 @@ typedef struct
NV_STATUS rmStatus; // Out
} UVM_TEST_QUERY_CDMM_DEVMEM_PARAMS;
#define UVM_TEST_EXPORT_DMA_BUF UVM_TEST_IOCTL_BASE(117)
typedef struct {
NvU64 size; // In
NvS32 fd; // Out
NV_STATUS rmStatus; // Out
} UVM_TEST_EXPORT_DMA_BUF_PARAMS;
#define UVM_TEST_REVOKE_DMA_BUF UVM_TEST_IOCTL_BASE(118)
typedef struct {
NvS32 fd; // In
NV_STATUS rmStatus; // Out
} UVM_TEST_REVOKE_DMA_BUF_PARAMS;
#define UVM_TEST_QUERY_EGM_STATE UVM_TEST_IOCTL_BASE(119)
typedef struct
{
NvProcessorUuid gpu_uuid; // In
NvBool enabled; // Out
NV_STATUS rmStatus; // Out
} UVM_TEST_QUERY_EGM_STATE_PARAMS;
#define UVM_TEST_PMM_LAZY_FREE_SET_DELAY UVM_TEST_IOCTL_BASE(120)
typedef struct
{
NvProcessorUuid gpu_uuid; // In
NvU32 delay_us; // In
NV_STATUS rmStatus; // Out
} UVM_TEST_PMM_LAZY_FREE_SET_DELAY_PARAMS;
#define UVM_TEST_PMM_EVICT_TEMP_PINNED UVM_TEST_IOCTL_BASE(121)
typedef struct
{
NvProcessorUuid gpu_uuid; // In
NV_STATUS rmStatus; // Out
} UVM_TEST_PMM_EVICT_TEMP_PINNED_PARAMS;
#ifdef __cplusplus
}
#endif
kernel-open/nvidia-uvm/uvm_turing_mmu.c
+4 -2
View File
@@ -254,7 +254,7 @@ static NvU64 make_pte_turing(uvm_aperture_t aperture, NvU64 address, uvm_prot_t
aperture_bits = NV_MMU_VER2_PTE_APERTURE_SYSTEM_COHERENT_MEMORY;
else if (aperture == UVM_APERTURE_VID)
aperture_bits = NV_MMU_VER2_PTE_APERTURE_VIDEO_MEMORY;
else if (aperture >= UVM_APERTURE_PEER_0 && aperture <= UVM_APERTURE_PEER_7)
else if (uvm_aperture_is_peer(aperture))
aperture_bits = NV_MMU_VER2_PTE_APERTURE_PEER_MEMORY;
else
UVM_ASSERT_MSG(0, "Invalid aperture: %d\n", aperture);
@@ -304,8 +304,10 @@ static NvU64 make_pte_turing(uvm_aperture_t aperture, NvU64 address, uvm_prot_t
pte_bits |= HWVALUE64(_MMU_VER2, PTE, COMPTAGLINE, addr_hi);
// peer id 35:33
if (aperture != UVM_APERTURE_VID)
if (aperture != UVM_APERTURE_VID) {
UVM_ASSERT(uvm_aperture_is_peer(aperture));
pte_bits |= HWVALUE64(_MMU_VER2, PTE, ADDRESS_VID_PEER, UVM_APERTURE_PEER_ID(aperture));
}
}
// kind 63:56
kernel-open/nvidia-uvm/uvm_va_block.c
+171 -34
View File
@@ -1336,6 +1336,8 @@ static void cpu_chunk_remove_sysmem_gpu_mapping(uvm_cpu_chunk_t *chunk, uvm_gpu_
static NV_STATUS cpu_chunk_add_sysmem_gpu_mapping(uvm_cpu_chunk_t *chunk, uvm_va_block_t *block, uvm_gpu_t *gpu)
{
NV_STATUS status;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_parent_gpu_t *routing_gpu;
// When the Confidential Computing feature is enabled the transfers don't
// use the DMA mapping of CPU chunks (since it's protected memory), but
@@ -1347,6 +1349,13 @@ static NV_STATUS cpu_chunk_add_sysmem_gpu_mapping(uvm_cpu_chunk_t *chunk, uvm_va
if (status != NV_OK)
return status;
routing_gpu = uvm_va_space_get_egm_routing_gpu(va_space, gpu, uvm_cpu_chunk_get_numa_node(chunk));
if (routing_gpu) {
status = uvm_cpu_chunk_map_gpu_egm(chunk, routing_gpu);
if (status != NV_OK)
return status;
}
// If this GPU requires physical invalidations for new DMA mappings, tell
// the next relevant operation to issue one before accessing the mapping.
if (gpu->parent->ats.dma_map_invalidation != UVM_DMA_MAP_INVALIDATION_NONE)
@@ -2702,7 +2711,7 @@ static bool block_check_egm_peer(uvm_va_space_t *va_space, uvm_gpu_t *gpu, int n
uvm_egm_numa_node_info_t *remote_node_info;
uvm_parent_gpu_t *parent_gpu;
if (!uvm_aperture_is_peer(phys_addr.aperture))
if (!uvm_aperture_is_egm(phys_addr.aperture))
return true;
remote_node_info = uvm_va_space_get_egm_numa_node_info(va_space, nid);
@@ -2710,10 +2719,10 @@ static bool block_check_egm_peer(uvm_va_space_t *va_space, uvm_gpu_t *gpu, int n
for_each_parent_gpu_in_mask(parent_gpu, &remote_node_info->parent_gpus) {
NvU64 page_addr = phys_addr.address;
UVM_ASSERT(parent_gpu->egm.enabled);
page_addr += parent_gpu->egm.base_address;
UVM_ASSERT(uvm_parent_gpu_egm_enabled(parent_gpu));
page_addr += uvm_parent_gpu_egm_iova_address(parent_gpu);
if (parent_gpu->nvswitch_info.is_nvswitch_connected && gpu->parent != parent_gpu)
page_addr -= parent_gpu->nvswitch_info.egm_fabric_memory_window_start;
page_addr -= uvm_parent_gpu_egm_fabric_address(parent_gpu);
if (page_addr >= remote_node_info->node_start && page_addr < remote_node_info->node_end &&
remote_node_info->routing_table[uvm_parent_id_gpu_index(gpu->parent->id)] == parent_gpu) {
@@ -2912,6 +2921,7 @@ static NV_STATUS block_populate_gpu_chunk(uvm_va_block_t *block,
block_retry_add_used_chunk(retry, chunk);
chunk->va_block = block;
chunk->va_space = uvm_va_block_get_va_space(block);
chunk->va_block_page_index = chunk_region.first;
if (uvm_va_block_is_hmm(block))
UVM_ASSERT(chunk_index == chunk_region.first);
@@ -2985,7 +2995,8 @@ static NV_STATUS block_populate_pages(uvm_va_block_t *block,
uvm_va_block_context_t *block_context,
uvm_processor_id_t dest_id,
uvm_va_block_region_t region,
const uvm_page_mask_t *page_mask)
const uvm_page_mask_t *page_mask,
uvm_va_block_p2p_mode_t p2p_mode)
{
NV_STATUS status;
const uvm_page_mask_t *resident_mask = block_resident_mask_get_alloc(block,
@@ -2995,6 +3006,8 @@ static NV_STATUS block_populate_pages(uvm_va_block_t *block,
uvm_page_mask_t *pages_staged = &block_context->make_resident.pages_staged;
uvm_page_mask_t *cpu_populate_mask;
uvm_memcg_context_t memcg_context;
static const uvm_processor_mask_t empty_mask = {};
const bool is_p2p_enabled = p2p_mode == UVM_VA_BLOCK_P2P_ENABLED;
if (!resident_mask)
return NV_ERR_NO_MEMORY;
@@ -3025,10 +3038,11 @@ static NV_STATUS block_populate_pages(uvm_va_block_t *block,
// Get the mask of all processors that have resident pages from which
// the destination cannot copy directly.
can_copy_from_processors = block_get_can_copy_from_mask(block, dest_id);
// If p2p is suspended, use an empty mask.
can_copy_from_processors = is_p2p_enabled ? block_get_can_copy_from_mask(block, dest_id) : &empty_mask;
if (!uvm_processor_mask_andnot(tmp_processor_mask, &block->resident, can_copy_from_processors)) {
uvm_processor_mask_cache_free(tmp_processor_mask);
return status;
return NV_OK;
}
// Compute the pages that will be staged through the CPU by:
@@ -3103,6 +3117,54 @@ typedef enum {
REMOTE_EGM_NOT_ALLOWED = 1,
} remote_egm_mode_t;
static uvm_gpu_phys_address_t block_phys_egm_address(uvm_va_block_t *block,
uvm_gpu_t *local_gpu,
uvm_parent_gpu_t *routing_gpu,
uvm_cpu_chunk_t *chunk,
uvm_page_index_t page_index)
{
uvm_aperture_t aperture;
uvm_parent_gpu_t *local_parent_gpu = local_gpu->parent;
NvU64 phys_addr;
UVM_ASSERT(uvm_parent_gpu_egm_enabled(routing_gpu));
aperture = uvm_gpu_egm_peer_aperture(local_parent_gpu, routing_gpu);
if (uvm_parent_gpu_egm_is_legacy(local_parent_gpu)) {
struct page *page = uvm_cpu_chunk_get_cpu_page(block, chunk, page_index);
phys_addr = page_to_phys(page);
// Remote EGM routing is based on both the EGM base address and EGM
// fabric memory window.
if (routing_gpu->nvswitch_info.is_nvswitch_connected && routing_gpu != local_parent_gpu)
phys_addr += uvm_parent_gpu_egm_fabric_address(routing_gpu);
phys_addr -= uvm_parent_gpu_egm_iova_address(routing_gpu);
}
else {
uvm_va_block_region_t chunk_region;
phys_addr = uvm_cpu_chunk_get_gpu_egm_phys_addr(chunk, routing_gpu);
// If there is no EGM mapping, the remote IOVA mapping has not been
// created either. In this case, the access will take the C2C path.
if (!phys_addr) {
aperture = UVM_APERTURE_SYS;
phys_addr = uvm_cpu_chunk_get_gpu_phys_addr(chunk, local_gpu);
UVM_ASSERT(phys_addr != 0);
}
else {
phys_addr += uvm_parent_gpu_egm_fabric_address(routing_gpu);
}
chunk_region = uvm_va_block_chunk_region(block, uvm_cpu_chunk_get_size(chunk), page_index);
phys_addr += (page_index - chunk_region.first) * PAGE_SIZE;
}
return uvm_gpu_phys_address(aperture, phys_addr);
}
// Get the physical GPU address of a block's page from the POV of the specified
// GPU. This is the address that should be used for making PTEs for the
// specified GPU.
@@ -3121,26 +3183,21 @@ static uvm_gpu_phys_address_t block_phys_page_address(uvm_va_block_t *block,
if (UVM_ID_IS_CPU(block_page.processor)) {
uvm_cpu_chunk_t *chunk = uvm_cpu_chunk_get_chunk_for_page(block, block_page.nid, block_page.page_index);
uvm_va_block_region_t chunk_region;
NvU64 phys_addr;
// Sysmem uses coherent SYS aperture
uvm_aperture_t aperture = UVM_APERTURE_SYS;
uvm_va_space_t *va_space = uvm_va_block_get_va_space(block);
uvm_parent_gpu_t *routing_gpu = uvm_va_space_get_egm_routing_gpu(va_space, gpu, block_page.nid);
phys_addr_t phys_addr;
if (routing_gpu && (allow_remote_egm || routing_gpu == gpu->parent)) {
struct page *page = uvm_cpu_chunk_get_cpu_page(block, chunk, block_page.page_index);
uvm_gpu_phys_address_t gpu_address;
phys_addr = page_to_phys(page);
aperture = uvm_gpu_egm_peer_aperture(gpu->parent, routing_gpu);
gpu_address = block_phys_egm_address(block, gpu, routing_gpu, chunk, block_page.page_index);
if (uvm_aperture_is_egm(gpu_address.aperture))
uvm_page_mask_set(&accessing_gpu_state->egm_pages, block_page.page_index);
// Remote EGM routing is based on both the EGM base address and EGM
// fabric memory window.
if (routing_gpu->nvswitch_info.is_nvswitch_connected && routing_gpu != gpu->parent)
phys_addr += routing_gpu->nvswitch_info.egm_fabric_memory_window_start;
uvm_page_mask_set(&accessing_gpu_state->egm_pages, block_page.page_index);
return uvm_gpu_phys_address(aperture, phys_addr - routing_gpu->egm.base_address);
return gpu_address;
}
// The page should be mapped for physical access already as we do that
@@ -3149,7 +3206,6 @@ static uvm_gpu_phys_address_t block_phys_page_address(uvm_va_block_t *block,
chunk_region = uvm_va_block_chunk_region(block, uvm_cpu_chunk_get_size(chunk), block_page.page_index);
UVM_ASSERT(phys_addr != 0);
phys_addr += (block_page.page_index - chunk_region.first) * PAGE_SIZE;
return uvm_gpu_phys_address(aperture, phys_addr);
}
@@ -3201,7 +3257,7 @@ static uvm_gpu_address_t block_phys_page_copy_address(uvm_va_block_t *block,
// systems
uvm_gpu_phys_address_t phys_addr = block_phys_page_address(block, block_page, gpu, REMOTE_EGM_NOT_ALLOWED);
if (uvm_aperture_is_peer(phys_addr.aperture)) {
if (uvm_aperture_is_egm(phys_addr.aperture)) {
UVM_ASSERT(block_check_egm_peer(uvm_va_block_get_va_space(block), gpu, block_page.nid, phys_addr));
return uvm_gpu_address_from_phys(phys_addr);
}
@@ -4464,7 +4520,8 @@ static NV_STATUS block_copy_resident_pages(uvm_va_block_t *block,
uvm_va_block_region_t region,
const uvm_page_mask_t *page_mask,
const uvm_page_mask_t *prefetch_page_mask,
uvm_va_block_transfer_mode_t transfer_mode)
uvm_va_block_transfer_mode_t transfer_mode,
uvm_va_block_p2p_mode_t p2p_mode)
{
NV_STATUS status = NV_OK;
NV_STATUS tracker_status;
@@ -4526,11 +4583,14 @@ static NV_STATUS block_copy_resident_pages(uvm_va_block_t *block,
uvm_processor_mask_zero(src_processor_mask);
if (UVM_ID_IS_GPU(dst_id)) {
if (UVM_ID_IS_GPU(dst_id) && (p2p_mode == UVM_VA_BLOCK_P2P_ENABLED)) {
// If the destination is a GPU, first copy everything from processors
// with copy access supported. Notably this will copy pages from the CPU
// as well even if later some extra copies from CPU are required for
// staged copies.
// If p2p is not enabled, keep the mask empty and let all copies be
// staged in sysmem.
uvm_processor_mask_and(src_processor_mask, block_get_can_copy_from_mask(block, dst_id), &block->resident);
uvm_processor_mask_clear(src_processor_mask, dst_id);
}
@@ -4702,6 +4762,7 @@ NV_STATUS uvm_va_block_make_resident_copy(uvm_va_block_t *va_block,
uvm_processor_mask_t *unmap_processor_mask;
uvm_page_mask_t *unmap_page_mask = &va_block_context->make_resident.page_mask;
uvm_page_mask_t *resident_mask;
uvm_va_block_p2p_mode_t p2p_mode = UVM_VA_BLOCK_P2P_ENABLED;
va_block_context->make_resident.dest_id = dest_id;
va_block_context->make_resident.cause = cause;
@@ -4769,7 +4830,9 @@ NV_STATUS uvm_va_block_make_resident_copy(uvm_va_block_t *va_block,
&va_block_context->discard.scratch_page_mask,
&va_block->discarded_pages);
status = block_populate_pages(va_block, va_block_retry, va_block_context, dest_id, region, page_mask);
retry_no_p2p:
status = block_populate_pages(va_block, va_block_retry, va_block_context, dest_id, region, page_mask, p2p_mode);
if (status != NV_OK)
goto out;
@@ -4779,7 +4842,14 @@ NV_STATUS uvm_va_block_make_resident_copy(uvm_va_block_t *va_block,
region,
page_mask,
prefetch_page_mask,
UVM_VA_BLOCK_TRANSFER_MODE_MOVE);
UVM_VA_BLOCK_TRANSFER_MODE_MOVE,
p2p_mode);
if (UVM_ID_IS_GPU(dest_id) && status == NV_ERR_BUSY_RETRY && p2p_mode == UVM_VA_BLOCK_P2P_ENABLED) {
p2p_mode = UVM_VA_BLOCK_P2P_DISABLED;
goto retry_no_p2p;
}
// HMM does its own clean up.
if (status != NV_OK && !uvm_va_block_is_hmm(va_block)) {
@@ -5010,6 +5080,7 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
uvm_page_mask_t *scratch_residency_mask;
uvm_page_mask_t *resident_mask;
uvm_page_mask_t *preprocess_page_mask = &va_block_context->make_resident.page_mask;
uvm_va_block_p2p_mode_t p2p_mode = UVM_VA_BLOCK_P2P_ENABLED;
// TODO: Bug 3660922: need to implement HMM read duplication support.
UVM_ASSERT(!uvm_va_block_is_hmm(va_block));
@@ -5090,7 +5161,9 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
goto out;
}
status = block_populate_pages(va_block, va_block_retry, va_block_context, dest_id, region, page_mask);
retry_no_p2p:
status = block_populate_pages(va_block, va_block_retry, va_block_context, dest_id, region, page_mask, p2p_mode);
if (status != NV_OK)
goto out;
@@ -5100,7 +5173,15 @@ NV_STATUS uvm_va_block_make_resident_read_duplicate(uvm_va_block_t *va_block,
region,
page_mask,
prefetch_page_mask,
UVM_VA_BLOCK_TRANSFER_MODE_COPY);
UVM_VA_BLOCK_TRANSFER_MODE_COPY,
p2p_mode);
if (UVM_ID_IS_GPU(dest_id) && status == NV_ERR_BUSY_RETRY && p2p_mode == UVM_VA_BLOCK_P2P_ENABLED) {
p2p_mode = UVM_VA_BLOCK_P2P_DISABLED;
goto retry_no_p2p;
}
if (status != NV_OK)
goto out;
@@ -7820,6 +7901,25 @@ NV_STATUS uvm_va_block_unmap(uvm_va_block_t *va_block,
return block_unmap_gpu(va_block, va_block_context, uvm_gpu_get(id), region_page_mask, out_tracker);
}
void uvm_va_block_unmap_egm(uvm_va_block_t *va_block, uvm_parent_gpu_t *gpu)
{
int nid;
uvm_mutex_lock(&va_block->lock);
for_each_possible_uvm_node (nid) {
uvm_cpu_chunk_t *chunk;
uvm_page_index_t page_index;
for_each_cpu_chunk_in_block (chunk, page_index, va_block, nid) {
if (nid == gpu->closest_cpu_numa_node)
uvm_cpu_chunk_unmap_gpu_egm(chunk, gpu);
}
}
uvm_mutex_unlock(&va_block->lock);
}
// This function essentially works as a wrapper around vm_insert_page (hence
// the similar function prototype). This is needed since vm_insert_page
// doesn't take permissions as input, but uses vma->vm_page_prot instead.
@@ -8470,6 +8570,36 @@ NV_STATUS uvm_va_block_map(uvm_va_block_t *va_block,
return status;
}
NV_STATUS uvm_va_block_map_egm(uvm_va_block_t *va_block, uvm_parent_gpu_t *gpu)
{
int nid;
NV_STATUS status = NV_OK;
uvm_mutex_lock(&va_block->lock);
// A future optimization would be to go through all peer GPUs and check if
// there are any existing mappings to SYSMEM pages on the NUMA node
// connected to this GPU.
// If so, unmap them and remap them as EGM mappings.
// TODO: 6211372: Unmap/remap existing SYSMEM mappings.
for_each_possible_uvm_node (nid) {
uvm_cpu_chunk_t *chunk;
uvm_page_index_t page_index;
for_each_cpu_chunk_in_block (chunk, page_index, va_block, nid) {
if (nid == gpu->closest_cpu_numa_node) {
status = uvm_cpu_chunk_map_gpu_egm(chunk, gpu);
if (status != NV_OK)
goto done;
}
}
}
done:
uvm_mutex_unlock(&va_block->lock);
return status;
}
NV_STATUS uvm_va_block_migrate_map_mapped_pages(uvm_va_block_t *va_block,
uvm_va_block_retry_t *va_block_retry,
uvm_va_block_context_t *va_block_context,
@@ -9122,7 +9252,8 @@ static void block_unmap_gpu_egm_mappings(uvm_va_block_t *va_block,
uvm_va_block_context_t *block_context = uvm_va_space_block_context(va_space, NULL);
uvm_va_block_gpu_state_t *gpu_state0 = uvm_va_block_gpu_state_get(va_block, gpu0->id);
if (gpu1->parent->egm.enabled && uvm_va_space_single_gpu_in_parent(va_space, gpu1->parent) && gpu_state0) {
if (uvm_parent_gpu_egm_enabled(gpu1->parent) && uvm_va_space_single_gpu_in_parent(va_space, gpu1->parent) &&
gpu_state0) {
uvm_page_mask_t *unmap_page_mask = &block_context->caller_page_mask;
const uvm_page_mask_t *resident0 = uvm_va_block_resident_mask_get(va_block, gpu0->id, NUMA_NO_NODE);
const uvm_page_mask_t *resident1 = uvm_va_block_resident_mask_get(va_block,
@@ -9139,6 +9270,11 @@ static void block_unmap_gpu_egm_mappings(uvm_va_block_t *va_block,
nvstatusToString(status),
uvm_gpu_name(gpu0));
}
uvm_page_mask_andnot(&gpu_state0->egm_pages, &gpu_state0->egm_pages, unmap_page_mask);
// Unlike legacy EGM, we have to destory all CPU chunk remote EGM
// mappings since the routing GPU is going away.
}
}
}
@@ -12917,8 +13053,11 @@ NV_STATUS uvm_va_block_evict_chunks(uvm_va_block_t *va_block,
if (uvm_va_block_is_hmm(va_block)) {
status = uvm_hmm_va_block_evict_chunk_prep(va_block, block_context, gpu_state->chunks[i], chunk_region);
if (status != NV_OK)
break;
if (status != NV_OK) {
// remove any migration PTEs we prepped
uvm_hmm_va_block_evict_chunk_cancel(block_context);
goto out;
}
}
uvm_page_mask_region_fill(pages_to_evict, chunk_region);
@@ -12955,7 +13094,7 @@ NV_STATUS uvm_va_block_evict_chunks(uvm_va_block_t *va_block,
// root chunk is in_eviction, chunk_free_locked() treats TEMP_PINNED
// chunks as a no-op, so each chunk ends up TEMP_PINNED with an empty
// list as merge_gpu_chunk() requires.
uvm_pmm_gpu_process_lazy_free(&gpu->pmm);
nv_kthread_q_flush(&gpu->parent->lazy_free_q);
}
else {
const uvm_va_policy_t *policy = &va_block->managed_range->policy;
@@ -13427,17 +13566,15 @@ NV_STATUS uvm_test_va_residency_info(UVM_TEST_VA_RESIDENCY_INFO_PARAMS *params,
if (UVM_ID_IS_CPU(block_page.processor)) {
uvm_parent_gpu_t *egm_routing_gpu = uvm_va_space_get_egm_routing_gpu(va_space, gpu, nid);
if (egm_routing_gpu) {
uvm_gpu_t *egm_gpu = uvm_parent_gpu_find_first_valid_gpu(egm_routing_gpu);
compute_egm_page_mask(block, block_context, gpu, egm_gpu->parent, &block_context->caller_page_mask);
if (uvm_page_mask_test(&block_context->caller_page_mask, block_page.page_index)) {
struct page *page = block_page_get(block, block_page);
phys_addr = page_to_phys(page) - egm_routing_gpu->egm.base_address;
phys_addr = page_to_phys(page) - uvm_parent_gpu_egm_iova_address(egm_routing_gpu);
if (egm_routing_gpu->nvswitch_info.is_nvswitch_connected && egm_routing_gpu != gpu->parent)
phys_addr += egm_routing_gpu->nvswitch_info.egm_fabric_memory_window_start;
phys_addr += uvm_parent_gpu_egm_fabric_address(egm_routing_gpu);
params->is_egm_mapping[count] = true;
}
kernel-open/nvidia-uvm/uvm_va_block.h
+8 -3
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2025 NVIDIA Corporation
Copyright (c) 2015-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -902,6 +902,9 @@ NV_STATUS uvm_va_block_map_mask(uvm_va_block_t *va_block,
uvm_prot_t new_prot,
UvmEventMapRemoteCause cause);
// Map all CPU chunks in the block for EGM access.
NV_STATUS uvm_va_block_map_egm(uvm_va_block_t *va_block, uvm_parent_gpu_t *gpu);
// Map pages not already mapped on the destination processor after a migration.
//
// LOCKING: The VA block lock must be held. If va_block_context->mm !=
@@ -954,6 +957,9 @@ NV_STATUS uvm_va_block_unmap(uvm_va_block_t *va_block,
const uvm_page_mask_t *unmap_page_mask,
uvm_tracker_t *out_tracker);
// Unmap any EGM mappings for CPU chunks.
void uvm_va_block_unmap_egm(uvm_va_block_t *block, uvm_parent_gpu_t *gpu);
// Like uvm_va_block_unmap, except it unmaps all processors in the input mask.
// The VA block tracker contains all map operations on return.
NV_STATUS uvm_va_block_unmap_mask(uvm_va_block_t *va_block,
@@ -1253,8 +1259,7 @@ NV_STATUS uvm_va_block_split_locked(uvm_va_block_t *existing_va_block,
// service_context->block_context.mm is ignored and vma->vm_mm is used instead.
//
// Returns NV_ERR_INVALID_ACCESS_TYPE if a CPU mapping to fault_addr cannot be
// accessed, for example because it's within a range group which is non-
// migratable.
// accessed.
NV_STATUS uvm_va_block_cpu_fault(uvm_va_block_t *va_block,
NvU64 fault_addr,
bool is_write,
kernel-open/nvidia-uvm/uvm_va_block_types.h
+6
View File
@@ -342,6 +342,12 @@ typedef enum
UVM_VA_BLOCK_TRANSFER_MODE_COPY = 2
} uvm_va_block_transfer_mode_t;
typedef enum
{
UVM_VA_BLOCK_P2P_DISABLED = 0,
UVM_VA_BLOCK_P2P_ENABLED = 1,
} uvm_va_block_p2p_mode_t;
typedef enum
{
UVM_SERVICE_OPERATION_REPLAYABLE_FAULTS,
kernel-open/nvidia-uvm/uvm_va_range.c
+178
View File
@@ -22,7 +22,9 @@
*******************************************************************************/
#include "uvm_common.h"
#include "uvm_forward_decl.h"
#include "uvm_linux.h"
#include "uvm_lock.h"
#include "uvm_types.h"
#include "uvm_api.h"
#include "uvm_processors.h"
@@ -34,13 +36,22 @@
#include "uvm_va_block.h"
#include "uvm_kvmalloc.h"
#include "uvm_map_external.h"
#include "uvm_va_range_dmabuf.h"
#include "uvm_perf_thrashing.h"
#include "nv_uvm_interface.h"
#include <linux/sched.h>
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
#include <linux/dma-buf.h>
#include <linux/dma-resv.h>
#endif
static struct kmem_cache *g_uvm_va_range_managed_cache __read_mostly;
static struct kmem_cache *g_uvm_va_range_external_cache __read_mostly;
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
static struct kmem_cache *g_uvm_va_range_dma_buf_cache __read_mostly;
#endif
static struct kmem_cache *g_uvm_va_range_channel_cache __read_mostly;
static struct kmem_cache *g_uvm_va_range_sked_reflected_cache __read_mostly;
static struct kmem_cache *g_uvm_va_range_semaphore_pool_cache __read_mostly;
@@ -58,6 +69,12 @@ NV_STATUS uvm_va_range_init(void)
if (!g_uvm_va_range_external_cache)
return NV_ERR_NO_MEMORY;
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
g_uvm_va_range_dma_buf_cache = NV_KMEM_CACHE_CREATE("uvm_va_range_dma_buf_t", uvm_va_range_dma_buf_t);
if (!g_uvm_va_range_dma_buf_cache)
return NV_ERR_NO_MEMORY;
#endif
g_uvm_va_range_channel_cache = NV_KMEM_CACHE_CREATE("uvm_va_range_channel_t", uvm_va_range_channel_t);
if (!g_uvm_va_range_channel_cache)
return NV_ERR_NO_MEMORY;
@@ -88,6 +105,9 @@ void uvm_va_range_exit(void)
uvm_va_block_exit();
kmem_cache_destroy_safe(&g_uvm_va_range_managed_cache);
kmem_cache_destroy_safe(&g_uvm_va_range_external_cache);
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
kmem_cache_destroy_safe(&g_uvm_va_range_dma_buf_cache);
#endif
kmem_cache_destroy_safe(&g_uvm_va_range_channel_cache);
kmem_cache_destroy_safe(&g_uvm_va_range_sked_reflected_cache);
kmem_cache_destroy_safe(&g_uvm_va_range_semaphore_pool_cache);
@@ -299,6 +319,74 @@ error:
return status;
}
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
NV_STATUS uvm_va_range_create_dma_buf(uvm_va_space_t *va_space,
struct mm_struct *mm,
struct dma_buf *dmabuf,
NvU64 start,
NvU64 length,
uvm_va_range_dma_buf_t **out_dmabuf_range)
{
NV_STATUS status;
uvm_va_range_dma_buf_t *dmabuf_range = NULL;
NvU32 i;
// Assigns to zero mapped_gpus, revoked_gpus bitmasks.
dmabuf_range = nv_kmem_cache_zalloc(g_uvm_va_range_dma_buf_cache, NV_UVM_GFP_FLAGS);
if (!dmabuf_range)
return NV_ERR_NO_MEMORY;
get_dma_buf(dmabuf);
dmabuf_range->dmabuf = dmabuf;
status = uvm_va_range_initialize_reclaim(&dmabuf_range->va_range,
mm,
UVM_VA_RANGE_TYPE_DMA_BUF,
va_space,
start,
start + length - 1);
if (status != NV_OK) {
kmem_cache_free(g_uvm_va_range_dma_buf_cache, dmabuf_range);
return status;
}
dmabuf_range->retained_mask = uvm_processor_mask_cache_alloc();
if (!dmabuf_range->retained_mask) {
status = NV_ERR_NO_MEMORY;
goto error;
}
for (i = 0; i < ARRAY_SIZE(dmabuf_range->gpu_ranges); i++) {
uvm_mutex_init(&dmabuf_range->gpu_ranges[i].base.lock, UVM_LOCK_ORDER_DMA_BUF_RANGE_TREE);
uvm_range_tree_init(&dmabuf_range->gpu_ranges[i].base.tree);
}
status = uvm_range_tree_add(&va_space->va_range_tree, &dmabuf_range->va_range.node);
if (status != NV_OK)
goto error;
if (out_dmabuf_range)
*out_dmabuf_range = dmabuf_range;
return NV_OK;
error:
uvm_va_range_destroy(&dmabuf_range->va_range, NULL);
return status;
}
#else
NV_STATUS uvm_va_range_create_dma_buf(uvm_va_space_t *va_space,
struct mm_struct *mm,
struct dma_buf *dmabuf,
NvU64 start,
NvU64 length,
uvm_va_range_dma_buf_t **out_dmabuf_range)
{
return NV_ERR_NOT_SUPPORTED;
}
#endif // UVM_PROVIDES_DMA_BUF_IMPORTER()
NV_STATUS uvm_va_range_create_channel(uvm_va_space_t *va_space,
struct mm_struct *mm,
NvU64 start,
@@ -530,6 +618,26 @@ out:
kmem_cache_free(g_uvm_va_range_external_cache, external_range);
}
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
static void uvm_va_range_destroy_dma_buf(uvm_va_range_dma_buf_t *dmabuf_range,
struct list_head *deferred_free_list)
{
uvm_gpu_t *gpu;
uvm_processor_mask_cache_free(dmabuf_range->retained_mask);
for_each_va_space_gpu(gpu, dmabuf_range->va_range.va_space)
uvm_dma_buf_gpu_range_tree_deinit(dmabuf_range, gpu, deferred_free_list);
UVM_ASSERT(uvm_processor_mask_empty(&dmabuf_range->mapped_gpus));
dma_buf_put(dmabuf_range->dmabuf);
kmem_cache_free(g_uvm_va_range_dma_buf_cache, dmabuf_range);
}
#else
static void uvm_va_range_destroy_dma_buf(uvm_va_range_dma_buf_t *dmabuf_range,
struct list_head *deferred_free_list) {}
#endif // UVM_PROVIDES_DMA_BUF_IMPORTER()
static void uvm_va_range_destroy_channel(uvm_va_range_channel_t *channel_range)
{
uvm_gpu_va_space_t *gpu_va_space = channel_range->gpu_va_space;
@@ -604,6 +712,13 @@ void uvm_va_range_destroy(uvm_va_range_t *va_range, struct list_head *deferred_f
case UVM_VA_RANGE_TYPE_EXTERNAL:
uvm_va_range_destroy_external(uvm_va_range_to_external(va_range), deferred_free_list);
return;
case UVM_VA_RANGE_TYPE_DMA_BUF:
#if !UVM_PROVIDES_DMA_BUF_IMPORTER()
UVM_ASSERT_MSG(0, "[0x%llx, 0x%llx] unexpected DMA_BUF range\n",
va_range->node.start, va_range->node.end);
#endif
uvm_va_range_destroy_dma_buf(uvm_va_range_to_dma_buf(va_range), deferred_free_list);
return;
case UVM_VA_RANGE_TYPE_CHANNEL:
uvm_va_range_destroy_channel(uvm_va_range_to_channel(va_range));
return;
@@ -662,6 +777,30 @@ static uvm_processor_mask_t *uvm_free_external(uvm_va_range_external_t *external
return retained_mask;
}
static uvm_processor_mask_t *uvm_free_dma_buf(uvm_va_range_dma_buf_t *dmabuf_range)
{
uvm_processor_mask_t *retained_mask = dmabuf_range->retained_mask;
uvm_va_space_t *va_space = dmabuf_range->va_range.va_space;
uvm_gpu_t *gpu;
// Set the retained_mask to NULL to prevent uvm_va_range_destroy_dma_buf()
// from freeing the mask, should no attachments require deferred free.
dmabuf_range->retained_mask = NULL;
UVM_ASSERT(retained_mask);
uvm_processor_mask_zero(retained_mask);
// DMA-BUF ranges defer free the destruction of attachments, so GPUs must be
// retained. Construct the mask of all the GPUs that need to be retained.
uvm_assert_rwsem_locked(&va_space->lock);
for_each_va_space_gpu(gpu, va_space) {
if (uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu)->attach)
uvm_processor_mask_set(retained_mask, gpu->id);
}
return retained_mask;
}
// This destroys VA ranges created by ioctl. VA ranges created by mmap, such as
// through UvmMemMap, go through munmap.
static NV_STATUS uvm_free(uvm_va_space_t *va_space, NvU64 base)
@@ -685,6 +824,10 @@ static NV_STATUS uvm_free(uvm_va_space_t *va_space, NvU64 base)
retained_mask = uvm_free_external(uvm_va_range_to_external(va_range));
break;
case UVM_VA_RANGE_TYPE_DMA_BUF:
retained_mask = uvm_free_dma_buf(uvm_va_range_to_dma_buf(va_range));
break;
case UVM_VA_RANGE_TYPE_SEMAPHORE_POOL:
status = uvm_free_semaphore_pool(uvm_va_range_to_semaphore_pool(va_range));
break;
@@ -1339,6 +1482,13 @@ static void va_range_remove_gpu_va_space_external(uvm_va_range_external_t *exter
uvm_mutex_unlock(&range_tree->lock);
}
static void va_range_remove_gpu_va_space_dma_buf(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
struct list_head *deferred_free_list)
{
uvm_dma_buf_gpu_range_tree_deinit(dmabuf_range, gpu, deferred_free_list);
}
static void va_range_remove_gpu_va_space_semaphore_pool(uvm_va_range_semaphore_pool_t *semaphore_pool_range,
uvm_gpu_t *gpu)
{
@@ -1363,6 +1513,15 @@ void uvm_va_range_remove_gpu_va_space(uvm_va_range_t *va_range,
gpu_va_space->gpu,
deferred_free_list);
break;
case UVM_VA_RANGE_TYPE_DMA_BUF:
#if !UVM_PROVIDES_DMA_BUF_IMPORTER()
UVM_ASSERT_MSG(0, "[0x%llx, 0x%llx] unexpected DMA_BUF range\n",
va_range->node.start, va_range->node.end);
#endif
va_range_remove_gpu_va_space_dma_buf(uvm_va_range_to_dma_buf(va_range),
gpu_va_space->gpu,
deferred_free_list);
break;
case UVM_VA_RANGE_TYPE_CHANNEL:
// All channels under this GPU VA space should've been removed before
// removing the GPU VA space.
@@ -1431,6 +1590,9 @@ NV_STATUS uvm_va_range_enable_peer(uvm_va_range_t *va_range, uvm_gpu_t *gpu0, uv
case UVM_VA_RANGE_TYPE_EXTERNAL:
// UVM_VA_RANGE_TYPE_EXTERNAL doesn't create new mappings when enabling peer access
return NV_OK;
case UVM_VA_RANGE_TYPE_DMA_BUF:
// UVM_VA_RANGE_TYPE_DMA_BUF should never have peer mappings
return NV_OK;
case UVM_VA_RANGE_TYPE_CHANNEL:
// UVM_VA_RANGE_TYPE_CHANNEL should never have peer mappings
return NV_OK;
@@ -1493,6 +1655,9 @@ void uvm_va_range_disable_peer(uvm_va_range_t *va_range,
// If GPU 1 has a mapping to GPU 0, remove GPU 1's mapping
uvm_va_range_disable_peer_external(uvm_va_range_to_external(va_range), gpu1, gpu0, deferred_free_list);
break;
case UVM_VA_RANGE_TYPE_DMA_BUF:
// UVM_VA_RANGE_TYPE_DMA_BUF should never have peer mappings
break;
case UVM_VA_RANGE_TYPE_CHANNEL:
// UVM_VA_RANGE_TYPE_CHANNEL should never have peer mappings
break;
@@ -1608,6 +1773,18 @@ void uvm_va_range_unregister_gpu(uvm_va_range_t *va_range,
case UVM_VA_RANGE_TYPE_EXTERNAL:
va_range_unregister_gpu_external(uvm_va_range_to_external(va_range), gpu, deferred_free_list);
break;
case UVM_VA_RANGE_TYPE_DMA_BUF:
#if !UVM_PROVIDES_DMA_BUF_IMPORTER()
UVM_ASSERT_MSG(0, "[0x%llx, 0x%llx] unexpected DMA_BUF range\n",
va_range->node.start, va_range->node.end);
#endif
// All ranges for this GPU should have been unmapped by GPU VA space
// unregister (va_range_remove_gpu_va_space_dma_buf), which should
// have already happened.
UVM_ASSERT(!uvm_processor_mask_test(&uvm_va_range_to_dma_buf(va_range)->mapped_gpus, gpu->id));
UVM_ASSERT(uvm_range_tree_empty(
&uvm_dma_buf_gpu_range_tree(uvm_va_range_to_dma_buf(va_range), gpu)->base.tree));
break;
case UVM_VA_RANGE_TYPE_CHANNEL:
// All ranges should have been destroyed by GPU VA space unregister,
// which should have already happened.
@@ -2316,6 +2493,7 @@ NV_STATUS uvm_test_va_range_info(UVM_TEST_VA_RANGE_INFO_PARAMS *params, struct f
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_INVALID != (int)UVM_VA_RANGE_TYPE_INVALID);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_MANAGED != (int)UVM_VA_RANGE_TYPE_MANAGED);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_EXTERNAL != (int)UVM_VA_RANGE_TYPE_EXTERNAL);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_DMA_BUF != (int)UVM_VA_RANGE_TYPE_DMA_BUF);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_CHANNEL != (int)UVM_VA_RANGE_TYPE_CHANNEL);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_SKED_REFLECTED != (int)UVM_VA_RANGE_TYPE_SKED_REFLECTED);
BUILD_BUG_ON((int)UVM_TEST_VA_RANGE_TYPE_SEMAPHORE_POOL != (int)UVM_VA_RANGE_TYPE_SEMAPHORE_POOL);
kernel-open/nvidia-uvm/uvm_va_range.h
+157 -16
View File
@@ -77,6 +77,16 @@
// These ranges do not have blocks. All state (page tables, mapping handles,
// etc) is maintained within the range.
//
// VA ranges with type == UVM_VA_RANGE_TYPE_DMA_BUF:
// These ranges track DMA-BUF buffers that have been imported. The UVM driver is
// responsible for mapping them to the GPU(s), but not to the CPU. These
// ranges do not support faulting nor migration, and they typically represent
// access to PCIe peer device memories not controlled by UVM or RM, and with no
// backing vma.
//
// These ranges do not have blocks. All state (page tables, mapping handles,
// etc) is maintained within the range.
//
// VA ranges with type == UVM_VA_RANGE_TYPE_CHANNEL:
// These are similar to EXTERNAL ranges, except they represent internal
// allocations required for user channels to operate (context save areas,
@@ -107,6 +117,7 @@ typedef enum
UVM_VA_RANGE_TYPE_INVALID = 0,
UVM_VA_RANGE_TYPE_MANAGED,
UVM_VA_RANGE_TYPE_EXTERNAL,
UVM_VA_RANGE_TYPE_DMA_BUF,
UVM_VA_RANGE_TYPE_CHANNEL,
UVM_VA_RANGE_TYPE_SKED_REFLECTED,
UVM_VA_RANGE_TYPE_SEMAPHORE_POOL,
@@ -141,10 +152,48 @@ typedef struct
nv_kref_t ref_count;
} uvm_ext_gpu_mem_handle;
// Common base for per-GPU range trees of sub-range mappings.
typedef struct
{
// Lock protecting the range tree.
uvm_mutex_t lock;
// Range tree that contains all of the mapped portions of a range which
// permits per-GPU sub-range mappings. The tree holds uvm_gpu_map_t
// instances.
uvm_range_tree_t tree;
} uvm_gpu_range_tree_t;
typedef uvm_gpu_range_tree_t uvm_ext_gpu_range_tree_t;
typedef struct
{
uvm_gpu_range_tree_t base;
// A borrowed reference to the GPU's attachment. The attachment is created
// at time of first mapping on the GPU, but detached after this range is
// destroyed as a deferred-free in UvmFree(), outside of the va_space lock.
// This is due to the locking requirement imposed externally by DMA-BUF.
struct dma_buf_attachment *attach;
} uvm_dma_buf_gpu_range_tree_t;
// Common base for per-GPU sub-range mappings.
typedef struct
{
uvm_range_tree_node_t node;
// GPU on which this allocation is mapped.
uvm_gpu_t *gpu;
uvm_page_table_range_vec_t pt_range_vec;
} uvm_gpu_map_t;
typedef uvm_gpu_map_t uvm_dma_buf_gpu_map_t;
typedef struct
{
uvm_gpu_map_t base;
// Handle to the physical user allocation dup'd into our client. This
// prevents the allocation from being removed until we free it, even if the
// user frees their handle without telling us.
@@ -158,9 +207,6 @@ typedef struct
// tracker before operating on pt_range_vec.
uvm_tracker_t tracker;
// GPU on which this allocation is mapped.
uvm_gpu_t *gpu;
// GPU which owns the allocation. For sysmem, this is the GPU that the
// sysmem was originally allocated under. For the allocation to remain valid
// we need to prevent the GPU from going away, similarly to P2P mapped
@@ -191,8 +237,6 @@ typedef struct
// Fabric memory. If true, the allocation is a fabric allocation.
bool is_fabricmem;
uvm_page_table_range_vec_t pt_range_vec;
// Node for the deferred free list where this allocation is stored upon
// unmapped.
//
@@ -210,16 +254,6 @@ typedef struct
bool need_l2_invalidate_at_unmap;
} uvm_ext_gpu_map_t;
typedef struct
{
// Lock protecting the range tree.
uvm_mutex_t lock;
// Range tree that contains all of the mapped portions of an External VA
// range. The tree holds uvm_ext_gpu_map_t instances.
uvm_range_tree_t tree;
} uvm_ext_gpu_range_tree_t;
struct uvm_va_range_struct
{
// Parent uvm_va_space.
@@ -308,6 +342,43 @@ struct uvm_va_range_external_struct
uvm_processor_mask_t *retained_mask;
};
struct uvm_va_range_dma_buf_struct
{
// Base class
uvm_va_range_t va_range;
struct dma_buf *dmabuf;
// Mask of GPUs which have GMMU mappings to this VA range. If a bit in this
// mask is set, at a minimum there must exist for the relevant GPU a DMA-BUF
// attachment and the attachment must be mapped.
// The bitmap can be safely accessed by following the locking rules:
// * If the VA space lock is held for write, the mask can be read or written
// normally.
// * If the VA space lock is held for read, and one of the range tree locks is
// held, only the bit corresponding to that GPU range tree can be accessed.
// Writes must use uvm_processor_mask_set_atomic and
// uvm_processor_mask_clear_atomic to avoid clobbering other bits in the
// mask. If no range tree lock is held, the mask cannot be accessed.
// * If the VA space lock is not held, the mask cannot be accessed
uvm_processor_mask_t mapped_gpus;
// Mask of GPUs that have received a move_notify() notification, prohibiting
// further mappings from being established on this GPU.
// Members of this bitmap are accessible under the same restricions that
// mapped_gpus members are.
uvm_processor_mask_t revoked_gpus;
// Per-GPU tree of mapped dma-buf allocations. This has to be per-GPU in the VA
// range because each GPU is able to map a completely different set of
// allocations to the same VA range.
uvm_dma_buf_gpu_range_tree_t gpu_ranges[UVM_ID_MAX_GPUS];
// Dynamically allocated page mask allocated in
// uvm_va_range_create_dma_buf() and used and freed in uvm_free().
uvm_processor_mask_t *retained_mask;
};
// Subclass of va_range state for va_range.type == UVM_VA_RANGE_TYPE_CHANNEL.
// This represents a channel buffer resource and mapping.
struct uvm_va_range_channel_struct
@@ -427,6 +498,14 @@ static inline uvm_va_range_external_t *uvm_va_range_to_external(uvm_va_range_t *
return container_of(va_range, uvm_va_range_external_t, va_range);
}
// DMA-BUF Range dynamic cast
static inline uvm_va_range_dma_buf_t *uvm_va_range_to_dma_buf(uvm_va_range_t *va_range)
{
UVM_ASSERT(va_range);
UVM_ASSERT(va_range->type == UVM_VA_RANGE_TYPE_DMA_BUF);
return container_of(va_range, uvm_va_range_dma_buf_t, va_range);
}
// Channel Range dynamic cast
static inline uvm_va_range_channel_t *uvm_va_range_to_channel(uvm_va_range_t *va_range)
{
@@ -515,6 +594,17 @@ NV_STATUS uvm_va_range_create_external(uvm_va_space_t *va_space,
NvU64 length,
uvm_va_range_external_t **out_external_range);
// Create an external range. out_dma_buf_range is optional.
//
// Returns NV_ERR_UVM_ADDRESS_IN_USE if the range overlaps with an existing
// range in the va_space tree.
NV_STATUS uvm_va_range_create_dma_buf(uvm_va_space_t *va_space,
struct mm_struct *mm,
struct dma_buf *dmabuf,
NvU64 start,
NvU64 length,
uvm_va_range_dma_buf_t **out_dmabuf_range);
// Create a channel range. out_channel_range is optional.
//
// Returns NV_ERR_UVM_ADDRESS_IN_USE if the range overlaps with an existing
@@ -654,6 +744,18 @@ static uvm_va_range_external_t *uvm_va_range_external_find(uvm_va_space_t *va_sp
return uvm_va_range_to_external(va_range);
}
static uvm_va_range_dma_buf_t *uvm_va_range_dma_buf_find(uvm_va_space_t *va_space, NvU64 addr)
{
uvm_va_range_t *va_range;
va_range = uvm_va_range_find(va_space, addr);
if (!va_range)
return NULL;
if (va_range->type != UVM_VA_RANGE_TYPE_DMA_BUF)
return NULL;
return uvm_va_range_to_dma_buf(va_range);
}
static uvm_va_range_semaphore_pool_t *uvm_va_range_semaphore_pool_find(uvm_va_space_t *va_space,
NvU64 addr)
{
@@ -693,9 +795,48 @@ static uvm_ext_gpu_map_t *uvm_ext_gpu_map_container(uvm_range_tree_node_t *node)
{
if (!node)
return NULL;
return container_of(node, uvm_ext_gpu_map_t, node);
return container_of(node, uvm_ext_gpu_map_t, base.node);
}
static uvm_dma_buf_gpu_map_t *uvm_dma_buf_gpu_map_container(uvm_range_tree_node_t *node)
{
if (!node)
return NULL;
return container_of(node, uvm_dma_buf_gpu_map_t, node);
}
// Shared range tree iteration primitives. Both the ext and dma_buf variants
// delegate to these; callers get typed pointers back via the container_of
// wrappers above.
static uvm_range_tree_node_t *uvm_gpu_range_tree_iter_first(uvm_gpu_range_tree_t *range_tree,
NvU64 start, NvU64 end)
{
return uvm_range_tree_iter_first(&range_tree->tree, start, end);
}
static uvm_range_tree_node_t *uvm_gpu_range_tree_iter_next(uvm_gpu_range_tree_t *range_tree,
uvm_gpu_map_t *map,
NvU64 end)
{
if (!map)
return NULL;
return uvm_range_tree_iter_next(&range_tree->tree, &map->node, end);
}
// Internal base macros for GPU map iteration; use the typed variants defined
// in uvm_map_external.h and uvm_va_range_dmabuf.h.
#define uvm_gpu_map_for_each_in(map, va_range, gpu, start, end, iter_first, iter_next) \
for ((map) = (iter_first)((va_range), (gpu), (start), (end)); \
(map); \
(map) = (iter_next)((va_range), (map), (end)))
#define uvm_gpu_map_for_each_in_safe(map, map_next, va_range, gpu, start, end, iter_first, iter_next) \
for ((map) = (iter_first)((va_range), (gpu), (start), (end)), \
(map_next) = (iter_next)((va_range), (map), (end)); \
(map); \
(map) = (map_next), \
(map_next) = (iter_next)((va_range), (map), (end)))
// Iterators for all va_ranges
#define uvm_for_each_va_range(va_range, va_space) \
kernel-open/nvidia-uvm/uvm_va_range_dmabuf.c
+764
View File
@@ -0,0 +1,764 @@
/*******************************************************************************
Copyright (c) 2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
#include "uvm_linux.h"
#include "uvm_common.h"
#include "uvm_api.h"
#include "uvm_forward_decl.h"
#include "uvm_va_range_dmabuf.h"
#include "uvm_gpu.h"
#include "uvm_hal.h"
#include "uvm_hal_types.h"
#include "uvm_lock.h"
#include "uvm_processors.h"
#include "uvm_tracker.h"
#include "uvm_va_space.h"
#include "uvm_va_range.h"
#include "uvm_va_space.h"
#include "uvm_ioctl.h"
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
#include <linux/dma-buf.h>
#include <linux/dma-resv.h>
// Mutable iterator data state passed to uvm_dma_buf_pte_maker()
typedef struct {
uvm_gpu_t *mapping_gpu;
struct sg_dma_page_iter dma_iter;
} uvm_dma_buf_pte_maker_data_t;
static void uvm_dma_buf_gpu_map_destroy(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_dma_buf_gpu_map_t *dmabuf_map);
static NV_STATUS uvm_create_dma_buf_range(uvm_va_space_t *va_space,
struct dma_buf *dmabuf,
UVM_IMPORT_DMA_BUF_PARAMS *params)
{
uvm_va_range_dma_buf_t *dmabuf_range = NULL;
struct mm_struct *mm;
NV_STATUS status = NV_OK;
// The DMA-BUF interface provides a sg_table of DMA mapped regions through
// dma_buf_map_attachment(), with base addresses and lengths required to be
// aligned to the system page size. Assert the same here.
if (uvm_api_range_invalid(params->base, params->length))
return NV_ERR_INVALID_ADDRESS;
// Require that the provided length for the va_range matches the full size
// of the DMA-BUF.
if (params->length != dmabuf->size)
return NV_ERR_INVALID_ADDRESS;
// The mm needs to be locked in order to remove stale HMM va_blocks.
mm = uvm_va_space_mm_or_current_retain_lock(va_space);
uvm_va_space_down_write(va_space);
// Create the new DMA-BUF VA range.
// uvm_va_range_create_dma_buf handles any collisions when it attempts to
// insert the new range into the va_space range tree.
status = uvm_va_range_create_dma_buf(va_space, mm, dmabuf, params->base, dmabuf->size, &dmabuf_range);
if (status != NV_OK) {
UVM_DBG_PRINT_RL("Failed to create DMA-BUF VA range [0x%llx, 0x%llx)\n",
params->base,
params->base + dmabuf->size);
}
uvm_va_space_up_write(va_space);
uvm_va_space_mm_or_current_release_unlock(va_space, mm);
return status;
}
NV_STATUS uvm_api_import_dma_buf(UVM_IMPORT_DMA_BUF_PARAMS *params, struct file *filp)
{
uvm_va_space_t *va_space = uvm_va_space_get(filp);
struct dma_buf *dmabuf;
NV_STATUS status;
dmabuf = dma_buf_get(params->handleFd);
if (IS_ERR(dmabuf))
return NV_ERR_INVALID_ARGUMENT;
status = uvm_create_dma_buf_range(va_space, dmabuf, params);
dma_buf_put(dmabuf);
return status;
}
// Retrieve the per-GPU sub-range mapping object spanning a given virtual
// address or NULL if no such mapping exists.
static uvm_dma_buf_gpu_map_t *uvm_va_range_dma_buf_get_gpu_map_or_null(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *mapping_gpu,
NvU64 addr)
{
uvm_dma_buf_gpu_map_t *dmabuf_map = NULL;
uvm_range_tree_node_t *node;
uvm_dma_buf_gpu_range_tree_t *range_tree;
uvm_assert_rwsem_locked(&dmabuf_range->va_range.va_space->lock);
range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, mapping_gpu);
if (uvm_processor_mask_test(&dmabuf_range->mapped_gpus, mapping_gpu->id)) {
UVM_ASSERT(!uvm_range_tree_empty(&range_tree->base.tree));
node = uvm_range_tree_find(&range_tree->base.tree, addr);
if (node) {
dmabuf_map = uvm_dma_buf_gpu_map_container(node);
UVM_ASSERT(dmabuf_map->gpu == mapping_gpu);
}
}
else {
UVM_ASSERT(uvm_range_tree_empty(&range_tree->base.tree));
}
return dmabuf_map;
}
// TODO: Bug 6164216 refactor for code reuse with external mappings
static NV_STATUS uvm_dma_buf_gpu_map_split(uvm_range_tree_t *tree,
uvm_dma_buf_gpu_map_t *existing_map,
NvU64 new_end,
uvm_dma_buf_gpu_map_t **new_map)
{
uvm_dma_buf_gpu_map_t *new;
NV_STATUS status;
NvU64 new_start = new_end + 1;
if (!IS_ALIGNED(new_start, existing_map->pt_range_vec.page_size))
return NV_ERR_INVALID_ADDRESS;
UVM_ASSERT(new_start >= existing_map->node.start && new_start < existing_map->node.end);
new = uvm_kvmalloc_zero(sizeof(*new));
if (!new)
return NV_ERR_NO_MEMORY;
RB_CLEAR_NODE(&new->node.rb_node);
new->gpu = existing_map->gpu;
status = uvm_page_table_range_vec_split_upper(&existing_map->pt_range_vec, new_start - 1, &new->pt_range_vec);
if (status != NV_OK) {
uvm_kvfree(new);
return status;
}
new->node.start = new_start;
uvm_range_tree_split(tree, &existing_map->node, &new->node);
if (new_map)
*new_map = new;
return NV_OK;
}
// See comment in uvm_unmap_external_buf_in_range for description of the
// implementation.
static NV_STATUS uvm_unmap_dma_buf_in_range(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
NvU64 start,
NvU64 end)
{
uvm_dma_buf_gpu_range_tree_t *range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu);
uvm_dma_buf_gpu_map_t *dmabuf_map, *dmabuf_map_next = NULL;
NV_STATUS status = NV_OK;
uvm_assert_mutex_locked(&range_tree->base.lock);
dmabuf_map = uvm_dma_buf_gpu_map_iter_first(dmabuf_range, gpu, start, end);
while (dmabuf_map) {
if (start > dmabuf_map->node.start) {
status = uvm_dma_buf_gpu_map_split(&range_tree->base.tree, dmabuf_map, start - 1, &dmabuf_map_next);
if (status != NV_OK)
break;
}
else {
if (end < dmabuf_map->node.end) {
status = uvm_dma_buf_gpu_map_split(&range_tree->base.tree, dmabuf_map, end, NULL);
if (status != NV_OK)
break;
dmabuf_map_next = NULL;
}
else {
dmabuf_map_next = uvm_dma_buf_gpu_map_iter_next(dmabuf_range, dmabuf_map, end);
}
uvm_dma_buf_gpu_map_destroy(dmabuf_range, dmabuf_map);
}
dmabuf_map = dmabuf_map_next;
}
return status;
}
static inline void uvm_dma_buf_pte_maker_data_init(uvm_gpu_t *mapping_gpu,
struct sg_table *sg_table,
NvU64 start_offset,
uvm_dma_buf_pte_maker_data_t *out)
{
memset(out, 0, sizeof(*out));
out->mapping_gpu = mapping_gpu;
__sg_page_iter_start(&out->dma_iter.base,
sg_table->sgl,
sg_table->nents,
start_offset >> PAGE_SHIFT);
}
static NvU64 uvm_dma_buf_pte_maker(uvm_page_table_range_vec_t *range_vec,
NvU64 offset,
void *caller_data)
{
uvm_dma_buf_pte_maker_data_t *data = caller_data;
dma_addr_t dma_addr;
NvU64 gpu_addr;
if (!__sg_page_iter_dma_next(&data->dma_iter)) {
UVM_ERR_PRINT("DMA-BUF sg_table exhausted before all PTEs were written, GPU %s, offset %llu\n",
uvm_gpu_name(data->mapping_gpu),
offset);
return 0;
}
dma_addr = uvm_sg_page_iter_dma_address(&data->dma_iter);
gpu_addr = uvm_parent_gpu_dma_addr_to_gpu_addr(data->mapping_gpu->parent, dma_addr);
// TODO: Bug 6172651: Revisit unconditional issue of RWA permissions.
return range_vec->tree->hal->make_pte(UVM_APERTURE_SYS,
gpu_addr,
UVM_PROT_READ_WRITE_ATOMIC,
UVM_MMU_PTE_FLAGS_ACCESS_COUNTERS_DISABLED);
}
// Implementation of the .invalidate_mappings() callback. It is required to
// signal that the memory backing the DMA-BUF is going to move, and so DMA
// mappings must be removed. The following handler is invoked per-GPU prior to
// the exporter revoking accesses to the buffer, and so also signals that
// no new mappings may be established. This callback may be safely issued
// multiple times, though subsequent calls will be no-ops.
static void uvm_dma_buf_invalidate_mappings(struct dma_buf_attachment *attach)
{
uvm_dma_buf_importer_priv_t *priv = attach->importer_priv;
uvm_va_space_t *va_space;
uvm_va_range_dma_buf_t *dmabuf_range;
uvm_dma_buf_gpu_range_tree_t *range_tree;
uvm_dma_buf_gpu_map_t *dmabuf_map, *dmabuf_map_next;
uvm_gpu_t *gpu;
// The exporter calls this callback with the dma_resv lock held.
uvm_record_lock_dma_resv();
dma_resv_assert_held(attach->dmabuf->resv);
UVM_ASSERT(priv);
va_space = priv->va_space;
uvm_va_space_down_read(va_space);
dmabuf_range = uvm_va_range_dma_buf_find(va_space, priv->base);
if (!dmabuf_range || dmabuf_range->dmabuf != attach->dmabuf) {
uvm_va_space_up_read(va_space);
goto invalidate_no_range;
}
// All GMMU mappings will have been destroyed if the GPU is no longer
// registered but had previously been attached.
if (!uvm_processor_mask_test(&va_space->registered_gpus, priv->gpu_id)) {
uvm_va_space_up_read(va_space);
goto invalidate_no_range;
}
gpu = uvm_gpu_get(priv->gpu_id);
UVM_ASSERT(gpu);
range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu);
uvm_mutex_lock(&range_tree->base.lock);
UVM_ASSERT(range_tree->attach == attach);
if (uvm_processor_mask_test(&dmabuf_range->mapped_gpus, gpu->id)) {
// Revoke GMMU mappings before calling dma_buf_unmap_attachment(), which is
// required of the .invalidate_mappings() callback.
uvm_dma_buf_gpu_map_for_each_safe(dmabuf_map, dmabuf_map_next, dmabuf_range, gpu)
uvm_dma_buf_gpu_map_destroy(dmabuf_range, dmabuf_map);
uvm_processor_mask_clear_atomic(&dmabuf_range->mapped_gpus, gpu->id);
}
uvm_processor_mask_set_atomic(&dmabuf_range->revoked_gpus, gpu->id);
uvm_mutex_unlock(&range_tree->base.lock);
uvm_va_space_up_read(va_space);
invalidate_no_range:
// Since the invalidate_mappings() callback is exporter-initiated, it may
// be called even after UVM has ceased to track or accept mappings for this
// GPU. In such an instance, the importer is still obligated to call
// dma_buf_unmap_attachment().
if (priv->sg_table) {
dma_buf_unmap_attachment(attach, priv->sg_table, DMA_BIDIRECTIONAL);
priv->sg_table = NULL;
}
uvm_record_unlock_dma_resv();
}
static const struct dma_buf_attach_ops uvm_dma_buf_importer_ops = {
.allow_peer2peer = true,
#if defined(NV_DMA_BUF_ATTACH_OPS_HAS_INVALIDATE_MAPPINGS)
.invalidate_mappings = uvm_dma_buf_invalidate_mappings,
#else
.move_notify = uvm_dma_buf_invalidate_mappings,
#endif
};
static NV_STATUS uvm_map_dma_buf_on_gpu(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *mapping_gpu,
NvU64 base,
NvU64 length,
struct dma_buf_attachment *new_attach,
bool *attachment_consumed)
{
uvm_va_space_t *va_space = dmabuf_range->va_range.va_space;
uvm_dma_buf_gpu_map_t *dmabuf_map = NULL;
uvm_dma_buf_gpu_range_tree_t *range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, mapping_gpu);
uvm_gpu_va_space_t *gpu_va_space = uvm_gpu_va_space_get(va_space, mapping_gpu);
uvm_dma_buf_importer_priv_t *attach_priv;
uvm_page_tree_t *page_tree;
uvm_page_table_range_vec_t *pt_range_vec = NULL;
uvm_dma_buf_pte_maker_data_t pte_maker_data;
bool previous_mapping;
NV_STATUS status;
uvm_assert_rwsem_locked_read(&va_space->lock);
uvm_mutex_lock(&range_tree->base.lock);
if (uvm_processor_mask_test(&dmabuf_range->revoked_gpus, mapping_gpu->id)) {
uvm_mutex_unlock(&range_tree->base.lock);
return NV_ERR_INVALID_ADDRESS;
}
if (!range_tree->attach) {
range_tree->attach = new_attach;
*attachment_consumed = true;
}
attach_priv = range_tree->attach->importer_priv;
status = uvm_unmap_dma_buf_in_range(dmabuf_range, mapping_gpu, base, base + length - 1);
if (status != NV_OK)
goto error;
// Create the dma mappings if this is the first mapping on this GPU.
UVM_ASSERT(attach_priv->sg_table || !uvm_processor_mask_test(&dmabuf_range->mapped_gpus, mapping_gpu->id));
if (!attach_priv->sg_table) {
struct sg_table *sg_table;
dma_resv_assert_held(dmabuf_range->dmabuf->resv);
sg_table = dma_buf_map_attachment(range_tree->attach,
DMA_BIDIRECTIONAL);
if (IS_ERR(sg_table)) {
status = (PTR_ERR(sg_table) == -ENOMEM) ? NV_ERR_NO_MEMORY
: NV_ERR_INVALID_DEVICE;
goto error;
}
attach_priv->sg_table = sg_table;
}
// Now create the GMMU mappings for the requested sub-region, after all
// per-GPU initialisation steps have completed.
dmabuf_map = uvm_kvmalloc_zero(sizeof(*dmabuf_map));
if (!dmabuf_map) {
status = NV_ERR_NO_MEMORY;
goto error;
}
dmabuf_map->node.start = base;
dmabuf_map->node.end = base + length - 1;
RB_CLEAR_NODE(&dmabuf_map->node.rb_node);
// Due to the fact that any overlapping mappings were already unmapped,
// adding the new mapping to the tree cannot fail.
status = uvm_range_tree_add(&range_tree->base.tree, &dmabuf_map->node);
UVM_ASSERT(status == NV_OK);
previous_mapping = uvm_processor_mask_test(&dmabuf_range->mapped_gpus, mapping_gpu->id);
uvm_processor_mask_set_atomic(&dmabuf_range->mapped_gpus, mapping_gpu->id);
dmabuf_map->gpu = mapping_gpu;
UVM_ASSERT(uvm_va_range_dma_buf_get_gpu_map_or_null(dmabuf_range, mapping_gpu, base) == dmabuf_map);
UVM_ASSERT(gpu_va_space);
page_tree = &gpu_va_space->page_tables;
pt_range_vec = &dmabuf_map->pt_range_vec;
// TODO: Bug 6164231: Currently mappings are of uniform size as the system
// page size. This is due to the fact that the sg_tables returned from
// dma_buf_map_attachment() are only guaranteed to have system page size
// alignment. To better suit larger DMA-BUF memories, larger page sizes
// should be chosen wherever possible.
status = uvm_page_table_range_vec_init(page_tree,
dmabuf_map->node.start,
uvm_range_tree_node_size(&dmabuf_map->node),
PAGE_SIZE,
UVM_PMM_ALLOC_FLAGS_EVICT,
pt_range_vec);
if (status != NV_OK) {
if (!previous_mapping)
uvm_processor_mask_clear_atomic(&dmabuf_range->mapped_gpus, mapping_gpu->id);
goto error;
}
uvm_dma_buf_pte_maker_data_init(mapping_gpu,
attach_priv->sg_table,
base - dmabuf_range->va_range.node.start,
&pte_maker_data);
status = uvm_page_table_range_vec_write_ptes(pt_range_vec,
UVM_MEMBAR_SYS,
uvm_dma_buf_pte_maker,
(void *) &pte_maker_data);
if (status != NV_OK) {
if (!previous_mapping)
uvm_processor_mask_clear_atomic(&dmabuf_range->mapped_gpus, mapping_gpu->id);
goto error;
}
uvm_mutex_unlock(&range_tree->base.lock);
return NV_OK;
error:
uvm_dma_buf_gpu_map_destroy(dmabuf_range, dmabuf_map);
uvm_mutex_unlock(&range_tree->base.lock);
return status;
}
NV_STATUS uvm_api_map_dma_buf(UVM_MAP_DMA_BUF_PARAMS *params, struct file *filp)
{
uvm_gpu_t *mapping_gpu;
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_va_range_dma_buf_t *dmabuf_range;
struct dma_buf *dmabuf;
struct dma_buf_attachment *attach;
uvm_dma_buf_importer_priv_t *importer_priv;
bool attachment_consumed = false;
NV_STATUS status;
if (uvm_api_range_invalid(params->base, params->length))
return NV_ERR_INVALID_ADDRESS;
// Phase 1: This may be the first attempt to map to this GPU, and so it is
// unknown whether this GPU has an attachment. Acquire the read lock on the
// va_space and retain the GPU and dmabuf as they will be needed to attempt
// to attach the GPU.
uvm_va_space_down_read(va_space);
dmabuf_range = uvm_va_range_dma_buf_find(va_space, params->base);
if (!dmabuf_range ||
dmabuf_range->va_range.node.end < params->base + params->length - 1) {
uvm_va_space_up_read(va_space);
return NV_ERR_INVALID_ADDRESS;
}
mapping_gpu = uvm_va_space_get_gpu_by_uuid_with_gpu_va_space(va_space, &params->gpuUuid);
if (!mapping_gpu) {
uvm_va_space_up_read(va_space);
return NV_ERR_INVALID_DEVICE;
}
if (!uvm_gpu_can_address(mapping_gpu, params->base, params->length)) {
uvm_va_space_up_read(va_space);
return NV_ERR_OUT_OF_RANGE;
}
dmabuf = dmabuf_range->dmabuf;
get_dma_buf(dmabuf);
uvm_gpu_retain(mapping_gpu);
uvm_va_space_up_read(va_space);
// Phase 2: Creating the attachment will require taking the DMA-BUF's
// dma_resv lock. This cannot be safely acquired with the va_space lock
// held, as dma_buf_invalidate_mappings() is called by the DMA-BUF interface
// with the dma_resv lock, and must itself acquire the va_space lock for
// read. Eagerly create an attachment, in case this is the first mapping
// on this GPU.
importer_priv = uvm_kvmalloc(sizeof(*importer_priv));
if (!importer_priv) {
status = NV_ERR_NO_MEMORY;
goto release;
}
importer_priv->gpu_id = mapping_gpu->id;
importer_priv->sg_table = NULL;
importer_priv->va_space = va_space;
importer_priv->base = params->base;
attach = uvm_dma_buf_attach(dmabuf,
&mapping_gpu->parent->pci_dev->dev,
&uvm_dma_buf_importer_ops,
importer_priv);
if (IS_ERR(attach)) {
uvm_kvfree(importer_priv);
status = (PTR_ERR(attach) == -ENOMEM) ? NV_ERR_NO_MEMORY
: NV_ERR_INVALID_DEVICE;
goto release;
}
// Phase 3: Re-acquire the va_space with read and revalidate - if the
// DMA-BUF or GPU is no longer in use in this va_space then remove the
// attachment. Acquire DMA-BUF lock first so that we dma mappings can be
// established if necessary with dma_buf_map_attachment().
uvm_dma_resv_lock(dmabuf->resv);
uvm_va_space_down_read(va_space);
dmabuf_range = uvm_va_range_dma_buf_find(va_space, params->base);
if (!dmabuf_range ||
dmabuf_range->va_range.node.end < params->base + params->length - 1 ||
dmabuf_range->dmabuf != dmabuf) {
status = NV_ERR_INVALID_ADDRESS;
goto unlock_detach;
}
if (!uvm_processor_mask_test(&va_space->registered_gpus, mapping_gpu->id)) {
status = NV_ERR_INVALID_DEVICE;
goto unlock_detach;
}
status = uvm_map_dma_buf_on_gpu(dmabuf_range,
mapping_gpu,
params->base,
params->length,
attach,
&attachment_consumed);
unlock_detach:
uvm_va_space_up_read(va_space);
uvm_dma_resv_unlock(dmabuf->resv);
// On failure, or if otherwise the mapping_gpu was attached by a previous
// mapping, drop the new attachment after releasing all locks. Attachments
// are otherwise freed at UvmFree().
if (!attachment_consumed) {
uvm_dma_buf_detach(dmabuf, attach);
uvm_kvfree(importer_priv);
}
release:
uvm_gpu_release(mapping_gpu);
dma_buf_put(dmabuf);
return status;
}
static void uvm_dma_buf_gpu_map_destroy(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_dma_buf_gpu_map_t *dmabuf_map)
{
NV_STATUS status;
uvm_dma_buf_gpu_range_tree_t *range_tree;
if (!dmabuf_map)
return;
UVM_ASSERT(uvm_gpu_va_space_get(dmabuf_range->va_range.va_space, dmabuf_map->gpu));
range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, dmabuf_map->gpu);
// The DMA-BUF map is inserted into the tree prior to the rest of the
// mapping steps. So, if it has not been inserted yet, there is nothing to
// clean up. Just free the memory.
if (RB_EMPTY_NODE(&dmabuf_map->node.rb_node)) {
uvm_kvfree(dmabuf_map);
return;
}
uvm_assert_mutex_locked(&range_tree->base.lock);
uvm_range_tree_remove(&range_tree->base.tree, &dmabuf_map->node);
// Unmap the PTEs
if (dmabuf_map->pt_range_vec.ranges) {
status = uvm_page_table_range_vec_clear_ptes(&dmabuf_map->pt_range_vec, UVM_MEMBAR_SYS);
if (status != NV_OK)
UVM_ERR_PRINT("Clearing PTEs failed: %s, GPU %s\n",
nvstatusToString(status),
uvm_gpu_name(dmabuf_map->gpu));
UVM_ASSERT(status == NV_OK);
uvm_page_table_range_vec_deinit(&dmabuf_map->pt_range_vec);
}
UVM_ASSERT(!dmabuf_map->pt_range_vec.ranges);
uvm_kvfree(dmabuf_map);
}
void uvm_dma_buf_gpu_range_tree_deinit(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
struct list_head *deferred_free_list)
{
uvm_dma_buf_gpu_range_tree_t *range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu);
uvm_dma_buf_gpu_map_t *dmabuf_map, *dmabuf_map_next;
struct dma_buf_attachment *attach = NULL;
uvm_dma_buf_importer_priv_t *importer_priv;
uvm_dma_buf_attach_deferred_t *deferred;
uvm_mutex_lock(&range_tree->base.lock);
if (uvm_processor_mask_test(&dmabuf_range->mapped_gpus, gpu->id)) {
uvm_dma_buf_gpu_map_for_each_safe(dmabuf_map, dmabuf_map_next, dmabuf_range, gpu)
uvm_dma_buf_gpu_map_destroy(dmabuf_range, dmabuf_map);
uvm_processor_mask_clear_atomic(&dmabuf_range->mapped_gpus, gpu->id);
}
// If this GPU ever received mappings, defer detachment to the DMA-BUF until
// after the va_space lock is released.
if (range_tree->attach) {
attach = range_tree->attach;
range_tree->attach = NULL;
UVM_ASSERT(deferred_free_list);
}
uvm_mutex_unlock(&range_tree->base.lock);
if (!attach)
return;
importer_priv = attach->importer_priv;
deferred = &importer_priv->deferred;
// Defer close of the DMA-BUF until the attachment is torn down following
// release of the va_space lock.
get_dma_buf(dmabuf_range->dmabuf);
deferred->dmabuf = dmabuf_range->dmabuf;
deferred->attach = attach;
deferred->gpu = gpu;
uvm_deferred_free_object_add(deferred_free_list,
&deferred->deferred_free,
UVM_DEFERRED_FREE_OBJECT_TYPE_DMA_BUF_ATTACHMENT);
}
void uvm_va_range_dma_buf_attach_deferred_free(uvm_dma_buf_attach_deferred_t *deferred)
{
uvm_dma_buf_importer_priv_t *importer_priv = deferred->attach->importer_priv;
// Under the dma_resv lock, check that this is the first time that
// dma_buf_unmap_attachment() has been called.
uvm_dma_resv_lock(deferred->dmabuf->resv);
if (importer_priv->sg_table) {
dma_buf_unmap_attachment(deferred->attach, importer_priv->sg_table, DMA_BIDIRECTIONAL);
importer_priv->sg_table = NULL;
}
uvm_dma_resv_unlock(deferred->dmabuf->resv);
uvm_dma_buf_detach(deferred->dmabuf, deferred->attach);
dma_buf_put(deferred->dmabuf);
uvm_kvfree(importer_priv);
}
NV_STATUS uvm_api_unmap_dma_buf(UVM_UNMAP_DMA_BUF_PARAMS *params, struct file *filp)
{
uvm_gpu_t *gpu;
uvm_va_space_t *va_space = uvm_va_space_get(filp);
uvm_va_range_dma_buf_t *dmabuf_range;
uvm_dma_buf_gpu_range_tree_t *range_tree;
NV_STATUS status = NV_OK;
// The DMA-BUF interface provides a sg_table of DMA mapped regions through
// dma_buf_map_attachment(), with base addresses and lengths required to be
// aligned to the system page size. Assert the same here.
if (uvm_api_range_invalid(params->base, params->length))
return NV_ERR_INVALID_ADDRESS;
uvm_va_space_down_read(va_space);
dmabuf_range = uvm_va_range_dma_buf_find(va_space, params->base);
if (!dmabuf_range ||
dmabuf_range->va_range.node.end < params->base + params->length - 1) {
uvm_va_space_up_read(va_space);
return NV_ERR_INVALID_ADDRESS;
}
gpu = uvm_va_space_get_gpu_by_uuid(va_space, &params->gpuUuid);
if (!gpu) {
uvm_va_space_up_read(va_space);
return NV_ERR_INVALID_DEVICE;
}
range_tree = uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu);
uvm_mutex_lock(&range_tree->base.lock);
status = uvm_unmap_dma_buf_in_range(dmabuf_range, gpu, params->base, params->base + params->length - 1);
uvm_mutex_unlock(&range_tree->base.lock);
uvm_va_space_up_read(va_space);
return status;
}
#else
NV_STATUS uvm_api_import_dma_buf(UVM_IMPORT_DMA_BUF_PARAMS *params, struct file *filp)
{
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS uvm_api_map_dma_buf(UVM_MAP_DMA_BUF_PARAMS *params, struct file *filp)
{
return NV_ERR_NOT_SUPPORTED;
}
NV_STATUS uvm_api_unmap_dma_buf(UVM_UNMAP_DMA_BUF_PARAMS *params, struct file *filp)
{
return NV_ERR_NOT_SUPPORTED;
}
void uvm_dma_buf_gpu_range_tree_deinit(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
struct list_head *deferred_free_list) {}
void uvm_va_range_dma_buf_attach_deferred_free(uvm_dma_buf_attach_deferred_t *deferred) {}
#endif // UVM_PROVIDES_DMA_BUF_IMPORTER()
NV_STATUS uvm_api_is_dma_buf_import_supported(UVM_IS_DMA_BUF_IMPORT_SUPPORTED_PARAMS *params, struct file *filp)
{
params->dmaBufImportSupported = UVM_PROVIDES_DMA_BUF_IMPORTER();
return NV_OK;
}
/*
* MODULE_IMPORT_NS() is added by commit id 8651ec01daeda
* ("module: add support for symbol namespaces") in 5.4
*/
#if defined(MODULE_IMPORT_NS)
/*
* DMA_BUF namespace is added by commit id 16b0314aa746
* ("dma-buf: move dma-buf symbols into the DMA_BUF module namespace") in 5.16
*/
#if defined(NV_MODULE_IMPORT_NS_TAKES_CONSTANT)
MODULE_IMPORT_NS(DMA_BUF);
#else
MODULE_IMPORT_NS("DMA_BUF");
#endif // defined(NV_MODULE_IMPORT_NS_TAKES_CONSTANT)
#endif // defined(MODULE_IMPORT_NS)
kernel-open/nvidia-uvm/uvm_va_range_dmabuf.h
+170
View File
@@ -0,0 +1,170 @@
/*******************************************************************************
Copyright (c) 2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
deal in the Software without restriction, including without limitation the
rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
sell copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
DEALINGS IN THE SOFTWARE.
*******************************************************************************/
//
// uvm_va_range_dmabuf.h
//
// This file contains the UVM implementation of the DMA-BUF kernel interface.
//
#ifndef __UVM_VA_RANGE_DMABUF_H__
#define __UVM_VA_RANGE_DMABUF_H__
#include "uvm_linux.h"
#include "uvm_forward_decl.h"
#include "uvm_va_range.h"
#include "uvm_tracker.h"
#include "nv_uvm_types.h"
#include "uvm_types.h"
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
#include <linux/dma-buf.h>
#include <linux/device.h>
#endif
static uvm_dma_buf_gpu_range_tree_t *uvm_dma_buf_gpu_range_tree(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu)
{
return &dmabuf_range->gpu_ranges[uvm_id_gpu_index(gpu->id)];
}
// The caller must hold the range tree lock.
static uvm_dma_buf_gpu_map_t *uvm_dma_buf_gpu_map_iter_first(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
NvU64 start,
NvU64 end)
{
UVM_ASSERT(start >= dmabuf_range->va_range.node.start);
UVM_ASSERT(end <= dmabuf_range->va_range.node.end);
return uvm_dma_buf_gpu_map_container(
uvm_gpu_range_tree_iter_first(&uvm_dma_buf_gpu_range_tree(dmabuf_range, gpu)->base, start, end));
}
// The caller must hold the range tree lock.
static uvm_dma_buf_gpu_map_t *uvm_dma_buf_gpu_map_iter_next(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_dma_buf_gpu_map_t *dmabuf_gpu_map,
NvU64 end)
{
if (!dmabuf_gpu_map)
return NULL;
UVM_ASSERT(end <= dmabuf_range->va_range.node.end);
return uvm_dma_buf_gpu_map_container(
uvm_gpu_range_tree_iter_next(&uvm_dma_buf_gpu_range_tree(dmabuf_range, dmabuf_gpu_map->gpu)->base,
dmabuf_gpu_map, end));
}
// The four iterators below require that the caller hold the gpu's range tree
// lock.
#define uvm_dma_buf_gpu_map_for_each_in(dmabuf_gpu_map, dmabuf_range, gpu, start, end) \
uvm_gpu_map_for_each_in(dmabuf_gpu_map, dmabuf_range, gpu, start, end, \
uvm_dma_buf_gpu_map_iter_first, uvm_dma_buf_gpu_map_iter_next)
#define uvm_dma_buf_gpu_map_for_each_in_safe(dmabuf_gpu_map, dmabuf_gpu_map_next, dmabuf_range, gpu, start, end) \
uvm_gpu_map_for_each_in_safe(dmabuf_gpu_map, dmabuf_gpu_map_next, dmabuf_range, gpu, start, end, \
uvm_dma_buf_gpu_map_iter_first, uvm_dma_buf_gpu_map_iter_next)
#define uvm_dma_buf_gpu_map_for_each(dmabuf_gpu_map, dmabuf_range, gpu) \
uvm_dma_buf_gpu_map_for_each_in(dmabuf_gpu_map, dmabuf_range, gpu, \
(dmabuf_range)->va_range.node.start, \
(dmabuf_range)->va_range.node.end)
#define uvm_dma_buf_gpu_map_for_each_safe(dmabuf_gpu_map, dmabuf_gpu_map_next, dmabuf_range, gpu) \
uvm_dma_buf_gpu_map_for_each_in_safe(dmabuf_gpu_map, dmabuf_gpu_map_next, dmabuf_range, gpu, \
(dmabuf_range)->va_range.node.start, \
(dmabuf_range)->va_range.node.end)
// The release of the attachment must occur after the va_space lock has been
// released at tear-down. This deferred_free object holds the retained gpu as
// well as a reference to the dmabuf which will be put on free.
typedef struct
{
struct dma_buf *dmabuf;
struct dma_buf_attachment *attach;
uvm_gpu_t *gpu;
uvm_deferred_free_object_t deferred_free;
} uvm_dma_buf_attach_deferred_t;
// Per attachment (per-GPU, per DMA-BUF import) state. Permits late unmap of
// DMA-BUF mappings. Available for use through deferred-free through the
// attachment.
typedef struct
{
// Reference to the attachment GPU, which is retained until the attachment
// is destroyed.
uvm_gpu_id_t gpu_id;
// Reference to the dma mappings retrieved through dma_buf_map_attachment(),
// To be revoked either at deferred-free or through the importer
// .invalidate_mappings callback in keeping with the DMA-BUF API.
// If this pointer is observed to not be NULL under the dma_resv lock, it
// can be revoked.
struct sg_table *sg_table;
// The va_space and base address of the first mapping are sufficient to
// identify the DMA-BUF range under the va_space lock. The importer's
// attachments to the DMA-BUF can outlive its va_range, in which case the
// sg_table will be torn down directly, as no GMMU mappings will exist.
uvm_va_space_t *va_space;
NvU64 base;
uvm_dma_buf_attach_deferred_t deferred;
} uvm_dma_buf_importer_priv_t;
// Destroy all GPU mapping associated with the VA range. Defer destruction of
// DMA mappings through call to dma_buf_unmap_attachment(), until after the
// va_space lock is dropped.
// LOCKING: The va_space lock must be held in at least read mode.
void uvm_dma_buf_gpu_range_tree_deinit(uvm_va_range_dma_buf_t *dmabuf_range,
uvm_gpu_t *gpu,
struct list_head *deferred_free_list);
// Run a deferred dma_buf_detach() and free the wrapper. Invoked by the
// deferred-free dispatcher in uvm_va_space.c.
void uvm_va_range_dma_buf_attach_deferred_free(uvm_dma_buf_attach_deferred_t *deferred);
#if UVM_PROVIDES_DMA_BUF_IMPORTER()
static inline struct dma_buf_attachment *uvm_dma_buf_attach(struct dma_buf *dmabuf,
struct device *dev,
const struct dma_buf_attach_ops *ops,
void *importer_priv)
{
struct dma_buf_attachment *ret;
uvm_record_lock_dma_resv();
ret = dma_buf_dynamic_attach(dmabuf, dev, ops, importer_priv);
uvm_record_unlock_dma_resv();
return ret;
}
static inline void uvm_dma_buf_detach(struct dma_buf *dmabuf,
struct dma_buf_attachment *attach)
{
uvm_record_lock_dma_resv();
dma_buf_detach(dmabuf, attach);
uvm_record_unlock_dma_resv();
}
#endif // UVM_PROVIDES_DMA_BUF_IMPORTER()
#endif // __UVM_VA_RANGE_DMABUF_H__
kernel-open/nvidia-uvm/uvm_va_space.c
+78 -12
View File
@@ -33,6 +33,7 @@
#include "uvm_thread_context.h"
#include "uvm_hal.h"
#include "uvm_map_external.h"
#include "uvm_va_range_dmabuf.h"
#include "uvm_ats.h"
#include "uvm_gpu_replayable_faults.h"
#include "uvm_gpu_non_replayable_faults.h"
@@ -46,12 +47,10 @@
#include "uvm_hmm.h"
#include <linux/mmzone.h>
static bool uvm_disable_sam_migration = false;
static bool uvm_disable_sam_migration = true;
MODULE_PARM_DESC(uvm_disable_sam_migration,
"Disable migration of system allocated memory for CDMM/HMM "
"Default: false (migration is enabled if possible). "
"However, even with uvm_disable_sam_migration=false, migation "
"will not be enabled if is not supported in this driver build ");
"Default: true (migration is disabled). ");
module_param(uvm_disable_sam_migration, bool, 0444);
static bool processor_mask_array_test(const uvm_processor_mask_t *mask,
@@ -364,13 +363,14 @@ static void va_space_parent_gpu_unregister(uvm_va_space_t *va_space, uvm_parent_
if (!uvm_va_space_single_gpu_in_parent(va_space, parent))
return;
if (parent->egm.enabled && parent->closest_cpu_numa_node != NUMA_NO_NODE) {
if (uvm_parent_gpu_egm_enabled(parent) && parent->closest_cpu_numa_node != NUMA_NO_NODE) {
uvm_egm_numa_node_info_t *node_info = uvm_va_space_get_egm_numa_node_info(va_space,
parent->closest_cpu_numa_node);
uvm_parent_processor_mask_clear(&node_info->parent_gpus, parent->id);
// Clear local EGM routing
node_info->routing_table[uvm_parent_id_gpu_index(parent->id)] = NULL;
if (uvm_parent_gpu_egm_is_legacy(parent))
node_info->routing_table[uvm_parent_id_gpu_index(parent->id)] = NULL;
}
if (uvm_parent_processor_mask_test(&va_space->test.non_replayable_delay_set, parent->id)) {
@@ -752,8 +752,7 @@ uvm_gpu_t *uvm_va_space_get_gpu_by_mem_info(uvm_va_space_t *va_space, const UvmG
uvm_gpu_t *gpu;
uvm_assert_rwsem_locked(&va_space->lock);
for_each_va_space_gpu(gpu, va_space) {
for_each_va_space_gpu (gpu, va_space) {
if (uvm_uuid_eq(&gpu->uuid, &mem_info->uuid) ||
(gpu->parent->smc.enabled && uvm_uuid_eq(&gpu->parent->uuid, &mem_info->uuid)))
return gpu;
@@ -766,8 +765,16 @@ static void va_space_parent_gpu_register(uvm_va_space_t *va_space, uvm_parent_gp
{
uvm_egm_numa_node_info_t *node_info;
// If EGM is enabled and is not "legacy" EGM, we don't exit early.
// The GPU needs to be set in the parents_gpus processor mask
// since it's connected to the local node. Even if EGM is not
// enabled on the GPU, it could access memory on remote nodes
// through GPUs that are EGM-enabled.
// Being in the parent_gpus mask is required in order to
// successfully find the correct routing table if the GPU is
// not EGM-enabled.
if (!uvm_va_space_single_gpu_in_parent(va_space, parent) ||
!parent->egm.enabled ||
(uvm_parent_gpu_egm_is_legacy(parent) && !uvm_parent_gpu_egm_enabled(parent)) ||
parent->closest_cpu_numa_node == -1)
return;
@@ -784,7 +791,8 @@ static void va_space_parent_gpu_register(uvm_va_space_t *va_space, uvm_parent_gp
// This is done here because local EGM routing does need not any peers.
// So, if there are no peers to this GPU, local EGM accesses should
// still be possible.
if (parent->egm.enabled)
// Local EGM accesses are only done on "legacy" EGM.
if (uvm_parent_gpu_egm_enabled(parent) && uvm_parent_gpu_egm_is_legacy(parent))
node_info->routing_table[uvm_parent_id_gpu_index(parent->id)] = parent;
}
@@ -842,8 +850,21 @@ NV_STATUS uvm_va_space_register_gpu(uvm_va_space_t *va_space,
va_space->pageable.cdmm_enabled = g_uvm_global.cdmm_enabled;
}
// HACK: Kernels with the bug UVM_CAN_USE_DEVICE_PRIVATE_MEMREMAP_PAGES
// tests for requries us to disable migration on non-coherent GPUs.
// Ideally, we'd reject registering those GPUs on a VA space with
// migration enabled. Unfortunately, doing this would require a big
// rework our test suite.
//
// Fixing the tests is fairly involved, so the hackfix we're doing here
// is unconditionally disabling migration if there are any non-ATS
// (i.e. non-coherent) devices in the system. This still allows for
// migration on systems that have only coherent devices.
//
// TODO: Bug 5969467
if (va_space->pageable.cdmm_enabled &&
va_space->pageable.migrations_enabled &&
g_uvm_global.have_non_ats_devices &&
!UVM_CAN_USE_DEVICE_PRIVATE_MEMREMAP_PAGES()) {
va_space->pageable.migrations_enabled = NV_FALSE;
}
@@ -1154,8 +1175,33 @@ static void disable_egm_peers(uvm_va_space_t *va_space, uvm_gpu_t *gpu0, uvm_gpu
uvm_parent_processor_mask_t proc_mask;
uvm_parent_gpu_id_t peer_parent_id;
if (!uvm_parent_gpu_egm_is_legacy(gpu0->parent)) {
uvm_va_range_managed_t *va_range;
uvm_for_each_va_range_managed (va_range, va_space) {
uvm_va_block_t *va_block;
for_each_va_block_in_va_range (va_range, va_block)
uvm_va_block_unmap_egm(va_block, gpu0->parent);
}
}
uvm_parent_processor_mask_copy(&proc_mask, &node_info->parent_gpus);
uvm_parent_processor_mask_clear(&proc_mask, gpu0->parent->id);
// EGM requires creating mappings on routing GPUs. This requires
// quiescing all access to remote EGM memory. In turn, this make
// re-assigning new remote GPU as the routing GPU racy.
// "Legacy" EGM, on the other hand, does not need any remapping,
// re-routing is only based on simple address calculation.
// So, for non-legacy EGM, we just remove the routing GPU, even
// though there could be other GPUs connected to the NUMA node.
// TODO: Bug 6196507: Re-assign routing GPU to the next GPU in
// the mask.
if (!uvm_parent_gpu_egm_is_legacy(gpu0->parent)) {
node_info->routing_table[uvm_parent_id_gpu_index(gpu1->parent->id)] = NULL;
continue;
}
peer_parent_id = uvm_parent_processor_mask_find_first_gpu_id(&proc_mask);
if (!UVM_PARENT_ID_IS_VALID(peer_parent_id)) {
node_info->routing_table[uvm_parent_id_gpu_index(gpu1->parent->id)] = NULL;
@@ -1210,7 +1256,7 @@ static void disable_peers(uvm_va_space_t *va_space,
static void enable_egm_peers(uvm_va_space_t *va_space, uvm_gpu_t *gpu0, uvm_gpu_t *gpu1)
{
if (gpu0->parent->egm.enabled) {
if (uvm_parent_gpu_egm_enabled(gpu0->parent)) {
uvm_egm_numa_node_info_t *node_info;
int nid;
@@ -1220,8 +1266,24 @@ static void enable_egm_peers(uvm_va_space_t *va_space, uvm_gpu_t *gpu0, uvm_gpu_
// same NUMA node. Otherwise, we want accesses from it to this CPU NUMA
// node to use gpu1's local EGM accesses.
if (!node_info->routing_table[uvm_parent_id_gpu_index(gpu1->parent->id)] &&
!uvm_parent_processor_mask_test(&node_info->parent_gpus, gpu1->parent->id))
gpu0->parent->closest_cpu_numa_node != gpu1->parent->closest_cpu_numa_node) {
node_info->routing_table[uvm_parent_id_gpu_index(gpu1->parent->id)] = gpu0->parent;
if (!uvm_parent_gpu_egm_is_legacy(gpu0->parent)) {
uvm_va_range_managed_t *va_range;
uvm_for_each_va_range_managed (va_range, va_space) {
uvm_va_block_t *va_block;
for_each_va_block_in_va_range (va_range, va_block) {
// We can ignore the status of uvm_va_block_map_egm().
// It it succeeded, accesses will be over gLinks. If
// not, they will take the cLink path. Either way
// there is no reason to fail the peer setup.
uvm_va_block_map_egm(va_block, gpu0->parent);
}
}
}
}
}
}
}
@@ -2030,6 +2092,10 @@ void uvm_deferred_free_object_list(struct list_head *deferred_free_list)
case UVM_DEFERRED_FREE_OBJECT_TYPE_DEVICE_P2P_MEM:
uvm_va_range_free_device_p2p_mem(container_of(object, uvm_device_p2p_mem_t, deferred_free));
break;
case UVM_DEFERRED_FREE_OBJECT_TYPE_DMA_BUF_ATTACHMENT:
uvm_va_range_dma_buf_attach_deferred_free(
container_of(object, uvm_dma_buf_attach_deferred_t, deferred_free));
break;
default:
UVM_ASSERT_MSG(0, "Invalid type %d\n", object->type);
}
kernel-open/nvidia-uvm/uvm_va_space.h
+3 -3
View File
@@ -1,5 +1,5 @@
/*******************************************************************************
Copyright (c) 2015-2025 NVIDIA Corporation
Copyright (c) 2015-2026 NVIDIA Corporation
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to
@@ -56,6 +56,7 @@ typedef enum
UVM_DEFERRED_FREE_OBJECT_GPU_VA_SPACE,
UVM_DEFERRED_FREE_OBJECT_TYPE_EXTERNAL_ALLOCATION,
UVM_DEFERRED_FREE_OBJECT_TYPE_DEVICE_P2P_MEM,
UVM_DEFERRED_FREE_OBJECT_TYPE_DMA_BUF_ATTACHMENT,
UVM_DEFERRED_FREE_OBJECT_TYPE_COUNT
} uvm_deferred_free_object_type_t;
@@ -881,8 +882,7 @@ NV_STATUS uvm_test_va_space_allow_movable_allocations(UVM_TEST_VA_SPACE_ALLOW_MO
// VM_FAULT_NOPAGE: if page was faulted in OK
// (possibly or'ed with VM_FAULT_MAJOR if a migration was needed).
// VM_FAULT_OOM: if system memory wasn't available.
// VM_FAULT_SIGBUS: if a CPU mapping to fault_addr cannot be accessed,
// for example because it's within a range group which is non-migratable.
// VM_FAULT_SIGBUS: if a CPU mapping to fault_addr cannot be accessed.
vm_fault_t uvm_va_space_cpu_fault_managed(uvm_va_space_t *va_space, struct vm_fault *vmf);
// Handle a CPU fault in the given VA space for a HMM allocation,
kernel-open/nvidia/ioctl_nvswitch.h
+34 -1
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2017-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -105,6 +105,35 @@ typedef struct
NvBool is_compatible;
} NVSWITCH_CHECK_VERSION_PARAMS;
/*
* NVSWITCH_CTL_CHECK_VERSION_V2
*
* V2 version check with explicit mode selection, modeled after RM's
* nv_ioctl_rm_api_version_t (NV_RM_API_VERSION_CMD_STRICT/RELAXED/QUERY).
*
* Parameters:
* cmd[in]
* Version check mode: STRICT, RELAXED, or QUERY.
* user[in]
* Version of the interface that the client is compiled with.
* Ignored when cmd is QUERY.
* kernel[out]
* Version of the interface that the kernel driver is compiled with.
* is_compatible[out]
* Result of version comparison. Always 0 for QUERY mode.
*/
#define NVSWITCH_CHECK_VERSION_CMD_STRICT 0
#define NVSWITCH_CHECK_VERSION_CMD_RELAXED 1
#define NVSWITCH_CHECK_VERSION_CMD_QUERY 2
typedef struct
{
NvU32 cmd;
NVSWITCH_VERSION user;
NVSWITCH_VERSION kernel;
NvBool is_compatible;
} NVSWITCH_CHECK_VERSION_V2_PARAMS;
/*
* Max devices supported by the driver
*
@@ -214,6 +243,7 @@ typedef struct
#define CTRL_NVSWITCH_CHECK_VERSION 0x02
#define CTRL_NVSWITCH_GET_DEVICES_V2 0x03
#define CTRL_NVSWITCH_GET_DEVICE_NODES 0x04
#define CTRL_NVSWITCH_CHECK_VERSION_V2 0x05
/*
* Nvswitchctl (device agnostic) IOCTLs
@@ -231,6 +261,9 @@ typedef struct
#define IOCTL_NVSWITCH_GET_DEVICE_NODES \
NVSWITCH_IOCTL_CODE(NVSWITCH_CTL_IO_TYPE, CTRL_NVSWITCH_GET_DEVICE_NODES, NVSWITCH_GET_DEVICE_NODES_PARAMS, \
NVSWITCH_IO_READ_ONLY)
#define IOCTL_NVSWITCH_CHECK_VERSION_V2 \
NVSWITCH_IOCTL_CODE(NVSWITCH_CTL_IO_TYPE, CTRL_NVSWITCH_CHECK_VERSION_V2, NVSWITCH_CHECK_VERSION_V2_PARAMS, \
NVSWITCH_IO_WRITE_READ)
#ifdef __cplusplus
}
kernel-open/nvidia/libspdm_ecc.c
+1 -1
View File
@@ -301,7 +301,7 @@ static bool lkca_ecdsa_verify_akcipher(void *ec_context, size_t hash_nid,
#else // USE_LKCA
struct ecc_ctx *ctx = ec_context;
u8 *pub_key;
int err;
int err = -EINVAL;
DECLARE_CRYPTO_WAIT(wait);
// Roundabout way
kernel-open/nvidia/linux_nvswitch.c
+90 -12
View File
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2016-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2016-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: MIT
*
* Permission is hereby granted, free of charge, to any person obtaining a
@@ -167,14 +167,15 @@ MODULE_PARM_DESC(NvSwitchBlacklist, "NvSwitchBlacklist=uuid[,uuid...]");
// .ioctl, .poll and other background tasks.
//
// The kernel guarantees that .close won't happen while .ioctl and .poll
// are going on and without successful .open one can't execute any file ops.
// This behavior guarantees correctness of the locking model.
// are going on for the same file. It does not serialize .close against
// operations on other fds for the same device, so .close must take
// nvswitch_dev.device_mutex before touching per-device state that is also
// accessed by those paths.
//
// If .close is invoked and holding the lock which is also used by threaded
// tasks such as interrupt, driver will deadlock while trying to stop such
// tasks. For example, when threaded interrupts are enabled, free_irq() calls
// kthread_stop() to flush pending interrupt tasks. The locking model
// makes sure that such deadlock cases don't happen.
// Do not hold nvswitch_dev.device_mutex across operations that stop
// threaded tasks such as interrupt. For example, when threaded interrupts
// are enabled, free_irq() calls kthread_stop() to flush pending interrupt
// tasks. The locking model makes sure that such deadlock cases don't happen.
//
// Lock ordering:
// nvswitch.driver_mutex
@@ -259,7 +260,7 @@ struct file_operations ctl_fops =
static int nvswitch_initialize_device_interrupt(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_shutdown_device_interrupt(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_load_bar_info(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_task_dispatch(NVSWITCH_DEV *nvswitch_dev);
static void nvswitch_task_dispatch(void *nvswitch_dev);
static NvBool
nvswitch_is_device_blacklisted
@@ -313,7 +314,7 @@ nvswitch_init_background_tasks
NV_ATOMIC_SET(nvswitch_dev->task_q_ready, 1);
nv_kthread_q_item_init(&nvswitch_dev->task_item,
(nv_q_func_t) &nvswitch_task_dispatch,
&nvswitch_task_dispatch,
nvswitch_dev);
if (!nv_kthread_q_schedule_q_item(&nvswitch_dev->task_q,
@@ -955,6 +956,75 @@ nvswitch_ctl_check_version(NVSWITCH_CHECK_VERSION_PARAMS *p)
return 0;
}
static int
nvswitch_ctl_check_version_v2(NVSWITCH_CHECK_VERSION_V2_PARAMS *p)
{
NvlStatus retval;
NvU32 i;
NvU32 u_major_len = 0, k_major_len = 0;
p->is_compatible = 0;
p->user.version[NVSWITCH_VERSION_STRING_LENGTH - 1] = '\0';
if (p->cmd != NVSWITCH_CHECK_VERSION_CMD_STRICT &&
p->cmd != NVSWITCH_CHECK_VERSION_CMD_RELAXED &&
p->cmd != NVSWITCH_CHECK_VERSION_CMD_QUERY)
{
return -EINVAL;
}
retval = nvswitch_lib_check_api_version(p->user.version, p->kernel.version,
NVSWITCH_VERSION_STRING_LENGTH);
if (p->cmd == NVSWITCH_CHECK_VERSION_CMD_QUERY)
{
return 0;
}
if (retval == NVL_SUCCESS)
{
p->is_compatible = 1;
return 0;
}
if (retval != -NVL_ERR_NOT_SUPPORTED)
{
return nvswitch_map_status(retval);
}
if (p->cmd == NVSWITCH_CHECK_VERSION_CMD_RELAXED)
{
for (i = 0; i < NVSWITCH_VERSION_STRING_LENGTH &&
p->user.version[i] != '\0' && p->user.version[i] != '.'; i++)
{
u_major_len++;
}
for (i = 0; i < NVSWITCH_VERSION_STRING_LENGTH &&
p->kernel.version[i] != '\0' && p->kernel.version[i] != '.'; i++)
{
k_major_len++;
}
if (u_major_len == k_major_len &&
strncmp(p->user.version, p->kernel.version, u_major_len) == 0)
{
p->is_compatible = 1;
printk(KERN_DEBUG "nvidia-nvswitch: Minor version mismatch "
"tolerated (process: %s, pid: %d), kernel %s user %s\n",
current->comm, task_pid_nr(current),
p->kernel.version, p->user.version);
return 0;
}
}
printk(KERN_ERR "nvidia-nvswitch: Version mismatch (process: %s, pid: %d), "
"kernel version %s user version %s\n",
current->comm, task_pid_nr(current),
p->kernel.version, p->user.version);
return 0;
}
static void
nvswitch_ctl_get_devices(NVSWITCH_GET_DEVICES_PARAMS *p)
{
@@ -1053,6 +1123,14 @@ nvswitch_ctl_cmd_dispatch
nvswitch_ctl_get_devices_v2(params);
}
break;
case CTRL_NVSWITCH_CHECK_VERSION_V2:
rc = NVSWITCH_CTL_CHECK_PARAMS(NVSWITCH_CHECK_VERSION_V2_PARAMS,
param_size);
if (!rc)
{
rc = nvswitch_ctl_check_version_v2(params);
}
break;
default:
rc = -EINVAL;
@@ -1208,9 +1286,10 @@ nvswitch_isr_thread
static void
nvswitch_task_dispatch
(
NVSWITCH_DEV *nvswitch_dev
void *_nvswitch_dev
)
{
NVSWITCH_DEV *nvswitch_dev = _nvswitch_dev;
NvU64 nsec;
NvU64 timeout;
NvS64 rc;
@@ -2609,7 +2688,6 @@ nvswitch_os_is_fabric_manager
{
nvswitch_file_private_t *private_data = (nvswitch_file_private_t *)osPrivate;
/* Make sure that fabric mgmt capbaility fd is valid */
if ((private_data == NULL) ||
(private_data->capability_fds.fabric_mgmt < 0))
{
kernel-open/nvidia/nv-caps.c
+3 -3
View File
@@ -66,11 +66,11 @@ static nv_cap_table_entry_t g_nv_cap_mig_table[] =
static nv_cap_table_entry_t g_nv_cap_sys_table[] =
{
{"/driver/nvidia/capabilities/fabric-imex-mgmt"}
,
{"/driver/nvidia/capabilities/fabric-imex-mgmt"},
{"/driver/nvidia/capabilities/profiler-device"},
{"/driver/nvidia/capabilities/profiler-context"},
{"/driver/nvidia/capabilities/trace-device"}
{"/driver/nvidia/capabilities/trace-device"},
{"/driver/nvidia/capabilities/wpps-access"},
};
#define NV_CAP_MIG_CI_ENTRIES(_gi) \
kernel-open/nvidia/nv-dma.c
+239
View File
@@ -26,6 +26,7 @@
#include "os-interface.h"
#include "nv-linux.h"
#include "nv-reg.h"
#include <linux/iommu.h>
#if IS_ENABLED(CONFIG_DRM)
#include <drm/drm_device.h>
@@ -156,6 +157,10 @@ NV_STATUS nv_create_dma_map_scatterlist(nv_dma_map_t *dma_map)
NV_STATUS status;
nv_dma_submap_t *submap;
NvU32 i;
#if defined(NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT)
dma_map->mapping.discontig.submap_count = 1;
#else
NvU64 allocated_size = 0;
NvU64 num_submaps = dma_map->page_count + NV_DMA_SUBMAP_MAX_PAGES - 1;
NvU64 total_size = dma_map->page_count << PAGE_SHIFT;
@@ -175,6 +180,7 @@ NV_STATUS nv_create_dma_map_scatterlist(nv_dma_map_t *dma_map)
}
dma_map->mapping.discontig.submap_count = NvU64_LO32(num_submaps);
#endif
status = os_alloc_mem((void **)&dma_map->mapping.discontig.submaps,
sizeof(nv_dma_submap_t) * dma_map->mapping.discontig.submap_count);
@@ -198,10 +204,14 @@ NV_STATUS nv_create_dma_map_scatterlist(nv_dma_map_t *dma_map)
NV_FOR_EACH_DMA_SUBMAP(dma_map, submap, i)
{
#if defined(NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT)
submap->page_count = (NvU32)dma_map->page_count;
#else
NvU64 submap_size = NV_MIN(NV_DMA_SUBMAP_MAX_PAGES << PAGE_SHIFT,
total_size - allocated_size);
submap->page_count = (NvU32)(submap_size >> PAGE_SHIFT);
#endif
status = NV_ALLOC_DMA_SUBMAP_SCATTERLIST(dma_map, submap, i);
if (status != NV_OK)
@@ -218,10 +228,14 @@ NV_STATUS nv_create_dma_map_scatterlist(nv_dma_map_t *dma_map)
}
#endif
#if !(defined(NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT))
allocated_size += submap_size;
#endif
}
#if !(defined(NV_SG_ALLOC_TABLE_FROM_PAGES_SEGMENT_PRESENT))
WARN_ON(allocated_size != total_size);
#endif
if (status != NV_OK)
{
@@ -1034,3 +1048,228 @@ void NV_API_CALL nv_dma_release_sgt
{
}
#endif /* IS_ENABLED(CONFIG_DRM) */
static NvBool nv_is_dma_domain
(
nv_state_t *nv
)
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
struct iommu_domain *domain = iommu_get_domain_for_dev(nvl->dev);
if (domain == NULL)
{
return NV_FALSE;
}
#if defined(NV_IOMMU_IS_DMA_DOMAIN_PRESENT)
return iommu_is_dma_domain(domain);
#else
return (domain->type & __IOMMU_DOMAIN_DMA_API) != 0;
#endif
}
static NV_STATUS _nv_dma_get_sysmem_range
(
nv_state_t *nv,
NvU64 *phys_base,
NvU64 *size
)
{
NvS32 node_id = nv->cpu_numa_node_id;
NvU64 start_pfn = NV_U64_MAX;
NvU64 end_pfn = 0ULL;
struct zone *zone;
struct pglist_data *pgdat;
NvU32 zone_id;
if (node_id < 0 || !node_online(node_id))
{
return NV_ERR_NOT_SUPPORTED;
}
//
// UVM uses ZONE_DEVICE for coherent device pages which adds into the
// available sysmem and the inflated size is included in sysmem accounting
// if node_start_pfn/node_end_pfn are used directly.
// Instead, compute the sysmem range from managed, non-ZONE_DEVICE zones
// so device PFNs don't inflate the result.
//
pgdat = NODE_DATA(node_id);
for (zone_id = 0; zone_id < MAX_NR_ZONES; zone_id++)
{
#ifdef CONFIG_ZONE_DEVICE
if (zone_id == ZONE_DEVICE)
continue;
#endif
zone = &(pgdat->node_zones[zone_id]);
if (!managed_zone(zone))
continue;
start_pfn = min(start_pfn, (NvU64)zone->zone_start_pfn);
end_pfn = max(end_pfn, (NvU64)zone_end_pfn(zone));
}
if (start_pfn >= end_pfn)
{
return NV_ERR_NOT_SUPPORTED;
}
*phys_base = start_pfn << PAGE_SHIFT;
*size = (end_pfn - start_pfn) << PAGE_SHIFT;
return NV_OK;
}
static NV_STATUS nv_dma_map_sysmem_dynamic
(
nv_state_t *nv,
NvU64 phys_size,
NvU64 *dma_addr,
NvU64 *dma_size
)
{
#if NV_IS_EXPORT_SYMBOL_GPL_dma_iova_try_alloc
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
struct dma_iova_state state;
if (!dma_iova_try_alloc(nvl->dev, &state, 0, phys_size))
{
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
"Failed to allocate IOVA region for sysmem size 0x%llx\n",
phys_size);
return NV_ERR_OPERATING_SYSTEM;
}
*dma_addr = (NvU64)state.addr;
*dma_size = (NvU64)dma_iova_size(&state);
NV_DEV_PRINTF(NV_DBG_INFO, nv,
"Sysmem mapped in DMA domain, IOVA base: 0x%llx size: 0x%llx\n",
*dma_addr, *dma_size);
return NV_OK;
#else
return NV_ERR_NOT_SUPPORTED;
#endif // NV_IS_EXPORT_SYMBOL_GPL_dma_iova_try_alloc
}
static NV_STATUS nv_dma_map_sysmem_identity
(
nv_state_t *nv,
NvU64 phys_base,
NvU64 phys_size,
NvU64 *dma_addr,
NvU64 *dma_size
)
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
dma_addr_t addr;
addr = dma_map_single(nvl->dev, phys_to_virt(phys_base),
phys_size, DMA_BIDIRECTIONAL);
if (dma_mapping_error(nvl->dev, addr))
{
NV_DEV_PRINTF(NV_DBG_ERRORS, nv,
"Local CPU NUMA sysmem mapping failed for phys: 0x%llx size: 0x%llx\n",
phys_base, phys_size);
return NV_ERR_OPERATING_SYSTEM;
}
*dma_addr = (NvU64)addr;
*dma_size = phys_size;
NV_DEV_PRINTF(NV_DBG_INFO, nv,
"Local CPU NUMA sysmem mapped in identity/off mode, addr: 0x%llx size: 0x%llx\n",
*dma_addr, *dma_size);
return NV_OK;
}
NV_STATUS NV_API_CALL nv_dma_init_sysmem_window_for_fabric_access
(
nv_state_t *nv,
NvU64 alignment,
NvU64 *dma_addr,
NvU64 *dma_size,
NvBool *dma_identity
)
{
NvU64 sysmem_base, sysmem_size;
NV_STATUS status;
if ((dma_addr == NULL) ||
(dma_size == NULL) ||
(dma_identity == NULL) ||
(alignment == 0))
{
return NV_ERR_INVALID_ARGUMENT;
}
status = _nv_dma_get_sysmem_range(nv, &sysmem_base, &sysmem_size);
if (status != NV_OK)
{
return status;
}
else if (sysmem_size == 0)
{
return NV_ERR_INVALID_STATE;
}
NV_DEV_PRINTF(NV_DBG_INFO, nv,
"Local CPU NUMA sysmem phys base: 0x%llx size: 0x%llx\n",
sysmem_base, sysmem_size);
if (nv_is_dma_domain(nv))
{
status = nv_dma_map_sysmem_dynamic(nv, NV_ALIGN_UP64(sysmem_size, alignment),
dma_addr, dma_size);
*dma_identity = NV_FALSE;
}
else
{
status = nv_dma_map_sysmem_identity(nv, sysmem_base, sysmem_size,
dma_addr, dma_size);
*dma_identity = NV_TRUE;
}
return status;
}
void NV_API_CALL nv_dma_destroy_sysmem_window_for_fabric_access
(
nv_state_t *nv,
NvU64 dma_addr,
NvU64 dma_size
)
{
nv_linux_state_t *nvl = NV_GET_NVL_FROM_NV_STATE(nv);
struct device *dev = nvl->dev;
if (!nv_is_dma_domain(nv))
{
NV_DEV_PRINTF(NV_DBG_INFO, nv,
"Unmapping local CPU NUMA sysmem DMA addr: 0x%llx size: 0x%llx\n",
dma_addr, dma_size);
dma_unmap_single(dev, (dma_addr_t)dma_addr,
(size_t)dma_size, DMA_BIDIRECTIONAL);
}
else
{
#if NV_IS_EXPORT_SYMBOL_GPL_dma_iova_try_alloc
struct dma_iova_state state;
state.addr = (dma_addr_t)dma_addr;
state.__size = (u64)dma_size;
NV_DEV_PRINTF(NV_DBG_INFO, nv,
"Freeing local NUMA sysmem IOVA range addr: 0x%llx size: 0x%llx\n",
dma_addr, dma_size);
dma_iova_free(dev, &state);
#endif
}
}

Some files were not shown because too many files have changed in this diff Show More