diff --git a/pkgs/kernel/hbiyik-panthor.patch b/pkgs/kernel/hbiyik-panthor.patch new file mode 100644 index 0000000..71a604d --- /dev/null +++ b/pkgs/kernel/hbiyik-panthor.patch @@ -0,0 +1,37573 @@ +From c57c1cc5d3d272cab131dc9c07ffa4b2fbfb8eb7 Mon Sep 17 00:00:00 2001 +From: Liviu Dudau +Date: Mon, 17 Jul 2023 13:12:25 +0100 +Subject: [PATCH 001/150] dt-bindings: gpu: mali-valhall-csf: Add support for + Arm Mali CSF GPUs + +Arm has introduced a new v10 GPU architecture that replaces the Job Manager +interface with a new Command Stream Frontend. It adds firmware driven +command stream queues that can be used by kernel and user space to submit +jobs to the GPU. + +Add the initial schema for the device tree that is based on support for +RK3588 SoC. The minimum number of clocks is one for the IP, but on Rockchip +platforms they will tend to expose the semi-independent clocks for better +power management. + +v5: +- Move the opp-table node under the gpu node + +v4: +- Fix formatting issue + +v3: +- Cleanup commit message to remove redundant text +- Added opp-table property and re-ordered entries +- Clarified power-domains and power-domain-names requirements for RK3588. +- Cleaned up example + +Note: power-domains and power-domain-names requirements for other platforms +are still work in progress, hence the bindings are left incomplete here. + +v2: +- New commit + +Signed-off-by: Liviu Dudau +Cc: Krzysztof Kozlowski +Cc: Rob Herring +Cc: Conor Dooley +Cc: devicetree@vger.kernel.org +Signed-off-by: Boris Brezillon +Reviewed-by: Rob Herring +--- + .../bindings/gpu/arm,mali-valhall-csf.yaml | 147 ++++++++++++++++++ + 1 file changed, 147 insertions(+) + create mode 100644 Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml + +diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml +new file mode 100644 +index 00000000000000..a5b4e00217587c +--- /dev/null ++++ b/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml +@@ -0,0 +1,147 @@ ++# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause ++%YAML 1.2 ++--- ++$id: http://devicetree.org/schemas/gpu/arm,mali-valhall-csf.yaml# ++$schema: http://devicetree.org/meta-schemas/core.yaml# ++ ++title: ARM Mali Valhall GPU ++ ++maintainers: ++ - Liviu Dudau ++ - Boris Brezillon ++ ++properties: ++ $nodename: ++ pattern: '^gpu@[a-f0-9]+$' ++ ++ compatible: ++ oneOf: ++ - items: ++ - enum: ++ - rockchip,rk3588-mali ++ - const: arm,mali-valhall-csf # Mali Valhall GPU model/revision is fully discoverable ++ ++ reg: ++ maxItems: 1 ++ ++ interrupts: ++ items: ++ - description: Job interrupt ++ - description: MMU interrupt ++ - description: GPU interrupt ++ ++ interrupt-names: ++ items: ++ - const: job ++ - const: mmu ++ - const: gpu ++ ++ clocks: ++ minItems: 1 ++ maxItems: 3 ++ ++ clock-names: ++ minItems: 1 ++ items: ++ - const: core ++ - const: coregroup ++ - const: stacks ++ ++ mali-supply: true ++ ++ operating-points-v2: true ++ opp-table: ++ type: object ++ ++ power-domains: ++ minItems: 1 ++ maxItems: 5 ++ ++ power-domain-names: ++ minItems: 1 ++ maxItems: 5 ++ ++ sram-supply: true ++ ++ "#cooling-cells": ++ const: 2 ++ ++ dynamic-power-coefficient: ++ $ref: /schemas/types.yaml#/definitions/uint32 ++ description: ++ A u32 value that represents the running time dynamic ++ power coefficient in units of uW/MHz/V^2. The ++ coefficient can either be calculated from power ++ measurements or derived by analysis. ++ ++ The dynamic power consumption of the GPU is ++ proportional to the square of the Voltage (V) and ++ the clock frequency (f). The coefficient is used to ++ calculate the dynamic power as below - ++ ++ Pdyn = dynamic-power-coefficient * V^2 * f ++ ++ where voltage is in V, frequency is in MHz. ++ ++ dma-coherent: true ++ ++required: ++ - compatible ++ - reg ++ - interrupts ++ - interrupt-names ++ - clocks ++ - mali-supply ++ ++additionalProperties: false ++ ++allOf: ++ - if: ++ properties: ++ compatible: ++ contains: ++ const: rockchip,rk3588-mali ++ then: ++ properties: ++ clocks: ++ minItems: 3 ++ power-domains: ++ maxItems: 1 ++ power-domain-names: false ++ ++examples: ++ - | ++ #include ++ #include ++ #include ++ #include ++ ++ gpu: gpu@fb000000 { ++ compatible = "rockchip,rk3588-mali", "arm,mali-valhall-csf"; ++ reg = <0xfb000000 0x200000>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "job", "mmu", "gpu"; ++ clock-names = "core", "coregroup", "stacks"; ++ clocks = <&cru CLK_GPU>, <&cru CLK_GPU_COREGROUP>, ++ <&cru CLK_GPU_STACKS>; ++ power-domains = <&power RK3588_PD_GPU>; ++ operating-points-v2 = <&gpu_opp_table>; ++ mali-supply = <&vdd_gpu_s0>; ++ sram-supply = <&vdd_gpu_mem_s0>; ++ ++ gpu_opp_table: opp-table { ++ compatible = "operating-points-v2"; ++ opp-300000000 { ++ opp-hz = /bits/ 64 <300000000>; ++ opp-microvolt = <675000 675000 850000>; ++ }; ++ opp-400000000 { ++ opp-hz = /bits/ 64 <400000000>; ++ opp-microvolt = <675000 675000 850000>; ++ }; ++ }; ++ }; ++ ++... + +From e44927d0d55d108ca161f8e625a62f301a0e7d34 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Thu, 7 Apr 2022 08:17:51 +0200 +Subject: [PATCH 002/150] drm: execution context for GEM buffers v7 +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This adds the infrastructure for an execution context for GEM buffers +which is similar to the existing TTMs execbuf util and intended to replace +it in the long term. + +The basic functionality is that we abstracts the necessary loop to lock +many different GEM buffers with automated deadlock and duplicate handling. + +v2: drop xarray and use dynamic resized array instead, the locking + overhead is unnecessary and measurable. +v3: drop duplicate tracking, radeon is really the only one needing that. +v4: fixes issues pointed out by Danilo, some typos in comments and a + helper for lock arrays of GEM objects. +v5: some suggestions by Boris Brezillon, especially just use one retry + macro, drop loop in prepare_array, use flags instead of bool +v6: minor changes suggested by Thomas, Boris and Danilo +v7: minor typos pointed out by checkpatch.pl fixed + +Signed-off-by: Christian König +Reviewed-by: Boris Brezillon +Reviewed-by: Danilo Krummrich +Tested-by: Danilo Krummrich +Acked-by: Alex Deucher +Link: https://patchwork.freedesktop.org/patch/msgid/20230711133122.3710-2-christian.koenig@amd.com +--- + Documentation/gpu/drm-mm.rst | 12 ++ + drivers/gpu/drm/Kconfig | 6 + + drivers/gpu/drm/Makefile | 2 + + drivers/gpu/drm/drm_exec.c | 333 +++++++++++++++++++++++++++++++++++ + include/drm/drm_exec.h | 123 +++++++++++++ + 5 files changed, 476 insertions(+) + create mode 100644 drivers/gpu/drm/drm_exec.c + create mode 100644 include/drm/drm_exec.h + +diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst +index a79fd3549ff8c2..a52e6f4117d63d 100644 +--- a/Documentation/gpu/drm-mm.rst ++++ b/Documentation/gpu/drm-mm.rst +@@ -493,6 +493,18 @@ DRM Sync Objects + .. kernel-doc:: drivers/gpu/drm/drm_syncobj.c + :export: + ++DRM Execution context ++===================== ++ ++.. kernel-doc:: drivers/gpu/drm/drm_exec.c ++ :doc: Overview ++ ++.. kernel-doc:: include/drm/drm_exec.h ++ :internal: ++ ++.. kernel-doc:: drivers/gpu/drm/drm_exec.c ++ :export: ++ + GPU Scheduler + ============= + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index cd641b1bce7b79..0e111ebc3c12e4 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -207,6 +207,12 @@ config DRM_TTM + GPU memory types. Will be enabled automatically if a device driver + uses it. + ++config DRM_EXEC ++ tristate ++ depends on DRM ++ help ++ Execution context for command submissions ++ + config DRM_BUDDY + tristate + depends on DRM +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index c611154667c201..93959cea38984a 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -40,6 +40,8 @@ obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o + # + # Memory-management helpers + # ++# ++obj-$(CONFIG_DRM_EXEC) += drm_exec.o + + obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o + +diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c +new file mode 100644 +index 00000000000000..ff69cf0fb42aab +--- /dev/null ++++ b/drivers/gpu/drm/drm_exec.c +@@ -0,0 +1,333 @@ ++// SPDX-License-Identifier: GPL-2.0 OR MIT ++ ++#include ++#include ++#include ++ ++/** ++ * DOC: Overview ++ * ++ * This component mainly abstracts the retry loop necessary for locking ++ * multiple GEM objects while preparing hardware operations (e.g. command ++ * submissions, page table updates etc..). ++ * ++ * If a contention is detected while locking a GEM object the cleanup procedure ++ * unlocks all previously locked GEM objects and locks the contended one first ++ * before locking any further objects. ++ * ++ * After an object is locked fences slots can optionally be reserved on the ++ * dma_resv object inside the GEM object. ++ * ++ * A typical usage pattern should look like this:: ++ * ++ * struct drm_gem_object *obj; ++ * struct drm_exec exec; ++ * unsigned long index; ++ * int ret; ++ * ++ * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT); ++ * drm_exec_until_all_locked(&exec) { ++ * ret = drm_exec_prepare_obj(&exec, boA, 1); ++ * drm_exec_retry_on_contention(&exec); ++ * if (ret) ++ * goto error; ++ * ++ * ret = drm_exec_prepare_obj(&exec, boB, 1); ++ * drm_exec_retry_on_contention(&exec); ++ * if (ret) ++ * goto error; ++ * } ++ * ++ * drm_exec_for_each_locked_object(&exec, index, obj) { ++ * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ); ++ * ... ++ * } ++ * drm_exec_fini(&exec); ++ * ++ * See struct dma_exec for more details. ++ */ ++ ++/* Dummy value used to initially enter the retry loop */ ++#define DRM_EXEC_DUMMY ((void *)~0) ++ ++/* Unlock all objects and drop references */ ++static void drm_exec_unlock_all(struct drm_exec *exec) ++{ ++ struct drm_gem_object *obj; ++ unsigned long index; ++ ++ drm_exec_for_each_locked_object(exec, index, obj) { ++ dma_resv_unlock(obj->resv); ++ drm_gem_object_put(obj); ++ } ++ ++ drm_gem_object_put(exec->prelocked); ++ exec->prelocked = NULL; ++} ++ ++/** ++ * drm_exec_init - initialize a drm_exec object ++ * @exec: the drm_exec object to initialize ++ * @flags: controls locking behavior, see DRM_EXEC_* defines ++ * ++ * Initialize the object and make sure that we can track locked objects. ++ */ ++void drm_exec_init(struct drm_exec *exec, uint32_t flags) ++{ ++ exec->flags = flags; ++ exec->objects = kmalloc(PAGE_SIZE, GFP_KERNEL); ++ ++ /* If allocation here fails, just delay that till the first use */ ++ exec->max_objects = exec->objects ? PAGE_SIZE / sizeof(void *) : 0; ++ exec->num_objects = 0; ++ exec->contended = DRM_EXEC_DUMMY; ++ exec->prelocked = NULL; ++} ++EXPORT_SYMBOL(drm_exec_init); ++ ++/** ++ * drm_exec_fini - finalize a drm_exec object ++ * @exec: the drm_exec object to finalize ++ * ++ * Unlock all locked objects, drop the references to objects and free all memory ++ * used for tracking the state. ++ */ ++void drm_exec_fini(struct drm_exec *exec) ++{ ++ drm_exec_unlock_all(exec); ++ kvfree(exec->objects); ++ if (exec->contended != DRM_EXEC_DUMMY) { ++ drm_gem_object_put(exec->contended); ++ ww_acquire_fini(&exec->ticket); ++ } ++} ++EXPORT_SYMBOL(drm_exec_fini); ++ ++/** ++ * drm_exec_cleanup - cleanup when contention is detected ++ * @exec: the drm_exec object to cleanup ++ * ++ * Cleanup the current state and return true if we should stay inside the retry ++ * loop, false if there wasn't any contention detected and we can keep the ++ * objects locked. ++ */ ++bool drm_exec_cleanup(struct drm_exec *exec) ++{ ++ if (likely(!exec->contended)) { ++ ww_acquire_done(&exec->ticket); ++ return false; ++ } ++ ++ if (likely(exec->contended == DRM_EXEC_DUMMY)) { ++ exec->contended = NULL; ++ ww_acquire_init(&exec->ticket, &reservation_ww_class); ++ return true; ++ } ++ ++ drm_exec_unlock_all(exec); ++ exec->num_objects = 0; ++ return true; ++} ++EXPORT_SYMBOL(drm_exec_cleanup); ++ ++/* Track the locked object in the array */ ++static int drm_exec_obj_locked(struct drm_exec *exec, ++ struct drm_gem_object *obj) ++{ ++ if (unlikely(exec->num_objects == exec->max_objects)) { ++ size_t size = exec->max_objects * sizeof(void *); ++ void *tmp; ++ ++ tmp = kvrealloc(exec->objects, size, size + PAGE_SIZE, ++ GFP_KERNEL); ++ if (!tmp) ++ return -ENOMEM; ++ ++ exec->objects = tmp; ++ exec->max_objects += PAGE_SIZE / sizeof(void *); ++ } ++ drm_gem_object_get(obj); ++ exec->objects[exec->num_objects++] = obj; ++ ++ return 0; ++} ++ ++/* Make sure the contended object is locked first */ ++static int drm_exec_lock_contended(struct drm_exec *exec) ++{ ++ struct drm_gem_object *obj = exec->contended; ++ int ret; ++ ++ if (likely(!obj)) ++ return 0; ++ ++ /* Always cleanup the contention so that error handling can kick in */ ++ exec->contended = NULL; ++ if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) { ++ ret = dma_resv_lock_slow_interruptible(obj->resv, ++ &exec->ticket); ++ if (unlikely(ret)) ++ goto error_dropref; ++ } else { ++ dma_resv_lock_slow(obj->resv, &exec->ticket); ++ } ++ ++ ret = drm_exec_obj_locked(exec, obj); ++ if (unlikely(ret)) ++ goto error_unlock; ++ ++ exec->prelocked = obj; ++ return 0; ++ ++error_unlock: ++ dma_resv_unlock(obj->resv); ++ ++error_dropref: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++/** ++ * drm_exec_lock_obj - lock a GEM object for use ++ * @exec: the drm_exec object with the state ++ * @obj: the GEM object to lock ++ * ++ * Lock a GEM object for use and grab a reference to it. ++ * ++ * Returns: -EDEADLK if a contention is detected, -EALREADY when object is ++ * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES ++ * flag), -ENOMEM when memory allocation failed and zero for success. ++ */ ++int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj) ++{ ++ int ret; ++ ++ ret = drm_exec_lock_contended(exec); ++ if (unlikely(ret)) ++ return ret; ++ ++ if (exec->prelocked == obj) { ++ drm_gem_object_put(exec->prelocked); ++ exec->prelocked = NULL; ++ return 0; ++ } ++ ++ if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) ++ ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket); ++ else ++ ret = dma_resv_lock(obj->resv, &exec->ticket); ++ ++ if (unlikely(ret == -EDEADLK)) { ++ drm_gem_object_get(obj); ++ exec->contended = obj; ++ return -EDEADLK; ++ } ++ ++ if (unlikely(ret == -EALREADY) && ++ exec->flags & DRM_EXEC_IGNORE_DUPLICATES) ++ return 0; ++ ++ if (unlikely(ret)) ++ return ret; ++ ++ ret = drm_exec_obj_locked(exec, obj); ++ if (ret) ++ goto error_unlock; ++ ++ return 0; ++ ++error_unlock: ++ dma_resv_unlock(obj->resv); ++ return ret; ++} ++EXPORT_SYMBOL(drm_exec_lock_obj); ++ ++/** ++ * drm_exec_unlock_obj - unlock a GEM object in this exec context ++ * @exec: the drm_exec object with the state ++ * @obj: the GEM object to unlock ++ * ++ * Unlock the GEM object and remove it from the collection of locked objects. ++ * Should only be used to unlock the most recently locked objects. It's not time ++ * efficient to unlock objects locked long ago. ++ */ ++void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj) ++{ ++ unsigned int i; ++ ++ for (i = exec->num_objects; i--;) { ++ if (exec->objects[i] == obj) { ++ dma_resv_unlock(obj->resv); ++ for (++i; i < exec->num_objects; ++i) ++ exec->objects[i - 1] = exec->objects[i]; ++ --exec->num_objects; ++ drm_gem_object_put(obj); ++ return; ++ } ++ ++ } ++} ++EXPORT_SYMBOL(drm_exec_unlock_obj); ++ ++/** ++ * drm_exec_prepare_obj - prepare a GEM object for use ++ * @exec: the drm_exec object with the state ++ * @obj: the GEM object to prepare ++ * @num_fences: how many fences to reserve ++ * ++ * Prepare a GEM object for use by locking it and reserving fence slots. ++ * ++ * Returns: -EDEADLK if a contention is detected, -EALREADY when object is ++ * already locked, -ENOMEM when memory allocation failed and zero for success. ++ */ ++int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, ++ unsigned int num_fences) ++{ ++ int ret; ++ ++ ret = drm_exec_lock_obj(exec, obj); ++ if (ret) ++ return ret; ++ ++ ret = dma_resv_reserve_fences(obj->resv, num_fences); ++ if (ret) { ++ drm_exec_unlock_obj(exec, obj); ++ return ret; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(drm_exec_prepare_obj); ++ ++/** ++ * drm_exec_prepare_array - helper to prepare an array of objects ++ * @exec: the drm_exec object with the state ++ * @objects: array of GEM object to prepare ++ * @num_objects: number of GEM objects in the array ++ * @num_fences: number of fences to reserve on each GEM object ++ * ++ * Prepares all GEM objects in an array, aborts on first error. ++ * Reserves @num_fences on each GEM object after locking it. ++ * ++ * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked, ++ * -ENOMEM when memory allocation failed and zero for success. ++ */ ++int drm_exec_prepare_array(struct drm_exec *exec, ++ struct drm_gem_object **objects, ++ unsigned int num_objects, ++ unsigned int num_fences) ++{ ++ int ret; ++ ++ for (unsigned int i = 0; i < num_objects; ++i) { ++ ret = drm_exec_prepare_obj(exec, objects[i], num_fences); ++ if (unlikely(ret)) ++ return ret; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL(drm_exec_prepare_array); ++ ++MODULE_DESCRIPTION("DRM execution context"); ++MODULE_LICENSE("Dual MIT/GPL"); +diff --git a/include/drm/drm_exec.h b/include/drm/drm_exec.h +new file mode 100644 +index 00000000000000..73205afec16211 +--- /dev/null ++++ b/include/drm/drm_exec.h +@@ -0,0 +1,123 @@ ++/* SPDX-License-Identifier: GPL-2.0 OR MIT */ ++ ++#ifndef __DRM_EXEC_H__ ++#define __DRM_EXEC_H__ ++ ++#include ++ ++#define DRM_EXEC_INTERRUPTIBLE_WAIT BIT(0) ++#define DRM_EXEC_IGNORE_DUPLICATES BIT(1) ++ ++struct drm_gem_object; ++ ++/** ++ * struct drm_exec - Execution context ++ */ ++struct drm_exec { ++ /** ++ * @flags: Flags to control locking behavior ++ */ ++ uint32_t flags; ++ ++ /** ++ * @ticket: WW ticket used for acquiring locks ++ */ ++ struct ww_acquire_ctx ticket; ++ ++ /** ++ * @num_objects: number of objects locked ++ */ ++ unsigned int num_objects; ++ ++ /** ++ * @max_objects: maximum objects in array ++ */ ++ unsigned int max_objects; ++ ++ /** ++ * @objects: array of the locked objects ++ */ ++ struct drm_gem_object **objects; ++ ++ /** ++ * @contended: contended GEM object we backed off for ++ */ ++ struct drm_gem_object *contended; ++ ++ /** ++ * @prelocked: already locked GEM object due to contention ++ */ ++ struct drm_gem_object *prelocked; ++}; ++ ++/** ++ * drm_exec_for_each_locked_object - iterate over all the locked objects ++ * @exec: drm_exec object ++ * @index: unsigned long index for the iteration ++ * @obj: the current GEM object ++ * ++ * Iterate over all the locked GEM objects inside the drm_exec object. ++ */ ++#define drm_exec_for_each_locked_object(exec, index, obj) \ ++ for (index = 0, obj = (exec)->objects[0]; \ ++ index < (exec)->num_objects; \ ++ ++index, obj = (exec)->objects[index]) ++ ++/** ++ * drm_exec_until_all_locked - loop until all GEM objects are locked ++ * @exec: drm_exec object ++ * ++ * Core functionality of the drm_exec object. Loops until all GEM objects are ++ * locked and no more contention exists. At the beginning of the loop it is ++ * guaranteed that no GEM object is locked. ++ * ++ * Since labels can't be defined local to the loops body we use a jump pointer ++ * to make sure that the retry is only used from within the loops body. ++ */ ++#define drm_exec_until_all_locked(exec) \ ++ for (void *__drm_exec_retry_ptr; ({ \ ++ __label__ __drm_exec_retry; \ ++__drm_exec_retry: \ ++ __drm_exec_retry_ptr = &&__drm_exec_retry; \ ++ (void)__drm_exec_retry_ptr; \ ++ drm_exec_cleanup(exec); \ ++ });) ++ ++/** ++ * drm_exec_retry_on_contention - restart the loop to grap all locks ++ * @exec: drm_exec object ++ * ++ * Control flow helper to continue when a contention was detected and we need to ++ * clean up and re-start the loop to prepare all GEM objects. ++ */ ++#define drm_exec_retry_on_contention(exec) \ ++ do { \ ++ if (unlikely(drm_exec_is_contended(exec))) \ ++ goto *__drm_exec_retry_ptr; \ ++ } while (0) ++ ++/** ++ * drm_exec_is_contended - check for contention ++ * @exec: drm_exec object ++ * ++ * Returns true if the drm_exec object has run into some contention while ++ * locking a GEM object and needs to clean up. ++ */ ++static inline bool drm_exec_is_contended(struct drm_exec *exec) ++{ ++ return !!exec->contended; ++} ++ ++void drm_exec_init(struct drm_exec *exec, uint32_t flags); ++void drm_exec_fini(struct drm_exec *exec); ++bool drm_exec_cleanup(struct drm_exec *exec); ++int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj); ++void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj); ++int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, ++ unsigned int num_fences); ++int drm_exec_prepare_array(struct drm_exec *exec, ++ struct drm_gem_object **objects, ++ unsigned int num_objects, ++ unsigned int num_fences); ++ ++#endif + +From f2d6af7368eb562999cb29b258d7eba3fd3095e6 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Thu, 20 Jul 2023 02:14:22 +0200 +Subject: [PATCH 003/150] drm: manager to keep track of GPUs VA mappings +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Add infrastructure to keep track of GPU virtual address (VA) mappings +with a decicated VA space manager implementation. + +New UAPIs, motivated by Vulkan sparse memory bindings graphics drivers +start implementing, allow userspace applications to request multiple and +arbitrary GPU VA mappings of buffer objects. The DRM GPU VA manager is +intended to serve the following purposes in this context. + +1) Provide infrastructure to track GPU VA allocations and mappings, + using an interval tree (RB-tree). + +2) Generically connect GPU VA mappings to their backing buffers, in + particular DRM GEM objects. + +3) Provide a common implementation to perform more complex mapping + operations on the GPU VA space. In particular splitting and merging + of GPU VA mappings, e.g. for intersecting mapping requests or partial + unmap requests. + +Acked-by: Thomas Hellström +Acked-by: Matthew Brost +Reviewed-by: Boris Brezillon +Tested-by: Matthew Brost +Tested-by: Donald Robson +Suggested-by: Dave Airlie +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20230720001443.2380-2-dakr@redhat.com +--- + Documentation/gpu/drm-mm.rst | 36 + + drivers/gpu/drm/Makefile | 1 + + drivers/gpu/drm/drm_gem.c | 3 + + drivers/gpu/drm/drm_gpuva_mgr.c | 1725 +++++++++++++++++++++++++++++++ + include/drm/drm_drv.h | 6 + + include/drm/drm_gem.h | 79 ++ + include/drm/drm_gpuva_mgr.h | 706 +++++++++++++ + 7 files changed, 2556 insertions(+) + create mode 100644 drivers/gpu/drm/drm_gpuva_mgr.c + create mode 100644 include/drm/drm_gpuva_mgr.h + +diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst +index a52e6f4117d63d..3d5dc9dc1bfe51 100644 +--- a/Documentation/gpu/drm-mm.rst ++++ b/Documentation/gpu/drm-mm.rst +@@ -466,6 +466,42 @@ DRM MM Range Allocator Function References + .. kernel-doc:: drivers/gpu/drm/drm_mm.c + :export: + ++DRM GPU VA Manager ++================== ++ ++Overview ++-------- ++ ++.. kernel-doc:: drivers/gpu/drm/drm_gpuva_mgr.c ++ :doc: Overview ++ ++Split and Merge ++--------------- ++ ++.. kernel-doc:: drivers/gpu/drm/drm_gpuva_mgr.c ++ :doc: Split and Merge ++ ++Locking ++------- ++ ++.. kernel-doc:: drivers/gpu/drm/drm_gpuva_mgr.c ++ :doc: Locking ++ ++Examples ++-------- ++ ++.. kernel-doc:: drivers/gpu/drm/drm_gpuva_mgr.c ++ :doc: Examples ++ ++DRM GPU VA Manager Function References ++-------------------------------------- ++ ++.. kernel-doc:: include/drm/drm_gpuva_mgr.h ++ :internal: ++ ++.. kernel-doc:: drivers/gpu/drm/drm_gpuva_mgr.c ++ :export: ++ + DRM Buddy Allocator + =================== + +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index 93959cea38984a..ad092cd7f2e95f 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -12,6 +12,7 @@ drm-y := drm_aperture.o drm_auth.o drm_cache.o \ + drm_crtc.o drm_fourcc.o drm_modes.o drm_displayid.o \ + drm_trace_points.o drm_prime.o \ + drm_vma_manager.o \ ++ drm_gpuva_mgr.o \ + drm_modeset_lock.o drm_atomic.o drm_bridge.o \ + drm_framebuffer.o drm_connector.o drm_blend.o \ + drm_encoder.o drm_mode_object.o drm_property.o \ +diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c +index b87ed4238fc836..fab07f0a46398b 100644 +--- a/drivers/gpu/drm/drm_gem.c ++++ b/drivers/gpu/drm/drm_gem.c +@@ -164,6 +164,9 @@ void drm_gem_private_object_init(struct drm_device *dev, + if (!obj->resv) + obj->resv = &obj->_resv; + ++ if (drm_core_check_feature(dev, DRIVER_GEM_GPUVA)) ++ drm_gem_gpuva_init(obj); ++ + drm_vma_node_reset(&obj->vma_node); + INIT_LIST_HEAD(&obj->lru_node); + } +diff --git a/drivers/gpu/drm/drm_gpuva_mgr.c b/drivers/gpu/drm/drm_gpuva_mgr.c +new file mode 100644 +index 00000000000000..0b80177592a649 +--- /dev/null ++++ b/drivers/gpu/drm/drm_gpuva_mgr.c +@@ -0,0 +1,1725 @@ ++// SPDX-License-Identifier: GPL-2.0-only ++/* ++ * Copyright (c) 2022 Red Hat. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice shall be included in ++ * all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR ++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ * ++ * Authors: ++ * Danilo Krummrich ++ * ++ */ ++ ++#include ++ ++#include ++#include ++ ++/** ++ * DOC: Overview ++ * ++ * The DRM GPU VA Manager, represented by struct drm_gpuva_manager keeps track ++ * of a GPU's virtual address (VA) space and manages the corresponding virtual ++ * mappings represented by &drm_gpuva objects. It also keeps track of the ++ * mapping's backing &drm_gem_object buffers. ++ * ++ * &drm_gem_object buffers maintain a list of &drm_gpuva objects representing ++ * all existent GPU VA mappings using this &drm_gem_object as backing buffer. ++ * ++ * GPU VAs can be flagged as sparse, such that drivers may use GPU VAs to also ++ * keep track of sparse PTEs in order to support Vulkan 'Sparse Resources'. ++ * ++ * The GPU VA manager internally uses a rb-tree to manage the ++ * &drm_gpuva mappings within a GPU's virtual address space. ++ * ++ * The &drm_gpuva_manager contains a special &drm_gpuva representing the ++ * portion of VA space reserved by the kernel. This node is initialized together ++ * with the GPU VA manager instance and removed when the GPU VA manager is ++ * destroyed. ++ * ++ * In a typical application drivers would embed struct drm_gpuva_manager and ++ * struct drm_gpuva within their own driver specific structures, there won't be ++ * any memory allocations of its own nor memory allocations of &drm_gpuva ++ * entries. ++ * ++ * The data structures needed to store &drm_gpuvas within the &drm_gpuva_manager ++ * are contained within struct drm_gpuva already. Hence, for inserting ++ * &drm_gpuva entries from within dma-fence signalling critical sections it is ++ * enough to pre-allocate the &drm_gpuva structures. ++ */ ++ ++/** ++ * DOC: Split and Merge ++ * ++ * Besides its capability to manage and represent a GPU VA space, the ++ * &drm_gpuva_manager also provides functions to let the &drm_gpuva_manager ++ * calculate a sequence of operations to satisfy a given map or unmap request. ++ * ++ * Therefore the DRM GPU VA manager provides an algorithm implementing splitting ++ * and merging of existent GPU VA mappings with the ones that are requested to ++ * be mapped or unmapped. This feature is required by the Vulkan API to ++ * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this ++ * as VM BIND. ++ * ++ * Drivers can call drm_gpuva_sm_map() to receive a sequence of callbacks ++ * containing map, unmap and remap operations for a given newly requested ++ * mapping. The sequence of callbacks represents the set of operations to ++ * execute in order to integrate the new mapping cleanly into the current state ++ * of the GPU VA space. ++ * ++ * Depending on how the new GPU VA mapping intersects with the existent mappings ++ * of the GPU VA space the &drm_gpuva_fn_ops callbacks contain an arbitrary ++ * amount of unmap operations, a maximum of two remap operations and a single ++ * map operation. The caller might receive no callback at all if no operation is ++ * required, e.g. if the requested mapping already exists in the exact same way. ++ * ++ * The single map operation represents the original map operation requested by ++ * the caller. ++ * ++ * &drm_gpuva_op_unmap contains a 'keep' field, which indicates whether the ++ * &drm_gpuva to unmap is physically contiguous with the original mapping ++ * request. Optionally, if 'keep' is set, drivers may keep the actual page table ++ * entries for this &drm_gpuva, adding the missing page table entries only and ++ * update the &drm_gpuva_manager's view of things accordingly. ++ * ++ * Drivers may do the same optimization, namely delta page table updates, also ++ * for remap operations. This is possible since &drm_gpuva_op_remap consists of ++ * one unmap operation and one or two map operations, such that drivers can ++ * derive the page table update delta accordingly. ++ * ++ * Note that there can't be more than two existent mappings to split up, one at ++ * the beginning and one at the end of the new mapping, hence there is a ++ * maximum of two remap operations. ++ * ++ * Analogous to drm_gpuva_sm_map() drm_gpuva_sm_unmap() uses &drm_gpuva_fn_ops ++ * to call back into the driver in order to unmap a range of GPU VA space. The ++ * logic behind this function is way simpler though: For all existent mappings ++ * enclosed by the given range unmap operations are created. For mappings which ++ * are only partically located within the given range, remap operations are ++ * created such that those mappings are split up and re-mapped partically. ++ * ++ * As an alternative to drm_gpuva_sm_map() and drm_gpuva_sm_unmap(), ++ * drm_gpuva_sm_map_ops_create() and drm_gpuva_sm_unmap_ops_create() can be used ++ * to directly obtain an instance of struct drm_gpuva_ops containing a list of ++ * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list ++ * contains the &drm_gpuva_ops analogous to the callbacks one would receive when ++ * calling drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). While this way requires ++ * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to ++ * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory ++ * allocations are possible (e.g. to allocate GPU page tables) and once in the ++ * dma-fence signalling critical path. ++ * ++ * To update the &drm_gpuva_manager's view of the GPU VA space ++ * drm_gpuva_insert() and drm_gpuva_remove() may be used. These functions can ++ * safely be used from &drm_gpuva_fn_ops callbacks originating from ++ * drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). However, it might be more ++ * convenient to use the provided helper functions drm_gpuva_map(), ++ * drm_gpuva_remap() and drm_gpuva_unmap() instead. ++ * ++ * The following diagram depicts the basic relationships of existent GPU VA ++ * mappings, a newly requested mapping and the resulting mappings as implemented ++ * by drm_gpuva_sm_map() - it doesn't cover any arbitrary combinations of these. ++ * ++ * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs ++ * could be kept. ++ * ++ * :: ++ * ++ * 0 a 1 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 a 1 ++ * req: |-----------| (bo_offset=n) ++ * ++ * 0 a 1 ++ * new: |-----------| (bo_offset=n) ++ * ++ * ++ * 2) Requested mapping is identical, except for the BO offset, hence replace ++ * the mapping. ++ * ++ * :: ++ * ++ * 0 a 1 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 a 1 ++ * req: |-----------| (bo_offset=m) ++ * ++ * 0 a 1 ++ * new: |-----------| (bo_offset=m) ++ * ++ * ++ * 3) Requested mapping is identical, except for the backing BO, hence replace ++ * the mapping. ++ * ++ * :: ++ * ++ * 0 a 1 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 b 1 ++ * req: |-----------| (bo_offset=n) ++ * ++ * 0 b 1 ++ * new: |-----------| (bo_offset=n) ++ * ++ * ++ * 4) Existent mapping is a left aligned subset of the requested one, hence ++ * replace the existent one. ++ * ++ * :: ++ * ++ * 0 a 1 ++ * old: |-----| (bo_offset=n) ++ * ++ * 0 a 2 ++ * req: |-----------| (bo_offset=n) ++ * ++ * 0 a 2 ++ * new: |-----------| (bo_offset=n) ++ * ++ * .. note:: ++ * We expect to see the same result for a request with a different BO ++ * and/or non-contiguous BO offset. ++ * ++ * ++ * 5) Requested mapping's range is a left aligned subset of the existent one, ++ * but backed by a different BO. Hence, map the requested mapping and split ++ * the existent one adjusting its BO offset. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 b 1 ++ * req: |-----| (bo_offset=n) ++ * ++ * 0 b 1 a' 2 ++ * new: |-----|-----| (b.bo_offset=n, a.bo_offset=n+1) ++ * ++ * .. note:: ++ * We expect to see the same result for a request with a different BO ++ * and/or non-contiguous BO offset. ++ * ++ * ++ * 6) Existent mapping is a superset of the requested mapping. Split it up, but ++ * indicate that the backing PTEs could be kept. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 a 1 ++ * req: |-----| (bo_offset=n) ++ * ++ * 0 a 1 a' 2 ++ * new: |-----|-----| (a.bo_offset=n, a'.bo_offset=n+1) ++ * ++ * ++ * 7) Requested mapping's range is a right aligned subset of the existent one, ++ * but backed by a different BO. Hence, map the requested mapping and split ++ * the existent one, without adjusting the BO offset. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 1 b 2 ++ * req: |-----| (bo_offset=m) ++ * ++ * 0 a 1 b 2 ++ * new: |-----|-----| (a.bo_offset=n,b.bo_offset=m) ++ * ++ * ++ * 8) Existent mapping is a superset of the requested mapping. Split it up, but ++ * indicate that the backing PTEs could be kept. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 1 a 2 ++ * req: |-----| (bo_offset=n+1) ++ * ++ * 0 a' 1 a 2 ++ * new: |-----|-----| (a'.bo_offset=n, a.bo_offset=n+1) ++ * ++ * ++ * 9) Existent mapping is overlapped at the end by the requested mapping backed ++ * by a different BO. Hence, map the requested mapping and split up the ++ * existent one, without adjusting the BO offset. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 1 b 3 ++ * req: |-----------| (bo_offset=m) ++ * ++ * 0 a 1 b 3 ++ * new: |-----|-----------| (a.bo_offset=n,b.bo_offset=m) ++ * ++ * ++ * 10) Existent mapping is overlapped by the requested mapping, both having the ++ * same backing BO with a contiguous offset. Indicate the backing PTEs of ++ * the old mapping could be kept. ++ * ++ * :: ++ * ++ * 0 a 2 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 1 a 3 ++ * req: |-----------| (bo_offset=n+1) ++ * ++ * 0 a' 1 a 3 ++ * new: |-----|-----------| (a'.bo_offset=n, a.bo_offset=n+1) ++ * ++ * ++ * 11) Requested mapping's range is a centered subset of the existent one ++ * having a different backing BO. Hence, map the requested mapping and split ++ * up the existent one in two mappings, adjusting the BO offset of the right ++ * one accordingly. ++ * ++ * :: ++ * ++ * 0 a 3 ++ * old: |-----------------| (bo_offset=n) ++ * ++ * 1 b 2 ++ * req: |-----| (bo_offset=m) ++ * ++ * 0 a 1 b 2 a' 3 ++ * new: |-----|-----|-----| (a.bo_offset=n,b.bo_offset=m,a'.bo_offset=n+2) ++ * ++ * ++ * 12) Requested mapping is a contiguous subset of the existent one. Split it ++ * up, but indicate that the backing PTEs could be kept. ++ * ++ * :: ++ * ++ * 0 a 3 ++ * old: |-----------------| (bo_offset=n) ++ * ++ * 1 a 2 ++ * req: |-----| (bo_offset=n+1) ++ * ++ * 0 a' 1 a 2 a'' 3 ++ * old: |-----|-----|-----| (a'.bo_offset=n, a.bo_offset=n+1, a''.bo_offset=n+2) ++ * ++ * ++ * 13) Existent mapping is a right aligned subset of the requested one, hence ++ * replace the existent one. ++ * ++ * :: ++ * ++ * 1 a 2 ++ * old: |-----| (bo_offset=n+1) ++ * ++ * 0 a 2 ++ * req: |-----------| (bo_offset=n) ++ * ++ * 0 a 2 ++ * new: |-----------| (bo_offset=n) ++ * ++ * .. note:: ++ * We expect to see the same result for a request with a different bo ++ * and/or non-contiguous bo_offset. ++ * ++ * ++ * 14) Existent mapping is a centered subset of the requested one, hence ++ * replace the existent one. ++ * ++ * :: ++ * ++ * 1 a 2 ++ * old: |-----| (bo_offset=n+1) ++ * ++ * 0 a 3 ++ * req: |----------------| (bo_offset=n) ++ * ++ * 0 a 3 ++ * new: |----------------| (bo_offset=n) ++ * ++ * .. note:: ++ * We expect to see the same result for a request with a different bo ++ * and/or non-contiguous bo_offset. ++ * ++ * ++ * 15) Existent mappings is overlapped at the beginning by the requested mapping ++ * backed by a different BO. Hence, map the requested mapping and split up ++ * the existent one, adjusting its BO offset accordingly. ++ * ++ * :: ++ * ++ * 1 a 3 ++ * old: |-----------| (bo_offset=n) ++ * ++ * 0 b 2 ++ * req: |-----------| (bo_offset=m) ++ * ++ * 0 b 2 a' 3 ++ * new: |-----------|-----| (b.bo_offset=m,a.bo_offset=n+2) ++ */ ++ ++/** ++ * DOC: Locking ++ * ++ * Generally, the GPU VA manager does not take care of locking itself, it is ++ * the drivers responsibility to take care about locking. Drivers might want to ++ * protect the following operations: inserting, removing and iterating ++ * &drm_gpuva objects as well as generating all kinds of operations, such as ++ * split / merge or prefetch. ++ * ++ * The GPU VA manager also does not take care of the locking of the backing ++ * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to ++ * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively ++ * a driver specific external lock. For the latter see also ++ * drm_gem_gpuva_set_lock(). ++ * ++ * However, the GPU VA manager contains lockdep checks to ensure callers of its ++ * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is ++ * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink(). ++ */ ++ ++/** ++ * DOC: Examples ++ * ++ * This section gives two examples on how to let the DRM GPUVA Manager generate ++ * &drm_gpuva_op in order to satisfy a given map or unmap request and how to ++ * make use of them. ++ * ++ * The below code is strictly limited to illustrate the generic usage pattern. ++ * To maintain simplicitly, it doesn't make use of any abstractions for common ++ * code, different (asyncronous) stages with fence signalling critical paths, ++ * any other helpers or error handling in terms of freeing memory and dropping ++ * previously taken locks. ++ * ++ * 1) Obtain a list of &drm_gpuva_op to create a new mapping:: ++ * ++ * // Allocates a new &drm_gpuva. ++ * struct drm_gpuva * driver_gpuva_alloc(void); ++ * ++ * // Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva ++ * // structure in individual driver structures and lock the dma-resv with ++ * // drm_exec or similar helpers. ++ * int driver_mapping_create(struct drm_gpuva_manager *mgr, ++ * u64 addr, u64 range, ++ * struct drm_gem_object *obj, u64 offset) ++ * { ++ * struct drm_gpuva_ops *ops; ++ * struct drm_gpuva_op *op ++ * ++ * driver_lock_va_space(); ++ * ops = drm_gpuva_sm_map_ops_create(mgr, addr, range, ++ * obj, offset); ++ * if (IS_ERR(ops)) ++ * return PTR_ERR(ops); ++ * ++ * drm_gpuva_for_each_op(op, ops) { ++ * struct drm_gpuva *va; ++ * ++ * switch (op->op) { ++ * case DRM_GPUVA_OP_MAP: ++ * va = driver_gpuva_alloc(); ++ * if (!va) ++ * ; // unwind previous VA space updates, ++ * // free memory and unlock ++ * ++ * driver_vm_map(); ++ * drm_gpuva_map(mgr, va, &op->map); ++ * drm_gpuva_link(va); ++ * ++ * break; ++ * case DRM_GPUVA_OP_REMAP: { ++ * struct drm_gpuva *prev = NULL, *next = NULL; ++ * ++ * va = op->remap.unmap->va; ++ * ++ * if (op->remap.prev) { ++ * prev = driver_gpuva_alloc(); ++ * if (!prev) ++ * ; // unwind previous VA space ++ * // updates, free memory and ++ * // unlock ++ * } ++ * ++ * if (op->remap.next) { ++ * next = driver_gpuva_alloc(); ++ * if (!next) ++ * ; // unwind previous VA space ++ * // updates, free memory and ++ * // unlock ++ * } ++ * ++ * driver_vm_remap(); ++ * drm_gpuva_remap(prev, next, &op->remap); ++ * ++ * drm_gpuva_unlink(va); ++ * if (prev) ++ * drm_gpuva_link(prev); ++ * if (next) ++ * drm_gpuva_link(next); ++ * ++ * break; ++ * } ++ * case DRM_GPUVA_OP_UNMAP: ++ * va = op->unmap->va; ++ * ++ * driver_vm_unmap(); ++ * drm_gpuva_unlink(va); ++ * drm_gpuva_unmap(&op->unmap); ++ * ++ * break; ++ * default: ++ * break; ++ * } ++ * } ++ * driver_unlock_va_space(); ++ * ++ * return 0; ++ * } ++ * ++ * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: ++ * ++ * struct driver_context { ++ * struct drm_gpuva_manager *mgr; ++ * struct drm_gpuva *new_va; ++ * struct drm_gpuva *prev_va; ++ * struct drm_gpuva *next_va; ++ * }; ++ * ++ * // ops to pass to drm_gpuva_manager_init() ++ * static const struct drm_gpuva_fn_ops driver_gpuva_ops = { ++ * .sm_step_map = driver_gpuva_map, ++ * .sm_step_remap = driver_gpuva_remap, ++ * .sm_step_unmap = driver_gpuva_unmap, ++ * }; ++ * ++ * // Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva ++ * // structure in individual driver structures and lock the dma-resv with ++ * // drm_exec or similar helpers. ++ * int driver_mapping_create(struct drm_gpuva_manager *mgr, ++ * u64 addr, u64 range, ++ * struct drm_gem_object *obj, u64 offset) ++ * { ++ * struct driver_context ctx; ++ * struct drm_gpuva_ops *ops; ++ * struct drm_gpuva_op *op; ++ * int ret = 0; ++ * ++ * ctx.mgr = mgr; ++ * ++ * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); ++ * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); ++ * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); ++ * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) { ++ * ret = -ENOMEM; ++ * goto out; ++ * } ++ * ++ * driver_lock_va_space(); ++ * ret = drm_gpuva_sm_map(mgr, &ctx, addr, range, obj, offset); ++ * driver_unlock_va_space(); ++ * ++ * out: ++ * kfree(ctx.new_va); ++ * kfree(ctx.prev_va); ++ * kfree(ctx.next_va); ++ * return ret; ++ * } ++ * ++ * int driver_gpuva_map(struct drm_gpuva_op *op, void *__ctx) ++ * { ++ * struct driver_context *ctx = __ctx; ++ * ++ * drm_gpuva_map(ctx->mgr, ctx->new_va, &op->map); ++ * ++ * drm_gpuva_link(ctx->new_va); ++ * ++ * // prevent the new GPUVA from being freed in ++ * // driver_mapping_create() ++ * ctx->new_va = NULL; ++ * ++ * return 0; ++ * } ++ * ++ * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) ++ * { ++ * struct driver_context *ctx = __ctx; ++ * ++ * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); ++ * ++ * drm_gpuva_unlink(op->remap.unmap->va); ++ * kfree(op->remap.unmap->va); ++ * ++ * if (op->remap.prev) { ++ * drm_gpuva_link(ctx->prev_va); ++ * ctx->prev_va = NULL; ++ * } ++ * ++ * if (op->remap.next) { ++ * drm_gpuva_link(ctx->next_va); ++ * ctx->next_va = NULL; ++ * } ++ * ++ * return 0; ++ * } ++ * ++ * int driver_gpuva_unmap(struct drm_gpuva_op *op, void *__ctx) ++ * { ++ * drm_gpuva_unlink(op->unmap.va); ++ * drm_gpuva_unmap(&op->unmap); ++ * kfree(op->unmap.va); ++ * ++ * return 0; ++ * } ++ */ ++ ++#define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node) ++ ++#define GPUVA_START(node) ((node)->va.addr) ++#define GPUVA_LAST(node) ((node)->va.addr + (node)->va.range - 1) ++ ++/* We do not actually use drm_gpuva_it_next(), tell the compiler to not complain ++ * about this. ++ */ ++INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last, ++ GPUVA_START, GPUVA_LAST, static __maybe_unused, ++ drm_gpuva_it) ++ ++static int __drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva *va); ++static void __drm_gpuva_remove(struct drm_gpuva *va); ++ ++static bool ++drm_gpuva_check_overflow(u64 addr, u64 range) ++{ ++ u64 end; ++ ++ return WARN(check_add_overflow(addr, range, &end), ++ "GPUVA address limited to %lu bytes.\n", sizeof(end)); ++} ++ ++static bool ++drm_gpuva_in_mm_range(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++{ ++ u64 end = addr + range; ++ u64 mm_start = mgr->mm_start; ++ u64 mm_end = mm_start + mgr->mm_range; ++ ++ return addr >= mm_start && end <= mm_end; ++} ++ ++static bool ++drm_gpuva_in_kernel_node(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++{ ++ u64 end = addr + range; ++ u64 kstart = mgr->kernel_alloc_node.va.addr; ++ u64 krange = mgr->kernel_alloc_node.va.range; ++ u64 kend = kstart + krange; ++ ++ return krange && addr < kend && kstart < end; ++} ++ ++static bool ++drm_gpuva_range_valid(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range) ++{ ++ return !drm_gpuva_check_overflow(addr, range) && ++ drm_gpuva_in_mm_range(mgr, addr, range) && ++ !drm_gpuva_in_kernel_node(mgr, addr, range); ++} ++ ++/** ++ * drm_gpuva_manager_init() - initialize a &drm_gpuva_manager ++ * @mgr: pointer to the &drm_gpuva_manager to initialize ++ * @name: the name of the GPU VA space ++ * @start_offset: the start offset of the GPU VA space ++ * @range: the size of the GPU VA space ++ * @reserve_offset: the start of the kernel reserved GPU VA area ++ * @reserve_range: the size of the kernel reserved GPU VA area ++ * @ops: &drm_gpuva_fn_ops called on &drm_gpuva_sm_map / &drm_gpuva_sm_unmap ++ * ++ * The &drm_gpuva_manager must be initialized with this function before use. ++ * ++ * Note that @mgr must be cleared to 0 before calling this function. The given ++ * &name is expected to be managed by the surrounding driver structures. ++ */ ++void ++drm_gpuva_manager_init(struct drm_gpuva_manager *mgr, ++ const char *name, ++ u64 start_offset, u64 range, ++ u64 reserve_offset, u64 reserve_range, ++ const struct drm_gpuva_fn_ops *ops) ++{ ++ mgr->rb.tree = RB_ROOT_CACHED; ++ INIT_LIST_HEAD(&mgr->rb.list); ++ ++ drm_gpuva_check_overflow(start_offset, range); ++ mgr->mm_start = start_offset; ++ mgr->mm_range = range; ++ ++ mgr->name = name ? name : "unknown"; ++ mgr->ops = ops; ++ ++ memset(&mgr->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); ++ ++ if (reserve_range) { ++ mgr->kernel_alloc_node.va.addr = reserve_offset; ++ mgr->kernel_alloc_node.va.range = reserve_range; ++ ++ if (likely(!drm_gpuva_check_overflow(reserve_offset, ++ reserve_range))) ++ __drm_gpuva_insert(mgr, &mgr->kernel_alloc_node); ++ } ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_manager_init); ++ ++/** ++ * drm_gpuva_manager_destroy() - cleanup a &drm_gpuva_manager ++ * @mgr: pointer to the &drm_gpuva_manager to clean up ++ * ++ * Note that it is a bug to call this function on a manager that still ++ * holds GPU VA mappings. ++ */ ++void ++drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr) ++{ ++ mgr->name = NULL; ++ ++ if (mgr->kernel_alloc_node.va.range) ++ __drm_gpuva_remove(&mgr->kernel_alloc_node); ++ ++ WARN(!RB_EMPTY_ROOT(&mgr->rb.tree.rb_root), ++ "GPUVA tree is not empty, potentially leaking memory."); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_manager_destroy); ++ ++static int ++__drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva *va) ++{ ++ struct rb_node *node; ++ struct list_head *head; ++ ++ if (drm_gpuva_it_iter_first(&mgr->rb.tree, ++ GPUVA_START(va), ++ GPUVA_LAST(va))) ++ return -EEXIST; ++ ++ va->mgr = mgr; ++ ++ drm_gpuva_it_insert(va, &mgr->rb.tree); ++ ++ node = rb_prev(&va->rb.node); ++ if (node) ++ head = &(to_drm_gpuva(node))->rb.entry; ++ else ++ head = &mgr->rb.list; ++ ++ list_add(&va->rb.entry, head); ++ ++ return 0; ++} ++ ++/** ++ * drm_gpuva_insert() - insert a &drm_gpuva ++ * @mgr: the &drm_gpuva_manager to insert the &drm_gpuva in ++ * @va: the &drm_gpuva to insert ++ * ++ * Insert a &drm_gpuva with a given address and range into a ++ * &drm_gpuva_manager. ++ * ++ * It is safe to use this function using the safe versions of iterating the GPU ++ * VA space, such as drm_gpuva_for_each_va_safe() and ++ * drm_gpuva_for_each_va_range_safe(). ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva *va) ++{ ++ u64 addr = va->va.addr; ++ u64 range = va->va.range; ++ ++ if (unlikely(!drm_gpuva_range_valid(mgr, addr, range))) ++ return -EINVAL; ++ ++ return __drm_gpuva_insert(mgr, va); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_insert); ++ ++static void ++__drm_gpuva_remove(struct drm_gpuva *va) ++{ ++ drm_gpuva_it_remove(va, &va->mgr->rb.tree); ++ list_del_init(&va->rb.entry); ++} ++ ++/** ++ * drm_gpuva_remove() - remove a &drm_gpuva ++ * @va: the &drm_gpuva to remove ++ * ++ * This removes the given &va from the underlaying tree. ++ * ++ * It is safe to use this function using the safe versions of iterating the GPU ++ * VA space, such as drm_gpuva_for_each_va_safe() and ++ * drm_gpuva_for_each_va_range_safe(). ++ */ ++void ++drm_gpuva_remove(struct drm_gpuva *va) ++{ ++ struct drm_gpuva_manager *mgr = va->mgr; ++ ++ if (unlikely(va == &mgr->kernel_alloc_node)) { ++ WARN(1, "Can't destroy kernel reserved node.\n"); ++ return; ++ } ++ ++ __drm_gpuva_remove(va); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_remove); ++ ++/** ++ * drm_gpuva_link() - link a &drm_gpuva ++ * @va: the &drm_gpuva to link ++ * ++ * This adds the given &va to the GPU VA list of the &drm_gem_object it is ++ * associated with. ++ * ++ * This function expects the caller to protect the GEM's GPUVA list against ++ * concurrent access using the GEMs dma_resv lock. ++ */ ++void ++drm_gpuva_link(struct drm_gpuva *va) ++{ ++ struct drm_gem_object *obj = va->gem.obj; ++ ++ if (unlikely(!obj)) ++ return; ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ ++ list_add_tail(&va->gem.entry, &obj->gpuva.list); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_link); ++ ++/** ++ * drm_gpuva_unlink() - unlink a &drm_gpuva ++ * @va: the &drm_gpuva to unlink ++ * ++ * This removes the given &va from the GPU VA list of the &drm_gem_object it is ++ * associated with. ++ * ++ * This function expects the caller to protect the GEM's GPUVA list against ++ * concurrent access using the GEMs dma_resv lock. ++ */ ++void ++drm_gpuva_unlink(struct drm_gpuva *va) ++{ ++ struct drm_gem_object *obj = va->gem.obj; ++ ++ if (unlikely(!obj)) ++ return; ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ ++ list_del_init(&va->gem.entry); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_unlink); ++ ++/** ++ * drm_gpuva_find_first() - find the first &drm_gpuva in the given range ++ * @mgr: the &drm_gpuva_manager to search in ++ * @addr: the &drm_gpuvas address ++ * @range: the &drm_gpuvas range ++ * ++ * Returns: the first &drm_gpuva within the given range ++ */ ++struct drm_gpuva * ++drm_gpuva_find_first(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range) ++{ ++ u64 last = addr + range - 1; ++ ++ return drm_gpuva_it_iter_first(&mgr->rb.tree, addr, last); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_find_first); ++ ++/** ++ * drm_gpuva_find() - find a &drm_gpuva ++ * @mgr: the &drm_gpuva_manager to search in ++ * @addr: the &drm_gpuvas address ++ * @range: the &drm_gpuvas range ++ * ++ * Returns: the &drm_gpuva at a given &addr and with a given &range ++ */ ++struct drm_gpuva * ++drm_gpuva_find(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range) ++{ ++ struct drm_gpuva *va; ++ ++ va = drm_gpuva_find_first(mgr, addr, range); ++ if (!va) ++ goto out; ++ ++ if (va->va.addr != addr || ++ va->va.range != range) ++ goto out; ++ ++ return va; ++ ++out: ++ return NULL; ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_find); ++ ++/** ++ * drm_gpuva_find_prev() - find the &drm_gpuva before the given address ++ * @mgr: the &drm_gpuva_manager to search in ++ * @start: the given GPU VA's start address ++ * ++ * Find the adjacent &drm_gpuva before the GPU VA with given &start address. ++ * ++ * Note that if there is any free space between the GPU VA mappings no mapping ++ * is returned. ++ * ++ * Returns: a pointer to the found &drm_gpuva or NULL if none was found ++ */ ++struct drm_gpuva * ++drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start) ++{ ++ if (!drm_gpuva_range_valid(mgr, start - 1, 1)) ++ return NULL; ++ ++ return drm_gpuva_it_iter_first(&mgr->rb.tree, start - 1, start); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_find_prev); ++ ++/** ++ * drm_gpuva_find_next() - find the &drm_gpuva after the given address ++ * @mgr: the &drm_gpuva_manager to search in ++ * @end: the given GPU VA's end address ++ * ++ * Find the adjacent &drm_gpuva after the GPU VA with given &end address. ++ * ++ * Note that if there is any free space between the GPU VA mappings no mapping ++ * is returned. ++ * ++ * Returns: a pointer to the found &drm_gpuva or NULL if none was found ++ */ ++struct drm_gpuva * ++drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end) ++{ ++ if (!drm_gpuva_range_valid(mgr, end, 1)) ++ return NULL; ++ ++ return drm_gpuva_it_iter_first(&mgr->rb.tree, end, end + 1); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_find_next); ++ ++/** ++ * drm_gpuva_interval_empty() - indicate whether a given interval of the VA space ++ * is empty ++ * @mgr: the &drm_gpuva_manager to check the range for ++ * @addr: the start address of the range ++ * @range: the range of the interval ++ * ++ * Returns: true if the interval is empty, false otherwise ++ */ ++bool ++drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++{ ++ return !drm_gpuva_find_first(mgr, addr, range); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_interval_empty); ++ ++/** ++ * drm_gpuva_map() - helper to insert a &drm_gpuva according to a ++ * &drm_gpuva_op_map ++ * @mgr: the &drm_gpuva_manager ++ * @va: the &drm_gpuva to insert ++ * @op: the &drm_gpuva_op_map to initialize @va with ++ * ++ * Initializes the @va from the @op and inserts it into the given @mgr. ++ */ ++void ++drm_gpuva_map(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva *va, ++ struct drm_gpuva_op_map *op) ++{ ++ drm_gpuva_init_from_op(va, op); ++ drm_gpuva_insert(mgr, va); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_map); ++ ++/** ++ * drm_gpuva_remap() - helper to remap a &drm_gpuva according to a ++ * &drm_gpuva_op_remap ++ * @prev: the &drm_gpuva to remap when keeping the start of a mapping ++ * @next: the &drm_gpuva to remap when keeping the end of a mapping ++ * @op: the &drm_gpuva_op_remap to initialize @prev and @next with ++ * ++ * Removes the currently mapped &drm_gpuva and remaps it using @prev and/or ++ * @next. ++ */ ++void ++drm_gpuva_remap(struct drm_gpuva *prev, ++ struct drm_gpuva *next, ++ struct drm_gpuva_op_remap *op) ++{ ++ struct drm_gpuva *curr = op->unmap->va; ++ struct drm_gpuva_manager *mgr = curr->mgr; ++ ++ drm_gpuva_remove(curr); ++ ++ if (op->prev) { ++ drm_gpuva_init_from_op(prev, op->prev); ++ drm_gpuva_insert(mgr, prev); ++ } ++ ++ if (op->next) { ++ drm_gpuva_init_from_op(next, op->next); ++ drm_gpuva_insert(mgr, next); ++ } ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_remap); ++ ++/** ++ * drm_gpuva_unmap() - helper to remove a &drm_gpuva according to a ++ * &drm_gpuva_op_unmap ++ * @op: the &drm_gpuva_op_unmap specifying the &drm_gpuva to remove ++ * ++ * Removes the &drm_gpuva associated with the &drm_gpuva_op_unmap. ++ */ ++void ++drm_gpuva_unmap(struct drm_gpuva_op_unmap *op) ++{ ++ drm_gpuva_remove(op->va); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_unmap); ++ ++static int ++op_map_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++ u64 addr, u64 range, ++ struct drm_gem_object *obj, u64 offset) ++{ ++ struct drm_gpuva_op op = {}; ++ ++ op.op = DRM_GPUVA_OP_MAP; ++ op.map.va.addr = addr; ++ op.map.va.range = range; ++ op.map.gem.obj = obj; ++ op.map.gem.offset = offset; ++ ++ return fn->sm_step_map(&op, priv); ++} ++ ++static int ++op_remap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++ struct drm_gpuva_op_map *prev, ++ struct drm_gpuva_op_map *next, ++ struct drm_gpuva_op_unmap *unmap) ++{ ++ struct drm_gpuva_op op = {}; ++ struct drm_gpuva_op_remap *r; ++ ++ op.op = DRM_GPUVA_OP_REMAP; ++ r = &op.remap; ++ r->prev = prev; ++ r->next = next; ++ r->unmap = unmap; ++ ++ return fn->sm_step_remap(&op, priv); ++} ++ ++static int ++op_unmap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++ struct drm_gpuva *va, bool merge) ++{ ++ struct drm_gpuva_op op = {}; ++ ++ op.op = DRM_GPUVA_OP_UNMAP; ++ op.unmap.va = va; ++ op.unmap.keep = merge; ++ ++ return fn->sm_step_unmap(&op, priv); ++} ++ ++static int ++__drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, ++ const struct drm_gpuva_fn_ops *ops, void *priv, ++ u64 req_addr, u64 req_range, ++ struct drm_gem_object *req_obj, u64 req_offset) ++{ ++ struct drm_gpuva *va, *next, *prev = NULL; ++ u64 req_end = req_addr + req_range; ++ int ret; ++ ++ if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range))) ++ return -EINVAL; ++ ++ drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) { ++ struct drm_gem_object *obj = va->gem.obj; ++ u64 offset = va->gem.offset; ++ u64 addr = va->va.addr; ++ u64 range = va->va.range; ++ u64 end = addr + range; ++ bool merge = !!va->gem.obj; ++ ++ if (addr == req_addr) { ++ merge &= obj == req_obj && ++ offset == req_offset; ++ ++ if (end == req_end) { ++ ret = op_unmap_cb(ops, priv, va, merge); ++ if (ret) ++ return ret; ++ break; ++ } ++ ++ if (end < req_end) { ++ ret = op_unmap_cb(ops, priv, va, merge); ++ if (ret) ++ return ret; ++ goto next; ++ } ++ ++ if (end > req_end) { ++ struct drm_gpuva_op_map n = { ++ .va.addr = req_end, ++ .va.range = range - req_range, ++ .gem.obj = obj, ++ .gem.offset = offset + req_range, ++ }; ++ struct drm_gpuva_op_unmap u = { ++ .va = va, ++ .keep = merge, ++ }; ++ ++ ret = op_remap_cb(ops, priv, NULL, &n, &u); ++ if (ret) ++ return ret; ++ break; ++ } ++ } else if (addr < req_addr) { ++ u64 ls_range = req_addr - addr; ++ struct drm_gpuva_op_map p = { ++ .va.addr = addr, ++ .va.range = ls_range, ++ .gem.obj = obj, ++ .gem.offset = offset, ++ }; ++ struct drm_gpuva_op_unmap u = { .va = va }; ++ ++ merge &= obj == req_obj && ++ offset + ls_range == req_offset; ++ u.keep = merge; ++ ++ if (end == req_end) { ++ ret = op_remap_cb(ops, priv, &p, NULL, &u); ++ if (ret) ++ return ret; ++ break; ++ } ++ ++ if (end < req_end) { ++ ret = op_remap_cb(ops, priv, &p, NULL, &u); ++ if (ret) ++ return ret; ++ goto next; ++ } ++ ++ if (end > req_end) { ++ struct drm_gpuva_op_map n = { ++ .va.addr = req_end, ++ .va.range = end - req_end, ++ .gem.obj = obj, ++ .gem.offset = offset + ls_range + ++ req_range, ++ }; ++ ++ ret = op_remap_cb(ops, priv, &p, &n, &u); ++ if (ret) ++ return ret; ++ break; ++ } ++ } else if (addr > req_addr) { ++ merge &= obj == req_obj && ++ offset == req_offset + ++ (addr - req_addr); ++ ++ if (end == req_end) { ++ ret = op_unmap_cb(ops, priv, va, merge); ++ if (ret) ++ return ret; ++ break; ++ } ++ ++ if (end < req_end) { ++ ret = op_unmap_cb(ops, priv, va, merge); ++ if (ret) ++ return ret; ++ goto next; ++ } ++ ++ if (end > req_end) { ++ struct drm_gpuva_op_map n = { ++ .va.addr = req_end, ++ .va.range = end - req_end, ++ .gem.obj = obj, ++ .gem.offset = offset + req_end - addr, ++ }; ++ struct drm_gpuva_op_unmap u = { ++ .va = va, ++ .keep = merge, ++ }; ++ ++ ret = op_remap_cb(ops, priv, NULL, &n, &u); ++ if (ret) ++ return ret; ++ break; ++ } ++ } ++next: ++ prev = va; ++ } ++ ++ return op_map_cb(ops, priv, ++ req_addr, req_range, ++ req_obj, req_offset); ++} ++ ++static int ++__drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, ++ const struct drm_gpuva_fn_ops *ops, void *priv, ++ u64 req_addr, u64 req_range) ++{ ++ struct drm_gpuva *va, *next; ++ u64 req_end = req_addr + req_range; ++ int ret; ++ ++ if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range))) ++ return -EINVAL; ++ ++ drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) { ++ struct drm_gpuva_op_map prev = {}, next = {}; ++ bool prev_split = false, next_split = false; ++ struct drm_gem_object *obj = va->gem.obj; ++ u64 offset = va->gem.offset; ++ u64 addr = va->va.addr; ++ u64 range = va->va.range; ++ u64 end = addr + range; ++ ++ if (addr < req_addr) { ++ prev.va.addr = addr; ++ prev.va.range = req_addr - addr; ++ prev.gem.obj = obj; ++ prev.gem.offset = offset; ++ ++ prev_split = true; ++ } ++ ++ if (end > req_end) { ++ next.va.addr = req_end; ++ next.va.range = end - req_end; ++ next.gem.obj = obj; ++ next.gem.offset = offset + (req_end - addr); ++ ++ next_split = true; ++ } ++ ++ if (prev_split || next_split) { ++ struct drm_gpuva_op_unmap unmap = { .va = va }; ++ ++ ret = op_remap_cb(ops, priv, ++ prev_split ? &prev : NULL, ++ next_split ? &next : NULL, ++ &unmap); ++ if (ret) ++ return ret; ++ } else { ++ ret = op_unmap_cb(ops, priv, va, false); ++ if (ret) ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * drm_gpuva_sm_map() - creates the &drm_gpuva_op split/merge steps ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @req_addr: the start address of the new mapping ++ * @req_range: the range of the new mapping ++ * @req_obj: the &drm_gem_object to map ++ * @req_offset: the offset within the &drm_gem_object ++ * @priv: pointer to a driver private data structure ++ * ++ * This function iterates the given range of the GPU VA space. It utilizes the ++ * &drm_gpuva_fn_ops to call back into the driver providing the split and merge ++ * steps. ++ * ++ * Drivers may use these callbacks to update the GPU VA space right away within ++ * the callback. In case the driver decides to copy and store the operations for ++ * later processing neither this function nor &drm_gpuva_sm_unmap is allowed to ++ * be called before the &drm_gpuva_manager's view of the GPU VA space was ++ * updated with the previous set of operations. To update the ++ * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be ++ * used. ++ * ++ * A sequence of callbacks can contain map, unmap and remap operations, but ++ * the sequence of callbacks might also be empty if no operation is required, ++ * e.g. if the requested mapping already exists in the exact same way. ++ * ++ * There can be an arbitrary amount of unmap operations, a maximum of two remap ++ * operations and a single map operation. The latter one represents the original ++ * map operation requested by the caller. ++ * ++ * Returns: 0 on success or a negative error code ++ */ ++int ++drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv, ++ u64 req_addr, u64 req_range, ++ struct drm_gem_object *req_obj, u64 req_offset) ++{ ++ const struct drm_gpuva_fn_ops *ops = mgr->ops; ++ ++ if (unlikely(!(ops && ops->sm_step_map && ++ ops->sm_step_remap && ++ ops->sm_step_unmap))) ++ return -EINVAL; ++ ++ return __drm_gpuva_sm_map(mgr, ops, priv, ++ req_addr, req_range, ++ req_obj, req_offset); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_sm_map); ++ ++/** ++ * drm_gpuva_sm_unmap() - creates the &drm_gpuva_ops to split on unmap ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @priv: pointer to a driver private data structure ++ * @req_addr: the start address of the range to unmap ++ * @req_range: the range of the mappings to unmap ++ * ++ * This function iterates the given range of the GPU VA space. It utilizes the ++ * &drm_gpuva_fn_ops to call back into the driver providing the operations to ++ * unmap and, if required, split existent mappings. ++ * ++ * Drivers may use these callbacks to update the GPU VA space right away within ++ * the callback. In case the driver decides to copy and store the operations for ++ * later processing neither this function nor &drm_gpuva_sm_map is allowed to be ++ * called before the &drm_gpuva_manager's view of the GPU VA space was updated ++ * with the previous set of operations. To update the &drm_gpuva_manager's view ++ * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or ++ * drm_gpuva_destroy_unlocked() should be used. ++ * ++ * A sequence of callbacks can contain unmap and remap operations, depending on ++ * whether there are actual overlapping mappings to split. ++ * ++ * There can be an arbitrary amount of unmap operations and a maximum of two ++ * remap operations. ++ * ++ * Returns: 0 on success or a negative error code ++ */ ++int ++drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv, ++ u64 req_addr, u64 req_range) ++{ ++ const struct drm_gpuva_fn_ops *ops = mgr->ops; ++ ++ if (unlikely(!(ops && ops->sm_step_remap && ++ ops->sm_step_unmap))) ++ return -EINVAL; ++ ++ return __drm_gpuva_sm_unmap(mgr, ops, priv, ++ req_addr, req_range); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap); ++ ++static struct drm_gpuva_op * ++gpuva_op_alloc(struct drm_gpuva_manager *mgr) ++{ ++ const struct drm_gpuva_fn_ops *fn = mgr->ops; ++ struct drm_gpuva_op *op; ++ ++ if (fn && fn->op_alloc) ++ op = fn->op_alloc(); ++ else ++ op = kzalloc(sizeof(*op), GFP_KERNEL); ++ ++ if (unlikely(!op)) ++ return NULL; ++ ++ return op; ++} ++ ++static void ++gpuva_op_free(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva_op *op) ++{ ++ const struct drm_gpuva_fn_ops *fn = mgr->ops; ++ ++ if (fn && fn->op_free) ++ fn->op_free(op); ++ else ++ kfree(op); ++} ++ ++static int ++drm_gpuva_sm_step(struct drm_gpuva_op *__op, ++ void *priv) ++{ ++ struct { ++ struct drm_gpuva_manager *mgr; ++ struct drm_gpuva_ops *ops; ++ } *args = priv; ++ struct drm_gpuva_manager *mgr = args->mgr; ++ struct drm_gpuva_ops *ops = args->ops; ++ struct drm_gpuva_op *op; ++ ++ op = gpuva_op_alloc(mgr); ++ if (unlikely(!op)) ++ goto err; ++ ++ memcpy(op, __op, sizeof(*op)); ++ ++ if (op->op == DRM_GPUVA_OP_REMAP) { ++ struct drm_gpuva_op_remap *__r = &__op->remap; ++ struct drm_gpuva_op_remap *r = &op->remap; ++ ++ r->unmap = kmemdup(__r->unmap, sizeof(*r->unmap), ++ GFP_KERNEL); ++ if (unlikely(!r->unmap)) ++ goto err_free_op; ++ ++ if (__r->prev) { ++ r->prev = kmemdup(__r->prev, sizeof(*r->prev), ++ GFP_KERNEL); ++ if (unlikely(!r->prev)) ++ goto err_free_unmap; ++ } ++ ++ if (__r->next) { ++ r->next = kmemdup(__r->next, sizeof(*r->next), ++ GFP_KERNEL); ++ if (unlikely(!r->next)) ++ goto err_free_prev; ++ } ++ } ++ ++ list_add_tail(&op->entry, &ops->list); ++ ++ return 0; ++ ++err_free_unmap: ++ kfree(op->remap.unmap); ++err_free_prev: ++ kfree(op->remap.prev); ++err_free_op: ++ gpuva_op_free(mgr, op); ++err: ++ return -ENOMEM; ++} ++ ++static const struct drm_gpuva_fn_ops gpuva_list_ops = { ++ .sm_step_map = drm_gpuva_sm_step, ++ .sm_step_remap = drm_gpuva_sm_step, ++ .sm_step_unmap = drm_gpuva_sm_step, ++}; ++ ++/** ++ * drm_gpuva_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @req_addr: the start address of the new mapping ++ * @req_range: the range of the new mapping ++ * @req_obj: the &drm_gem_object to map ++ * @req_offset: the offset within the &drm_gem_object ++ * ++ * This function creates a list of operations to perform splitting and merging ++ * of existent mapping(s) with the newly requested one. ++ * ++ * The list can be iterated with &drm_gpuva_for_each_op and must be processed ++ * in the given order. It can contain map, unmap and remap operations, but it ++ * also can be empty if no operation is required, e.g. if the requested mapping ++ * already exists is the exact same way. ++ * ++ * There can be an arbitrary amount of unmap operations, a maximum of two remap ++ * operations and a single map operation. The latter one represents the original ++ * map operation requested by the caller. ++ * ++ * Note that before calling this function again with another mapping request it ++ * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The ++ * previously obtained operations must be either processed or abandoned. To ++ * update the &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be ++ * used. ++ * ++ * After the caller finished processing the returned &drm_gpuva_ops, they must ++ * be freed with &drm_gpuva_ops_free. ++ * ++ * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure ++ */ ++struct drm_gpuva_ops * ++drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, ++ u64 req_addr, u64 req_range, ++ struct drm_gem_object *req_obj, u64 req_offset) ++{ ++ struct drm_gpuva_ops *ops; ++ struct { ++ struct drm_gpuva_manager *mgr; ++ struct drm_gpuva_ops *ops; ++ } args; ++ int ret; ++ ++ ops = kzalloc(sizeof(*ops), GFP_KERNEL); ++ if (unlikely(!ops)) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&ops->list); ++ ++ args.mgr = mgr; ++ args.ops = ops; ++ ++ ret = __drm_gpuva_sm_map(mgr, &gpuva_list_ops, &args, ++ req_addr, req_range, ++ req_obj, req_offset); ++ if (ret) ++ goto err_free_ops; ++ ++ return ops; ++ ++err_free_ops: ++ drm_gpuva_ops_free(mgr, ops); ++ return ERR_PTR(ret); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_sm_map_ops_create); ++ ++/** ++ * drm_gpuva_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on ++ * unmap ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @req_addr: the start address of the range to unmap ++ * @req_range: the range of the mappings to unmap ++ * ++ * This function creates a list of operations to perform unmapping and, if ++ * required, splitting of the mappings overlapping the unmap range. ++ * ++ * The list can be iterated with &drm_gpuva_for_each_op and must be processed ++ * in the given order. It can contain unmap and remap operations, depending on ++ * whether there are actual overlapping mappings to split. ++ * ++ * There can be an arbitrary amount of unmap operations and a maximum of two ++ * remap operations. ++ * ++ * Note that before calling this function again with another range to unmap it ++ * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The ++ * previously obtained operations must be processed or abandoned. To update the ++ * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be ++ * used. ++ * ++ * After the caller finished processing the returned &drm_gpuva_ops, they must ++ * be freed with &drm_gpuva_ops_free. ++ * ++ * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure ++ */ ++struct drm_gpuva_ops * ++drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, ++ u64 req_addr, u64 req_range) ++{ ++ struct drm_gpuva_ops *ops; ++ struct { ++ struct drm_gpuva_manager *mgr; ++ struct drm_gpuva_ops *ops; ++ } args; ++ int ret; ++ ++ ops = kzalloc(sizeof(*ops), GFP_KERNEL); ++ if (unlikely(!ops)) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&ops->list); ++ ++ args.mgr = mgr; ++ args.ops = ops; ++ ++ ret = __drm_gpuva_sm_unmap(mgr, &gpuva_list_ops, &args, ++ req_addr, req_range); ++ if (ret) ++ goto err_free_ops; ++ ++ return ops; ++ ++err_free_ops: ++ drm_gpuva_ops_free(mgr, ops); ++ return ERR_PTR(ret); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap_ops_create); ++ ++/** ++ * drm_gpuva_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @addr: the start address of the range to prefetch ++ * @range: the range of the mappings to prefetch ++ * ++ * This function creates a list of operations to perform prefetching. ++ * ++ * The list can be iterated with &drm_gpuva_for_each_op and must be processed ++ * in the given order. It can contain prefetch operations. ++ * ++ * There can be an arbitrary amount of prefetch operations. ++ * ++ * After the caller finished processing the returned &drm_gpuva_ops, they must ++ * be freed with &drm_gpuva_ops_free. ++ * ++ * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure ++ */ ++struct drm_gpuva_ops * ++drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range) ++{ ++ struct drm_gpuva_ops *ops; ++ struct drm_gpuva_op *op; ++ struct drm_gpuva *va; ++ u64 end = addr + range; ++ int ret; ++ ++ ops = kzalloc(sizeof(*ops), GFP_KERNEL); ++ if (!ops) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&ops->list); ++ ++ drm_gpuva_for_each_va_range(va, mgr, addr, end) { ++ op = gpuva_op_alloc(mgr); ++ if (!op) { ++ ret = -ENOMEM; ++ goto err_free_ops; ++ } ++ ++ op->op = DRM_GPUVA_OP_PREFETCH; ++ op->prefetch.va = va; ++ list_add_tail(&op->entry, &ops->list); ++ } ++ ++ return ops; ++ ++err_free_ops: ++ drm_gpuva_ops_free(mgr, ops); ++ return ERR_PTR(ret); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_prefetch_ops_create); ++ ++/** ++ * drm_gpuva_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM ++ * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @obj: the &drm_gem_object to unmap ++ * ++ * This function creates a list of operations to perform unmapping for every ++ * GPUVA attached to a GEM. ++ * ++ * The list can be iterated with &drm_gpuva_for_each_op and consists out of an ++ * arbitrary amount of unmap operations. ++ * ++ * After the caller finished processing the returned &drm_gpuva_ops, they must ++ * be freed with &drm_gpuva_ops_free. ++ * ++ * It is the callers responsibility to protect the GEMs GPUVA list against ++ * concurrent access using the GEMs dma_resv lock. ++ * ++ * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure ++ */ ++struct drm_gpuva_ops * ++drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, ++ struct drm_gem_object *obj) ++{ ++ struct drm_gpuva_ops *ops; ++ struct drm_gpuva_op *op; ++ struct drm_gpuva *va; ++ int ret; ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ ++ ops = kzalloc(sizeof(*ops), GFP_KERNEL); ++ if (!ops) ++ return ERR_PTR(-ENOMEM); ++ ++ INIT_LIST_HEAD(&ops->list); ++ ++ drm_gem_for_each_gpuva(va, obj) { ++ op = gpuva_op_alloc(mgr); ++ if (!op) { ++ ret = -ENOMEM; ++ goto err_free_ops; ++ } ++ ++ op->op = DRM_GPUVA_OP_UNMAP; ++ op->unmap.va = va; ++ list_add_tail(&op->entry, &ops->list); ++ } ++ ++ return ops; ++ ++err_free_ops: ++ drm_gpuva_ops_free(mgr, ops); ++ return ERR_PTR(ret); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_gem_unmap_ops_create); ++ ++/** ++ * drm_gpuva_ops_free() - free the given &drm_gpuva_ops ++ * @mgr: the &drm_gpuva_manager the ops were created for ++ * @ops: the &drm_gpuva_ops to free ++ * ++ * Frees the given &drm_gpuva_ops structure including all the ops associated ++ * with it. ++ */ ++void ++drm_gpuva_ops_free(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva_ops *ops) ++{ ++ struct drm_gpuva_op *op, *next; ++ ++ drm_gpuva_for_each_op_safe(op, next, ops) { ++ list_del(&op->entry); ++ ++ if (op->op == DRM_GPUVA_OP_REMAP) { ++ kfree(op->remap.prev); ++ kfree(op->remap.next); ++ kfree(op->remap.unmap); ++ } ++ ++ gpuva_op_free(mgr, op); ++ } ++ ++ kfree(ops); ++} ++EXPORT_SYMBOL_GPL(drm_gpuva_ops_free); +diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h +index f6159acb885695..4ae996ce8d4037 100644 +--- a/include/drm/drm_drv.h ++++ b/include/drm/drm_drv.h +@@ -94,6 +94,12 @@ enum drm_driver_feature { + * synchronization of command submission. + */ + DRIVER_SYNCOBJ_TIMELINE = BIT(6), ++ /** ++ * @DRIVER_GEM_GPUVA: ++ * ++ * Driver supports user defined GPU VA bindings for GEM objects. ++ */ ++ DRIVER_GEM_GPUVA = BIT(8), + + /* IMPORTANT: Below are all the legacy flags, add new ones above. */ + +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 60b2dda8d964b3..f6bb5e8ec3eda5 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -36,6 +36,8 @@ + + #include + #include ++#include ++#include + + #include + +@@ -337,6 +339,22 @@ struct drm_gem_object { + */ + struct dma_resv _resv; + ++ /** ++ * @gpuva: ++ * ++ * Provides the list of GPU VAs attached to this GEM object. ++ * ++ * Drivers should lock list accesses with the GEMs &dma_resv lock ++ * (&drm_gem_object.resv) or a custom lock if one is provided. ++ */ ++ struct { ++ struct list_head list; ++ ++#ifdef CONFIG_LOCKDEP ++ struct lockdep_map *lock_dep_map; ++#endif ++ } gpuva; ++ + /** + * @funcs: + * +@@ -477,4 +495,65 @@ unsigned long drm_gem_lru_scan(struct drm_gem_lru *lru, + unsigned long *remaining, + bool (*shrink)(struct drm_gem_object *obj)); + ++#ifdef CONFIG_LOCKDEP ++/** ++ * drm_gem_gpuva_set_lock() - Set the lock protecting accesses to the gpuva list. ++ * @obj: the &drm_gem_object ++ * @lock: the lock used to protect the gpuva list. The locking primitive ++ * must contain a dep_map field. ++ * ++ * Call this if you're not proctecting access to the gpuva list ++ * with the dma-resv lock, otherwise, drm_gem_gpuva_init() takes care ++ * of initializing lock_dep_map for you. ++ */ ++#define drm_gem_gpuva_set_lock(obj, lock) \ ++ if (!(obj)->gpuva.lock_dep_map) \ ++ (obj)->gpuva.lock_dep_map = &(lock)->dep_map ++#define drm_gem_gpuva_assert_lock_held(obj) \ ++ lockdep_assert(lock_is_held((obj)->gpuva.lock_dep_map)) ++#else ++#define drm_gem_gpuva_set_lock(obj, lock) do {} while (0) ++#define drm_gem_gpuva_assert_lock_held(obj) do {} while (0) ++#endif ++ ++/** ++ * drm_gem_gpuva_init() - initialize the gpuva list of a GEM object ++ * @obj: the &drm_gem_object ++ * ++ * This initializes the &drm_gem_object's &drm_gpuva list. ++ * ++ * Calling this function is only necessary for drivers intending to support the ++ * &drm_driver_feature DRIVER_GEM_GPUVA. ++ */ ++static inline void drm_gem_gpuva_init(struct drm_gem_object *obj) ++{ ++ INIT_LIST_HEAD(&obj->gpuva.list); ++ drm_gem_gpuva_set_lock(obj, &obj->resv->lock.base); ++} ++ ++/** ++ * drm_gem_for_each_gpuva() - iternator to walk over a list of gpuvas ++ * @entry__: &drm_gpuva structure to assign to in each iteration step ++ * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the ++ * &drm_gpuva_manager. ++ */ ++#define drm_gem_for_each_gpuva(entry__, obj__) \ ++ list_for_each_entry(entry__, &(obj__)->gpuva.list, gem.entry) ++ ++/** ++ * drm_gem_for_each_gpuva_safe() - iternator to safely walk over a list of ++ * gpuvas ++ * @entry__: &drm_gpuva structure to assign to in each iteration step ++ * @next__: &next &drm_gpuva to store the next step ++ * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the ++ * &drm_gem_object. It is implemented with list_for_each_entry_safe(), hence ++ * it is save against removal of elements. ++ */ ++#define drm_gem_for_each_gpuva_safe(entry__, next__, obj__) \ ++ list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, gem.entry) ++ + #endif /* __DRM_GEM_H__ */ +diff --git a/include/drm/drm_gpuva_mgr.h b/include/drm/drm_gpuva_mgr.h +new file mode 100644 +index 00000000000000..ed8d50200cc3a9 +--- /dev/null ++++ b/include/drm/drm_gpuva_mgr.h +@@ -0,0 +1,706 @@ ++/* SPDX-License-Identifier: GPL-2.0-only */ ++ ++#ifndef __DRM_GPUVA_MGR_H__ ++#define __DRM_GPUVA_MGR_H__ ++ ++/* ++ * Copyright (c) 2022 Red Hat. ++ * ++ * Permission is hereby granted, free of charge, to any person obtaining a ++ * copy of this software and associated documentation files (the "Software"), ++ * to deal in the Software without restriction, including without limitation ++ * the rights to use, copy, modify, merge, publish, distribute, sublicense, ++ * and/or sell copies of the Software, and to permit persons to whom the ++ * Software is furnished to do so, subject to the following conditions: ++ * ++ * The above copyright notice and this permission notice shall be included in ++ * all copies or substantial portions of the Software. ++ * ++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR ++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL ++ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR ++ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ++ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR ++ * OTHER DEALINGS IN THE SOFTWARE. ++ */ ++ ++#include ++#include ++#include ++ ++#include ++ ++struct drm_gpuva_manager; ++struct drm_gpuva_fn_ops; ++ ++/** ++ * enum drm_gpuva_flags - flags for struct drm_gpuva ++ */ ++enum drm_gpuva_flags { ++ /** ++ * @DRM_GPUVA_INVALIDATED: ++ * ++ * Flag indicating that the &drm_gpuva's backing GEM is invalidated. ++ */ ++ DRM_GPUVA_INVALIDATED = (1 << 0), ++ ++ /** ++ * @DRM_GPUVA_SPARSE: ++ * ++ * Flag indicating that the &drm_gpuva is a sparse mapping. ++ */ ++ DRM_GPUVA_SPARSE = (1 << 1), ++ ++ /** ++ * @DRM_GPUVA_USERBITS: user defined bits ++ */ ++ DRM_GPUVA_USERBITS = (1 << 2), ++}; ++ ++/** ++ * struct drm_gpuva - structure to track a GPU VA mapping ++ * ++ * This structure represents a GPU VA mapping and is associated with a ++ * &drm_gpuva_manager. ++ * ++ * Typically, this structure is embedded in bigger driver structures. ++ */ ++struct drm_gpuva { ++ /** ++ * @mgr: the &drm_gpuva_manager this object is associated with ++ */ ++ struct drm_gpuva_manager *mgr; ++ ++ /** ++ * @flags: the &drm_gpuva_flags for this mapping ++ */ ++ enum drm_gpuva_flags flags; ++ ++ /** ++ * @va: structure containing the address and range of the &drm_gpuva ++ */ ++ struct { ++ /** ++ * @addr: the start address ++ */ ++ u64 addr; ++ ++ /* ++ * @range: the range ++ */ ++ u64 range; ++ } va; ++ ++ /** ++ * @gem: structure containing the &drm_gem_object and it's offset ++ */ ++ struct { ++ /** ++ * @offset: the offset within the &drm_gem_object ++ */ ++ u64 offset; ++ ++ /** ++ * @obj: the mapped &drm_gem_object ++ */ ++ struct drm_gem_object *obj; ++ ++ /** ++ * @entry: the &list_head to attach this object to a &drm_gem_object ++ */ ++ struct list_head entry; ++ } gem; ++ ++ /** ++ * @rb: structure containing data to store &drm_gpuvas in a rb-tree ++ */ ++ struct { ++ /** ++ * @rb: the rb-tree node ++ */ ++ struct rb_node node; ++ ++ /** ++ * @entry: The &list_head to additionally connect &drm_gpuvas ++ * in the same order they appear in the interval tree. This is ++ * useful to keep iterating &drm_gpuvas from a start node found ++ * through the rb-tree while doing modifications on the rb-tree ++ * itself. ++ */ ++ struct list_head entry; ++ ++ /** ++ * @__subtree_last: needed by the interval tree, holding last-in-subtree ++ */ ++ u64 __subtree_last; ++ } rb; ++}; ++ ++int drm_gpuva_insert(struct drm_gpuva_manager *mgr, struct drm_gpuva *va); ++void drm_gpuva_remove(struct drm_gpuva *va); ++ ++void drm_gpuva_link(struct drm_gpuva *va); ++void drm_gpuva_unlink(struct drm_gpuva *va); ++ ++struct drm_gpuva *drm_gpuva_find(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range); ++struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range); ++struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start); ++struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end); ++ ++bool drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range); ++ ++static inline void drm_gpuva_init(struct drm_gpuva *va, u64 addr, u64 range, ++ struct drm_gem_object *obj, u64 offset) ++{ ++ va->va.addr = addr; ++ va->va.range = range; ++ va->gem.obj = obj; ++ va->gem.offset = offset; ++} ++ ++/** ++ * drm_gpuva_invalidate() - sets whether the backing GEM of this &drm_gpuva is ++ * invalidated ++ * @va: the &drm_gpuva to set the invalidate flag for ++ * @invalidate: indicates whether the &drm_gpuva is invalidated ++ */ ++static inline void drm_gpuva_invalidate(struct drm_gpuva *va, bool invalidate) ++{ ++ if (invalidate) ++ va->flags |= DRM_GPUVA_INVALIDATED; ++ else ++ va->flags &= ~DRM_GPUVA_INVALIDATED; ++} ++ ++/** ++ * drm_gpuva_invalidated() - indicates whether the backing BO of this &drm_gpuva ++ * is invalidated ++ * @va: the &drm_gpuva to check ++ */ ++static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) ++{ ++ return va->flags & DRM_GPUVA_INVALIDATED; ++} ++ ++/** ++ * struct drm_gpuva_manager - DRM GPU VA Manager ++ * ++ * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using ++ * &maple_tree structures. Typically, this structure is embedded in bigger ++ * driver structures. ++ * ++ * Drivers can pass addresses and ranges in an arbitrary unit, e.g. bytes or ++ * pages. ++ * ++ * There should be one manager instance per GPU virtual address space. ++ */ ++struct drm_gpuva_manager { ++ /** ++ * @name: the name of the DRM GPU VA space ++ */ ++ const char *name; ++ ++ /** ++ * @mm_start: start of the VA space ++ */ ++ u64 mm_start; ++ ++ /** ++ * @mm_range: length of the VA space ++ */ ++ u64 mm_range; ++ ++ /** ++ * @rb: structures to track &drm_gpuva entries ++ */ ++ struct { ++ /** ++ * @tree: the rb-tree to track GPU VA mappings ++ */ ++ struct rb_root_cached tree; ++ ++ /** ++ * @list: the &list_head to track GPU VA mappings ++ */ ++ struct list_head list; ++ } rb; ++ ++ /** ++ * @kernel_alloc_node: ++ * ++ * &drm_gpuva representing the address space cutout reserved for ++ * the kernel ++ */ ++ struct drm_gpuva kernel_alloc_node; ++ ++ /** ++ * @ops: &drm_gpuva_fn_ops providing the split/merge steps to drivers ++ */ ++ const struct drm_gpuva_fn_ops *ops; ++}; ++ ++void drm_gpuva_manager_init(struct drm_gpuva_manager *mgr, ++ const char *name, ++ u64 start_offset, u64 range, ++ u64 reserve_offset, u64 reserve_range, ++ const struct drm_gpuva_fn_ops *ops); ++void drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr); ++ ++static inline struct drm_gpuva * ++__drm_gpuva_next(struct drm_gpuva *va) ++{ ++ if (va && !list_is_last(&va->rb.entry, &va->mgr->rb.list)) ++ return list_next_entry(va, rb.entry); ++ ++ return NULL; ++} ++ ++/** ++ * drm_gpuva_for_each_va_range() - iterate over a range of &drm_gpuvas ++ * @va__: &drm_gpuva structure to assign to in each iteration step ++ * @mgr__: &drm_gpuva_manager to walk over ++ * @start__: starting offset, the first gpuva will overlap this ++ * @end__: ending offset, the last gpuva will start before this (but may ++ * overlap) ++ * ++ * This iterator walks over all &drm_gpuvas in the &drm_gpuva_manager that lie ++ * between @start__ and @end__. It is implemented similarly to list_for_each(), ++ * but is using the &drm_gpuva_manager's internal interval tree to accelerate ++ * the search for the starting &drm_gpuva, and hence isn't safe against removal ++ * of elements. It assumes that @end__ is within (or is the upper limit of) the ++ * &drm_gpuva_manager. This iterator does not skip over the &drm_gpuva_manager's ++ * @kernel_alloc_node. ++ */ ++#define drm_gpuva_for_each_va_range(va__, mgr__, start__, end__) \ ++ for (va__ = drm_gpuva_find_first((mgr__), (start__), (end__) - (start__)); \ ++ va__ && (va__->va.addr < (end__)); \ ++ va__ = __drm_gpuva_next(va__)) ++ ++/** ++ * drm_gpuva_for_each_va_range_safe() - safely iterate over a range of ++ * &drm_gpuvas ++ * @va__: &drm_gpuva to assign to in each iteration step ++ * @next__: another &drm_gpuva to use as temporary storage ++ * @mgr__: &drm_gpuva_manager to walk over ++ * @start__: starting offset, the first gpuva will overlap this ++ * @end__: ending offset, the last gpuva will start before this (but may ++ * overlap) ++ * ++ * This iterator walks over all &drm_gpuvas in the &drm_gpuva_manager that lie ++ * between @start__ and @end__. It is implemented similarly to ++ * list_for_each_safe(), but is using the &drm_gpuva_manager's internal interval ++ * tree to accelerate the search for the starting &drm_gpuva, and hence is safe ++ * against removal of elements. It assumes that @end__ is within (or is the ++ * upper limit of) the &drm_gpuva_manager. This iterator does not skip over the ++ * &drm_gpuva_manager's @kernel_alloc_node. ++ */ ++#define drm_gpuva_for_each_va_range_safe(va__, next__, mgr__, start__, end__) \ ++ for (va__ = drm_gpuva_find_first((mgr__), (start__), (end__) - (start__)), \ ++ next__ = __drm_gpuva_next(va__); \ ++ va__ && (va__->va.addr < (end__)); \ ++ va__ = next__, next__ = __drm_gpuva_next(va__)) ++ ++/** ++ * drm_gpuva_for_each_va() - iterate over all &drm_gpuvas ++ * @va__: &drm_gpuva to assign to in each iteration step ++ * @mgr__: &drm_gpuva_manager to walk over ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the given ++ * &drm_gpuva_manager. ++ */ ++#define drm_gpuva_for_each_va(va__, mgr__) \ ++ list_for_each_entry(va__, &(mgr__)->rb.list, rb.entry) ++ ++/** ++ * drm_gpuva_for_each_va_safe() - safely iterate over all &drm_gpuvas ++ * @va__: &drm_gpuva to assign to in each iteration step ++ * @next__: another &drm_gpuva to use as temporary storage ++ * @mgr__: &drm_gpuva_manager to walk over ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the given ++ * &drm_gpuva_manager. It is implemented with list_for_each_entry_safe(), and ++ * hence safe against the removal of elements. ++ */ ++#define drm_gpuva_for_each_va_safe(va__, next__, mgr__) \ ++ list_for_each_entry_safe(va__, next__, &(mgr__)->rb.list, rb.entry) ++ ++/** ++ * enum drm_gpuva_op_type - GPU VA operation type ++ * ++ * Operations to alter the GPU VA mappings tracked by the &drm_gpuva_manager. ++ */ ++enum drm_gpuva_op_type { ++ /** ++ * @DRM_GPUVA_OP_MAP: the map op type ++ */ ++ DRM_GPUVA_OP_MAP, ++ ++ /** ++ * @DRM_GPUVA_OP_REMAP: the remap op type ++ */ ++ DRM_GPUVA_OP_REMAP, ++ ++ /** ++ * @DRM_GPUVA_OP_UNMAP: the unmap op type ++ */ ++ DRM_GPUVA_OP_UNMAP, ++ ++ /** ++ * @DRM_GPUVA_OP_PREFETCH: the prefetch op type ++ */ ++ DRM_GPUVA_OP_PREFETCH, ++}; ++ ++/** ++ * struct drm_gpuva_op_map - GPU VA map operation ++ * ++ * This structure represents a single map operation generated by the ++ * DRM GPU VA manager. ++ */ ++struct drm_gpuva_op_map { ++ /** ++ * @va: structure containing address and range of a map ++ * operation ++ */ ++ struct { ++ /** ++ * @addr: the base address of the new mapping ++ */ ++ u64 addr; ++ ++ /** ++ * @range: the range of the new mapping ++ */ ++ u64 range; ++ } va; ++ ++ /** ++ * @gem: structure containing the &drm_gem_object and it's offset ++ */ ++ struct { ++ /** ++ * @offset: the offset within the &drm_gem_object ++ */ ++ u64 offset; ++ ++ /** ++ * @obj: the &drm_gem_object to map ++ */ ++ struct drm_gem_object *obj; ++ } gem; ++}; ++ ++/** ++ * struct drm_gpuva_op_unmap - GPU VA unmap operation ++ * ++ * This structure represents a single unmap operation generated by the ++ * DRM GPU VA manager. ++ */ ++struct drm_gpuva_op_unmap { ++ /** ++ * @va: the &drm_gpuva to unmap ++ */ ++ struct drm_gpuva *va; ++ ++ /** ++ * @keep: ++ * ++ * Indicates whether this &drm_gpuva is physically contiguous with the ++ * original mapping request. ++ * ++ * Optionally, if &keep is set, drivers may keep the actual page table ++ * mappings for this &drm_gpuva, adding the missing page table entries ++ * only and update the &drm_gpuva_manager accordingly. ++ */ ++ bool keep; ++}; ++ ++/** ++ * struct drm_gpuva_op_remap - GPU VA remap operation ++ * ++ * This represents a single remap operation generated by the DRM GPU VA manager. ++ * ++ * A remap operation is generated when an existing GPU VA mmapping is split up ++ * by inserting a new GPU VA mapping or by partially unmapping existent ++ * mapping(s), hence it consists of a maximum of two map and one unmap ++ * operation. ++ * ++ * The @unmap operation takes care of removing the original existing mapping. ++ * @prev is used to remap the preceding part, @next the subsequent part. ++ * ++ * If either a new mapping's start address is aligned with the start address ++ * of the old mapping or the new mapping's end address is aligned with the ++ * end address of the old mapping, either @prev or @next is NULL. ++ * ++ * Note, the reason for a dedicated remap operation, rather than arbitrary ++ * unmap and map operations, is to give drivers the chance of extracting driver ++ * specific data for creating the new mappings from the unmap operations's ++ * &drm_gpuva structure which typically is embedded in larger driver specific ++ * structures. ++ */ ++struct drm_gpuva_op_remap { ++ /** ++ * @prev: the preceding part of a split mapping ++ */ ++ struct drm_gpuva_op_map *prev; ++ ++ /** ++ * @next: the subsequent part of a split mapping ++ */ ++ struct drm_gpuva_op_map *next; ++ ++ /** ++ * @unmap: the unmap operation for the original existing mapping ++ */ ++ struct drm_gpuva_op_unmap *unmap; ++}; ++ ++/** ++ * struct drm_gpuva_op_prefetch - GPU VA prefetch operation ++ * ++ * This structure represents a single prefetch operation generated by the ++ * DRM GPU VA manager. ++ */ ++struct drm_gpuva_op_prefetch { ++ /** ++ * @va: the &drm_gpuva to prefetch ++ */ ++ struct drm_gpuva *va; ++}; ++ ++/** ++ * struct drm_gpuva_op - GPU VA operation ++ * ++ * This structure represents a single generic operation. ++ * ++ * The particular type of the operation is defined by @op. ++ */ ++struct drm_gpuva_op { ++ /** ++ * @entry: ++ * ++ * The &list_head used to distribute instances of this struct within ++ * &drm_gpuva_ops. ++ */ ++ struct list_head entry; ++ ++ /** ++ * @op: the type of the operation ++ */ ++ enum drm_gpuva_op_type op; ++ ++ union { ++ /** ++ * @map: the map operation ++ */ ++ struct drm_gpuva_op_map map; ++ ++ /** ++ * @remap: the remap operation ++ */ ++ struct drm_gpuva_op_remap remap; ++ ++ /** ++ * @unmap: the unmap operation ++ */ ++ struct drm_gpuva_op_unmap unmap; ++ ++ /** ++ * @prefetch: the prefetch operation ++ */ ++ struct drm_gpuva_op_prefetch prefetch; ++ }; ++}; ++ ++/** ++ * struct drm_gpuva_ops - wraps a list of &drm_gpuva_op ++ */ ++struct drm_gpuva_ops { ++ /** ++ * @list: the &list_head ++ */ ++ struct list_head list; ++}; ++ ++/** ++ * drm_gpuva_for_each_op() - iterator to walk over &drm_gpuva_ops ++ * @op: &drm_gpuva_op to assign in each iteration step ++ * @ops: &drm_gpuva_ops to walk ++ * ++ * This iterator walks over all ops within a given list of operations. ++ */ ++#define drm_gpuva_for_each_op(op, ops) list_for_each_entry(op, &(ops)->list, entry) ++ ++/** ++ * drm_gpuva_for_each_op_safe() - iterator to safely walk over &drm_gpuva_ops ++ * @op: &drm_gpuva_op to assign in each iteration step ++ * @next: &next &drm_gpuva_op to store the next step ++ * @ops: &drm_gpuva_ops to walk ++ * ++ * This iterator walks over all ops within a given list of operations. It is ++ * implemented with list_for_each_safe(), so save against removal of elements. ++ */ ++#define drm_gpuva_for_each_op_safe(op, next, ops) \ ++ list_for_each_entry_safe(op, next, &(ops)->list, entry) ++ ++/** ++ * drm_gpuva_for_each_op_from_reverse() - iterate backwards from the given point ++ * @op: &drm_gpuva_op to assign in each iteration step ++ * @ops: &drm_gpuva_ops to walk ++ * ++ * This iterator walks over all ops within a given list of operations beginning ++ * from the given operation in reverse order. ++ */ ++#define drm_gpuva_for_each_op_from_reverse(op, ops) \ ++ list_for_each_entry_from_reverse(op, &(ops)->list, entry) ++ ++/** ++ * drm_gpuva_first_op() - returns the first &drm_gpuva_op from &drm_gpuva_ops ++ * @ops: the &drm_gpuva_ops to get the fist &drm_gpuva_op from ++ */ ++#define drm_gpuva_first_op(ops) \ ++ list_first_entry(&(ops)->list, struct drm_gpuva_op, entry) ++ ++/** ++ * drm_gpuva_last_op() - returns the last &drm_gpuva_op from &drm_gpuva_ops ++ * @ops: the &drm_gpuva_ops to get the last &drm_gpuva_op from ++ */ ++#define drm_gpuva_last_op(ops) \ ++ list_last_entry(&(ops)->list, struct drm_gpuva_op, entry) ++ ++/** ++ * drm_gpuva_prev_op() - previous &drm_gpuva_op in the list ++ * @op: the current &drm_gpuva_op ++ */ ++#define drm_gpuva_prev_op(op) list_prev_entry(op, entry) ++ ++/** ++ * drm_gpuva_next_op() - next &drm_gpuva_op in the list ++ * @op: the current &drm_gpuva_op ++ */ ++#define drm_gpuva_next_op(op) list_next_entry(op, entry) ++ ++struct drm_gpuva_ops * ++drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range, ++ struct drm_gem_object *obj, u64 offset); ++struct drm_gpuva_ops * ++drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range); ++ ++struct drm_gpuva_ops * ++drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, ++ u64 addr, u64 range); ++ ++struct drm_gpuva_ops * ++drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, ++ struct drm_gem_object *obj); ++ ++void drm_gpuva_ops_free(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva_ops *ops); ++ ++static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, ++ struct drm_gpuva_op_map *op) ++{ ++ drm_gpuva_init(va, op->va.addr, op->va.range, ++ op->gem.obj, op->gem.offset); ++} ++ ++/** ++ * struct drm_gpuva_fn_ops - callbacks for split/merge steps ++ * ++ * This structure defines the callbacks used by &drm_gpuva_sm_map and ++ * &drm_gpuva_sm_unmap to provide the split/merge steps for map and unmap ++ * operations to drivers. ++ */ ++struct drm_gpuva_fn_ops { ++ /** ++ * @op_alloc: called when the &drm_gpuva_manager allocates ++ * a struct drm_gpuva_op ++ * ++ * Some drivers may want to embed struct drm_gpuva_op into driver ++ * specific structures. By implementing this callback drivers can ++ * allocate memory accordingly. ++ * ++ * This callback is optional. ++ */ ++ struct drm_gpuva_op *(*op_alloc)(void); ++ ++ /** ++ * @op_free: called when the &drm_gpuva_manager frees a ++ * struct drm_gpuva_op ++ * ++ * Some drivers may want to embed struct drm_gpuva_op into driver ++ * specific structures. By implementing this callback drivers can ++ * free the previously allocated memory accordingly. ++ * ++ * This callback is optional. ++ */ ++ void (*op_free)(struct drm_gpuva_op *op); ++ ++ /** ++ * @sm_step_map: called from &drm_gpuva_sm_map to finally insert the ++ * mapping once all previous steps were completed ++ * ++ * The &priv pointer matches the one the driver passed to ++ * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * ++ * Can be NULL if &drm_gpuva_sm_map is used. ++ */ ++ int (*sm_step_map)(struct drm_gpuva_op *op, void *priv); ++ ++ /** ++ * @sm_step_remap: called from &drm_gpuva_sm_map and ++ * &drm_gpuva_sm_unmap to split up an existent mapping ++ * ++ * This callback is called when existent mapping needs to be split up. ++ * This is the case when either a newly requested mapping overlaps or ++ * is enclosed by an existent mapping or a partial unmap of an existent ++ * mapping is requested. ++ * ++ * The &priv pointer matches the one the driver passed to ++ * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * ++ * Can be NULL if neither &drm_gpuva_sm_map nor &drm_gpuva_sm_unmap is ++ * used. ++ */ ++ int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv); ++ ++ /** ++ * @sm_step_unmap: called from &drm_gpuva_sm_map and ++ * &drm_gpuva_sm_unmap to unmap an existent mapping ++ * ++ * This callback is called when existent mapping needs to be unmapped. ++ * This is the case when either a newly requested mapping encloses an ++ * existent mapping or an unmap of an existent mapping is requested. ++ * ++ * The &priv pointer matches the one the driver passed to ++ * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * ++ * Can be NULL if neither &drm_gpuva_sm_map nor &drm_gpuva_sm_unmap is ++ * used. ++ */ ++ int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); ++}; ++ ++int drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv, ++ u64 addr, u64 range, ++ struct drm_gem_object *obj, u64 offset); ++ ++int drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv, ++ u64 addr, u64 range); ++ ++void drm_gpuva_map(struct drm_gpuva_manager *mgr, ++ struct drm_gpuva *va, ++ struct drm_gpuva_op_map *op); ++ ++void drm_gpuva_remap(struct drm_gpuva *prev, ++ struct drm_gpuva *next, ++ struct drm_gpuva_op_remap *op); ++ ++void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op); ++ ++#endif /* __DRM_GPUVA_MGR_H__ */ + +From 49468df11599ee74a447a01b9633b421e2fa53c6 Mon Sep 17 00:00:00 2001 +From: Steven Price +Date: Thu, 20 Jul 2023 11:44:02 +0100 +Subject: [PATCH 004/150] drm: manager: Fix printk format for size_t + +sizeof() returns a size_t which may be different to an unsigned long. +Use the correct format specifier of '%zu' to prevent compiler warnings. + +Fixes: e6303f323b1a ("drm: manager to keep track of GPUs VA mappings") +Reviewed-by: Danilo Krummrich +Signed-off-by: Steven Price +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/2bf64010-c40a-8b84-144c-5387412b579e@arm.com +--- + drivers/gpu/drm/drm_gpuva_mgr.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/drm_gpuva_mgr.c b/drivers/gpu/drm/drm_gpuva_mgr.c +index 0b80177592a649..f86bfad74ff8a2 100644 +--- a/drivers/gpu/drm/drm_gpuva_mgr.c ++++ b/drivers/gpu/drm/drm_gpuva_mgr.c +@@ -619,7 +619,7 @@ drm_gpuva_check_overflow(u64 addr, u64 range) + u64 end; + + return WARN(check_add_overflow(addr, range, &end), +- "GPUVA address limited to %lu bytes.\n", sizeof(end)); ++ "GPUVA address limited to %zu bytes.\n", sizeof(end)); + } + + static bool + +From 9480612b58add60f40fae96a1bea7edfcde80eb5 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Thu, 24 Aug 2023 01:31:12 +0200 +Subject: [PATCH 005/150] drm/gpuva_mgr: remove unused prev pointer in + __drm_gpuva_sm_map() + +The prev pointer in __drm_gpuva_sm_map() was used to implement automatic +merging of mappings. Since automatic merging did not make its way +upstream, remove this leftover. + +Fixes: e6303f323b1a ("drm: manager to keep track of GPUs VA mappings") +Signed-off-by: Danilo Krummrich +Reviewed-by: Dave Airlie +Link: https://patchwork.freedesktop.org/patch/msgid/20230823233119.2891-1-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuva_mgr.c | 10 ++++------ + 1 file changed, 4 insertions(+), 6 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuva_mgr.c b/drivers/gpu/drm/drm_gpuva_mgr.c +index f86bfad74ff8a2..ad99c9cfedac70 100644 +--- a/drivers/gpu/drm/drm_gpuva_mgr.c ++++ b/drivers/gpu/drm/drm_gpuva_mgr.c +@@ -1076,7 +1076,7 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + u64 req_addr, u64 req_range, + struct drm_gem_object *req_obj, u64 req_offset) + { +- struct drm_gpuva *va, *next, *prev = NULL; ++ struct drm_gpuva *va, *next; + u64 req_end = req_addr + req_range; + int ret; + +@@ -1106,7 +1106,7 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + ret = op_unmap_cb(ops, priv, va, merge); + if (ret) + return ret; +- goto next; ++ continue; + } + + if (end > req_end) { +@@ -1151,7 +1151,7 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + ret = op_remap_cb(ops, priv, &p, NULL, &u); + if (ret) + return ret; +- goto next; ++ continue; + } + + if (end > req_end) { +@@ -1184,7 +1184,7 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + ret = op_unmap_cb(ops, priv, va, merge); + if (ret) + return ret; +- goto next; ++ continue; + } + + if (end > req_end) { +@@ -1205,8 +1205,6 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + break; + } + } +-next: +- prev = va; + } + + return op_map_cb(ops, priv, + +From 657b776c17aed16814a9bab5a17d56f3d914bb9b Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 20 Sep 2023 16:42:34 +0200 +Subject: [PATCH 006/150] [MOD] drm/gpuvm: rename struct drm_gpuva_manager to + struct drm_gpuvm +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Rename struct drm_gpuva_manager to struct drm_gpuvm including +corresponding functions. This way the GPUVA manager's structures align +very well with the documentation of VM_BIND [1] and VM_BIND locking [2]. + +It also provides a better foundation for the naming of data structures +and functions introduced for implementing a common dma-resv per GPU-VM +including tracking of external and evicted objects in subsequent +patches. + +[1] Documentation/gpu/drm-vm-bind-async.rst +[2] Documentation/gpu/drm-vm-bind-locking.rst + +Cc: Thomas Hellström +Cc: Matthew Brost +Acked-by: Dave Airlie +Acked-by: Christian König +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20230920144343.64830-2-dakr@redhat.com +--- + drivers/gpu/drm/Makefile | 2 +- + .../gpu/drm/{drm_gpuva_mgr.c => drm_gpuvm.c} | 404 +++++++++--------- + include/drm/{drm_gpuva_mgr.h => drm_gpuvm.h} | 155 ++++--- + 3 files changed, 280 insertions(+), 281 deletions(-) + rename drivers/gpu/drm/{drm_gpuva_mgr.c => drm_gpuvm.c} (78%) + rename include/drm/{drm_gpuva_mgr.h => drm_gpuvm.h} (78%) + +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index ad092cd7f2e95f..f718d2c83ed215 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -12,7 +12,7 @@ drm-y := drm_aperture.o drm_auth.o drm_cache.o \ + drm_crtc.o drm_fourcc.o drm_modes.o drm_displayid.o \ + drm_trace_points.o drm_prime.o \ + drm_vma_manager.o \ +- drm_gpuva_mgr.o \ ++ drm_gpuvm.o \ + drm_modeset_lock.o drm_atomic.o drm_bridge.o \ + drm_framebuffer.o drm_connector.o drm_blend.o \ + drm_encoder.o drm_mode_object.o drm_property.o \ +diff --git a/drivers/gpu/drm/drm_gpuva_mgr.c b/drivers/gpu/drm/drm_gpuvm.c +similarity index 78% +rename from drivers/gpu/drm/drm_gpuva_mgr.c +rename to drivers/gpu/drm/drm_gpuvm.c +index ad99c9cfedac70..89dcc6c34de45e 100644 +--- a/drivers/gpu/drm/drm_gpuva_mgr.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -25,7 +25,7 @@ + * + */ + +-#include ++#include + + #include + #include +@@ -33,8 +33,8 @@ + /** + * DOC: Overview + * +- * The DRM GPU VA Manager, represented by struct drm_gpuva_manager keeps track +- * of a GPU's virtual address (VA) space and manages the corresponding virtual ++ * The DRM GPU VA Manager, represented by struct drm_gpuvm keeps track of a ++ * GPU's virtual address (VA) space and manages the corresponding virtual + * mappings represented by &drm_gpuva objects. It also keeps track of the + * mapping's backing &drm_gem_object buffers. + * +@@ -47,28 +47,28 @@ + * The GPU VA manager internally uses a rb-tree to manage the + * &drm_gpuva mappings within a GPU's virtual address space. + * +- * The &drm_gpuva_manager contains a special &drm_gpuva representing the ++ * The &drm_gpuvm structure contains a special &drm_gpuva representing the + * portion of VA space reserved by the kernel. This node is initialized together + * with the GPU VA manager instance and removed when the GPU VA manager is + * destroyed. + * +- * In a typical application drivers would embed struct drm_gpuva_manager and ++ * In a typical application drivers would embed struct drm_gpuvm and + * struct drm_gpuva within their own driver specific structures, there won't be + * any memory allocations of its own nor memory allocations of &drm_gpuva + * entries. + * +- * The data structures needed to store &drm_gpuvas within the &drm_gpuva_manager +- * are contained within struct drm_gpuva already. Hence, for inserting +- * &drm_gpuva entries from within dma-fence signalling critical sections it is +- * enough to pre-allocate the &drm_gpuva structures. ++ * The data structures needed to store &drm_gpuvas within the &drm_gpuvm are ++ * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva ++ * entries from within dma-fence signalling critical sections it is enough to ++ * pre-allocate the &drm_gpuva structures. + */ + + /** + * DOC: Split and Merge + * + * Besides its capability to manage and represent a GPU VA space, the +- * &drm_gpuva_manager also provides functions to let the &drm_gpuva_manager +- * calculate a sequence of operations to satisfy a given map or unmap request. ++ * GPU VA manager also provides functions to let the &drm_gpuvm calculate a ++ * sequence of operations to satisfy a given map or unmap request. + * + * Therefore the DRM GPU VA manager provides an algorithm implementing splitting + * and merging of existent GPU VA mappings with the ones that are requested to +@@ -76,16 +76,16 @@ + * implement Vulkan 'Sparse Memory Bindings' - drivers UAPIs often refer to this + * as VM BIND. + * +- * Drivers can call drm_gpuva_sm_map() to receive a sequence of callbacks ++ * Drivers can call drm_gpuvm_sm_map() to receive a sequence of callbacks + * containing map, unmap and remap operations for a given newly requested + * mapping. The sequence of callbacks represents the set of operations to + * execute in order to integrate the new mapping cleanly into the current state + * of the GPU VA space. + * + * Depending on how the new GPU VA mapping intersects with the existent mappings +- * of the GPU VA space the &drm_gpuva_fn_ops callbacks contain an arbitrary +- * amount of unmap operations, a maximum of two remap operations and a single +- * map operation. The caller might receive no callback at all if no operation is ++ * of the GPU VA space the &drm_gpuvm_ops callbacks contain an arbitrary amount ++ * of unmap operations, a maximum of two remap operations and a single map ++ * operation. The caller might receive no callback at all if no operation is + * required, e.g. if the requested mapping already exists in the exact same way. + * + * The single map operation represents the original map operation requested by +@@ -95,7 +95,7 @@ + * &drm_gpuva to unmap is physically contiguous with the original mapping + * request. Optionally, if 'keep' is set, drivers may keep the actual page table + * entries for this &drm_gpuva, adding the missing page table entries only and +- * update the &drm_gpuva_manager's view of things accordingly. ++ * update the &drm_gpuvm's view of things accordingly. + * + * Drivers may do the same optimization, namely delta page table updates, also + * for remap operations. This is possible since &drm_gpuva_op_remap consists of +@@ -106,34 +106,34 @@ + * the beginning and one at the end of the new mapping, hence there is a + * maximum of two remap operations. + * +- * Analogous to drm_gpuva_sm_map() drm_gpuva_sm_unmap() uses &drm_gpuva_fn_ops +- * to call back into the driver in order to unmap a range of GPU VA space. The ++ * Analogous to drm_gpuvm_sm_map() drm_gpuvm_sm_unmap() uses &drm_gpuvm_ops to ++ * call back into the driver in order to unmap a range of GPU VA space. The + * logic behind this function is way simpler though: For all existent mappings + * enclosed by the given range unmap operations are created. For mappings which + * are only partically located within the given range, remap operations are + * created such that those mappings are split up and re-mapped partically. + * +- * As an alternative to drm_gpuva_sm_map() and drm_gpuva_sm_unmap(), +- * drm_gpuva_sm_map_ops_create() and drm_gpuva_sm_unmap_ops_create() can be used ++ * As an alternative to drm_gpuvm_sm_map() and drm_gpuvm_sm_unmap(), ++ * drm_gpuvm_sm_map_ops_create() and drm_gpuvm_sm_unmap_ops_create() can be used + * to directly obtain an instance of struct drm_gpuva_ops containing a list of + * &drm_gpuva_op, which can be iterated with drm_gpuva_for_each_op(). This list + * contains the &drm_gpuva_ops analogous to the callbacks one would receive when +- * calling drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). While this way requires ++ * calling drm_gpuvm_sm_map() or drm_gpuvm_sm_unmap(). While this way requires + * more memory (to allocate the &drm_gpuva_ops), it provides drivers a way to + * iterate the &drm_gpuva_op multiple times, e.g. once in a context where memory + * allocations are possible (e.g. to allocate GPU page tables) and once in the + * dma-fence signalling critical path. + * +- * To update the &drm_gpuva_manager's view of the GPU VA space +- * drm_gpuva_insert() and drm_gpuva_remove() may be used. These functions can +- * safely be used from &drm_gpuva_fn_ops callbacks originating from +- * drm_gpuva_sm_map() or drm_gpuva_sm_unmap(). However, it might be more +- * convenient to use the provided helper functions drm_gpuva_map(), +- * drm_gpuva_remap() and drm_gpuva_unmap() instead. ++ * To update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert() and ++ * drm_gpuva_remove() may be used. These functions can safely be used from ++ * &drm_gpuvm_ops callbacks originating from drm_gpuvm_sm_map() or ++ * drm_gpuvm_sm_unmap(). However, it might be more convenient to use the ++ * provided helper functions drm_gpuva_map(), drm_gpuva_remap() and ++ * drm_gpuva_unmap() instead. + * + * The following diagram depicts the basic relationships of existent GPU VA + * mappings, a newly requested mapping and the resulting mappings as implemented +- * by drm_gpuva_sm_map() - it doesn't cover any arbitrary combinations of these. ++ * by drm_gpuvm_sm_map() - it doesn't cover any arbitrary combinations of these. + * + * 1) Requested mapping is identical. Replace it, but indicate the backing PTEs + * could be kept. +@@ -421,10 +421,10 @@ + * // Allocates a new &drm_gpuva. + * struct drm_gpuva * driver_gpuva_alloc(void); + * +- * // Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva ++ * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva + * // structure in individual driver structures and lock the dma-resv with + * // drm_exec or similar helpers. +- * int driver_mapping_create(struct drm_gpuva_manager *mgr, ++ * int driver_mapping_create(struct drm_gpuvm *gpuvm, + * u64 addr, u64 range, + * struct drm_gem_object *obj, u64 offset) + * { +@@ -432,7 +432,7 @@ + * struct drm_gpuva_op *op + * + * driver_lock_va_space(); +- * ops = drm_gpuva_sm_map_ops_create(mgr, addr, range, ++ * ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range, + * obj, offset); + * if (IS_ERR(ops)) + * return PTR_ERR(ops); +@@ -448,7 +448,7 @@ + * // free memory and unlock + * + * driver_vm_map(); +- * drm_gpuva_map(mgr, va, &op->map); ++ * drm_gpuva_map(gpuvm, va, &op->map); + * drm_gpuva_link(va); + * + * break; +@@ -504,23 +504,23 @@ + * 2) Receive a callback for each &drm_gpuva_op to create a new mapping:: + * + * struct driver_context { +- * struct drm_gpuva_manager *mgr; ++ * struct drm_gpuvm *gpuvm; + * struct drm_gpuva *new_va; + * struct drm_gpuva *prev_va; + * struct drm_gpuva *next_va; + * }; + * +- * // ops to pass to drm_gpuva_manager_init() +- * static const struct drm_gpuva_fn_ops driver_gpuva_ops = { ++ * // ops to pass to drm_gpuvm_init() ++ * static const struct drm_gpuvm_ops driver_gpuvm_ops = { + * .sm_step_map = driver_gpuva_map, + * .sm_step_remap = driver_gpuva_remap, + * .sm_step_unmap = driver_gpuva_unmap, + * }; + * +- * // Typically drivers would embedd the &drm_gpuva_manager and &drm_gpuva ++ * // Typically drivers would embedd the &drm_gpuvm and &drm_gpuva + * // structure in individual driver structures and lock the dma-resv with + * // drm_exec or similar helpers. +- * int driver_mapping_create(struct drm_gpuva_manager *mgr, ++ * int driver_mapping_create(struct drm_gpuvm *gpuvm, + * u64 addr, u64 range, + * struct drm_gem_object *obj, u64 offset) + * { +@@ -529,7 +529,7 @@ + * struct drm_gpuva_op *op; + * int ret = 0; + * +- * ctx.mgr = mgr; ++ * ctx.gpuvm = gpuvm; + * + * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); + * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); +@@ -540,7 +540,7 @@ + * } + * + * driver_lock_va_space(); +- * ret = drm_gpuva_sm_map(mgr, &ctx, addr, range, obj, offset); ++ * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); + * driver_unlock_va_space(); + * + * out: +@@ -554,7 +554,7 @@ + * { + * struct driver_context *ctx = __ctx; + * +- * drm_gpuva_map(ctx->mgr, ctx->new_va, &op->map); ++ * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); + * + * drm_gpuva_link(ctx->new_va); + * +@@ -609,12 +609,12 @@ INTERVAL_TREE_DEFINE(struct drm_gpuva, rb.node, u64, rb.__subtree_last, + GPUVA_START, GPUVA_LAST, static __maybe_unused, + drm_gpuva_it) + +-static int __drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va); + static void __drm_gpuva_remove(struct drm_gpuva *va); + + static bool +-drm_gpuva_check_overflow(u64 addr, u64 range) ++drm_gpuvm_check_overflow(u64 addr, u64 range) + { + u64 end; + +@@ -623,121 +623,121 @@ drm_gpuva_check_overflow(u64 addr, u64 range) + } + + static bool +-drm_gpuva_in_mm_range(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++drm_gpuvm_in_mm_range(struct drm_gpuvm *gpuvm, u64 addr, u64 range) + { + u64 end = addr + range; +- u64 mm_start = mgr->mm_start; +- u64 mm_end = mm_start + mgr->mm_range; ++ u64 mm_start = gpuvm->mm_start; ++ u64 mm_end = mm_start + gpuvm->mm_range; + + return addr >= mm_start && end <= mm_end; + } + + static bool +-drm_gpuva_in_kernel_node(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range) + { + u64 end = addr + range; +- u64 kstart = mgr->kernel_alloc_node.va.addr; +- u64 krange = mgr->kernel_alloc_node.va.range; ++ u64 kstart = gpuvm->kernel_alloc_node.va.addr; ++ u64 krange = gpuvm->kernel_alloc_node.va.range; + u64 kend = kstart + krange; + + return krange && addr < kend && kstart < end; + } + + static bool +-drm_gpuva_range_valid(struct drm_gpuva_manager *mgr, ++drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { +- return !drm_gpuva_check_overflow(addr, range) && +- drm_gpuva_in_mm_range(mgr, addr, range) && +- !drm_gpuva_in_kernel_node(mgr, addr, range); ++ return !drm_gpuvm_check_overflow(addr, range) && ++ drm_gpuvm_in_mm_range(gpuvm, addr, range) && ++ !drm_gpuvm_in_kernel_node(gpuvm, addr, range); + } + + /** +- * drm_gpuva_manager_init() - initialize a &drm_gpuva_manager +- * @mgr: pointer to the &drm_gpuva_manager to initialize ++ * drm_gpuvm_init() - initialize a &drm_gpuvm ++ * @gpuvm: pointer to the &drm_gpuvm to initialize + * @name: the name of the GPU VA space + * @start_offset: the start offset of the GPU VA space + * @range: the size of the GPU VA space + * @reserve_offset: the start of the kernel reserved GPU VA area + * @reserve_range: the size of the kernel reserved GPU VA area +- * @ops: &drm_gpuva_fn_ops called on &drm_gpuva_sm_map / &drm_gpuva_sm_unmap ++ * @ops: &drm_gpuvm_ops called on &drm_gpuvm_sm_map / &drm_gpuvm_sm_unmap + * +- * The &drm_gpuva_manager must be initialized with this function before use. ++ * The &drm_gpuvm must be initialized with this function before use. + * +- * Note that @mgr must be cleared to 0 before calling this function. The given ++ * Note that @gpuvm must be cleared to 0 before calling this function. The given + * &name is expected to be managed by the surrounding driver structures. + */ + void +-drm_gpuva_manager_init(struct drm_gpuva_manager *mgr, +- const char *name, +- u64 start_offset, u64 range, +- u64 reserve_offset, u64 reserve_range, +- const struct drm_gpuva_fn_ops *ops) ++drm_gpuvm_init(struct drm_gpuvm *gpuvm, ++ const char *name, ++ u64 start_offset, u64 range, ++ u64 reserve_offset, u64 reserve_range, ++ const struct drm_gpuvm_ops *ops) + { +- mgr->rb.tree = RB_ROOT_CACHED; +- INIT_LIST_HEAD(&mgr->rb.list); ++ gpuvm->rb.tree = RB_ROOT_CACHED; ++ INIT_LIST_HEAD(&gpuvm->rb.list); + +- drm_gpuva_check_overflow(start_offset, range); +- mgr->mm_start = start_offset; +- mgr->mm_range = range; ++ drm_gpuvm_check_overflow(start_offset, range); ++ gpuvm->mm_start = start_offset; ++ gpuvm->mm_range = range; + +- mgr->name = name ? name : "unknown"; +- mgr->ops = ops; ++ gpuvm->name = name ? name : "unknown"; ++ gpuvm->ops = ops; + +- memset(&mgr->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); ++ memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); + + if (reserve_range) { +- mgr->kernel_alloc_node.va.addr = reserve_offset; +- mgr->kernel_alloc_node.va.range = reserve_range; ++ gpuvm->kernel_alloc_node.va.addr = reserve_offset; ++ gpuvm->kernel_alloc_node.va.range = reserve_range; + +- if (likely(!drm_gpuva_check_overflow(reserve_offset, ++ if (likely(!drm_gpuvm_check_overflow(reserve_offset, + reserve_range))) +- __drm_gpuva_insert(mgr, &mgr->kernel_alloc_node); ++ __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); + } + } +-EXPORT_SYMBOL_GPL(drm_gpuva_manager_init); ++EXPORT_SYMBOL_GPL(drm_gpuvm_init); + + /** +- * drm_gpuva_manager_destroy() - cleanup a &drm_gpuva_manager +- * @mgr: pointer to the &drm_gpuva_manager to clean up ++ * drm_gpuvm_destroy() - cleanup a &drm_gpuvm ++ * @gpuvm: pointer to the &drm_gpuvm to clean up + * + * Note that it is a bug to call this function on a manager that still + * holds GPU VA mappings. + */ + void +-drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr) ++drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) + { +- mgr->name = NULL; ++ gpuvm->name = NULL; + +- if (mgr->kernel_alloc_node.va.range) +- __drm_gpuva_remove(&mgr->kernel_alloc_node); ++ if (gpuvm->kernel_alloc_node.va.range) ++ __drm_gpuva_remove(&gpuvm->kernel_alloc_node); + +- WARN(!RB_EMPTY_ROOT(&mgr->rb.tree.rb_root), ++ WARN(!RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), + "GPUVA tree is not empty, potentially leaking memory."); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_manager_destroy); ++EXPORT_SYMBOL_GPL(drm_gpuvm_destroy); + + static int +-__drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++__drm_gpuva_insert(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va) + { + struct rb_node *node; + struct list_head *head; + +- if (drm_gpuva_it_iter_first(&mgr->rb.tree, ++ if (drm_gpuva_it_iter_first(&gpuvm->rb.tree, + GPUVA_START(va), + GPUVA_LAST(va))) + return -EEXIST; + +- va->mgr = mgr; ++ va->vm = gpuvm; + +- drm_gpuva_it_insert(va, &mgr->rb.tree); ++ drm_gpuva_it_insert(va, &gpuvm->rb.tree); + + node = rb_prev(&va->rb.node); + if (node) + head = &(to_drm_gpuva(node))->rb.entry; + else +- head = &mgr->rb.list; ++ head = &gpuvm->rb.list; + + list_add(&va->rb.entry, head); + +@@ -746,36 +746,36 @@ __drm_gpuva_insert(struct drm_gpuva_manager *mgr, + + /** + * drm_gpuva_insert() - insert a &drm_gpuva +- * @mgr: the &drm_gpuva_manager to insert the &drm_gpuva in ++ * @gpuvm: the &drm_gpuvm to insert the &drm_gpuva in + * @va: the &drm_gpuva to insert + * + * Insert a &drm_gpuva with a given address and range into a +- * &drm_gpuva_manager. ++ * &drm_gpuvm. + * + * It is safe to use this function using the safe versions of iterating the GPU +- * VA space, such as drm_gpuva_for_each_va_safe() and +- * drm_gpuva_for_each_va_range_safe(). ++ * VA space, such as drm_gpuvm_for_each_va_safe() and ++ * drm_gpuvm_for_each_va_range_safe(). + * + * Returns: 0 on success, negative error code on failure. + */ + int +-drm_gpuva_insert(struct drm_gpuva_manager *mgr, ++drm_gpuva_insert(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va) + { + u64 addr = va->va.addr; + u64 range = va->va.range; + +- if (unlikely(!drm_gpuva_range_valid(mgr, addr, range))) ++ if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range))) + return -EINVAL; + +- return __drm_gpuva_insert(mgr, va); ++ return __drm_gpuva_insert(gpuvm, va); + } + EXPORT_SYMBOL_GPL(drm_gpuva_insert); + + static void + __drm_gpuva_remove(struct drm_gpuva *va) + { +- drm_gpuva_it_remove(va, &va->mgr->rb.tree); ++ drm_gpuva_it_remove(va, &va->vm->rb.tree); + list_del_init(&va->rb.entry); + } + +@@ -786,15 +786,15 @@ __drm_gpuva_remove(struct drm_gpuva *va) + * This removes the given &va from the underlaying tree. + * + * It is safe to use this function using the safe versions of iterating the GPU +- * VA space, such as drm_gpuva_for_each_va_safe() and +- * drm_gpuva_for_each_va_range_safe(). ++ * VA space, such as drm_gpuvm_for_each_va_safe() and ++ * drm_gpuvm_for_each_va_range_safe(). + */ + void + drm_gpuva_remove(struct drm_gpuva *va) + { +- struct drm_gpuva_manager *mgr = va->mgr; ++ struct drm_gpuvm *gpuvm = va->vm; + +- if (unlikely(va == &mgr->kernel_alloc_node)) { ++ if (unlikely(va == &gpuvm->kernel_alloc_node)) { + WARN(1, "Can't destroy kernel reserved node.\n"); + return; + } +@@ -853,37 +853,37 @@ EXPORT_SYMBOL_GPL(drm_gpuva_unlink); + + /** + * drm_gpuva_find_first() - find the first &drm_gpuva in the given range +- * @mgr: the &drm_gpuva_manager to search in ++ * @gpuvm: the &drm_gpuvm to search in + * @addr: the &drm_gpuvas address + * @range: the &drm_gpuvas range + * + * Returns: the first &drm_gpuva within the given range + */ + struct drm_gpuva * +-drm_gpuva_find_first(struct drm_gpuva_manager *mgr, ++drm_gpuva_find_first(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { + u64 last = addr + range - 1; + +- return drm_gpuva_it_iter_first(&mgr->rb.tree, addr, last); ++ return drm_gpuva_it_iter_first(&gpuvm->rb.tree, addr, last); + } + EXPORT_SYMBOL_GPL(drm_gpuva_find_first); + + /** + * drm_gpuva_find() - find a &drm_gpuva +- * @mgr: the &drm_gpuva_manager to search in ++ * @gpuvm: the &drm_gpuvm to search in + * @addr: the &drm_gpuvas address + * @range: the &drm_gpuvas range + * + * Returns: the &drm_gpuva at a given &addr and with a given &range + */ + struct drm_gpuva * +-drm_gpuva_find(struct drm_gpuva_manager *mgr, ++drm_gpuva_find(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { + struct drm_gpuva *va; + +- va = drm_gpuva_find_first(mgr, addr, range); ++ va = drm_gpuva_find_first(gpuvm, addr, range); + if (!va) + goto out; + +@@ -900,7 +900,7 @@ EXPORT_SYMBOL_GPL(drm_gpuva_find); + + /** + * drm_gpuva_find_prev() - find the &drm_gpuva before the given address +- * @mgr: the &drm_gpuva_manager to search in ++ * @gpuvm: the &drm_gpuvm to search in + * @start: the given GPU VA's start address + * + * Find the adjacent &drm_gpuva before the GPU VA with given &start address. +@@ -911,18 +911,18 @@ EXPORT_SYMBOL_GPL(drm_gpuva_find); + * Returns: a pointer to the found &drm_gpuva or NULL if none was found + */ + struct drm_gpuva * +-drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start) ++drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start) + { +- if (!drm_gpuva_range_valid(mgr, start - 1, 1)) ++ if (!drm_gpuvm_range_valid(gpuvm, start - 1, 1)) + return NULL; + +- return drm_gpuva_it_iter_first(&mgr->rb.tree, start - 1, start); ++ return drm_gpuva_it_iter_first(&gpuvm->rb.tree, start - 1, start); + } + EXPORT_SYMBOL_GPL(drm_gpuva_find_prev); + + /** + * drm_gpuva_find_next() - find the &drm_gpuva after the given address +- * @mgr: the &drm_gpuva_manager to search in ++ * @gpuvm: the &drm_gpuvm to search in + * @end: the given GPU VA's end address + * + * Find the adjacent &drm_gpuva after the GPU VA with given &end address. +@@ -933,47 +933,47 @@ EXPORT_SYMBOL_GPL(drm_gpuva_find_prev); + * Returns: a pointer to the found &drm_gpuva or NULL if none was found + */ + struct drm_gpuva * +-drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end) ++drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end) + { +- if (!drm_gpuva_range_valid(mgr, end, 1)) ++ if (!drm_gpuvm_range_valid(gpuvm, end, 1)) + return NULL; + +- return drm_gpuva_it_iter_first(&mgr->rb.tree, end, end + 1); ++ return drm_gpuva_it_iter_first(&gpuvm->rb.tree, end, end + 1); + } + EXPORT_SYMBOL_GPL(drm_gpuva_find_next); + + /** +- * drm_gpuva_interval_empty() - indicate whether a given interval of the VA space ++ * drm_gpuvm_interval_empty() - indicate whether a given interval of the VA space + * is empty +- * @mgr: the &drm_gpuva_manager to check the range for ++ * @gpuvm: the &drm_gpuvm to check the range for + * @addr: the start address of the range + * @range: the range of the interval + * + * Returns: true if the interval is empty, false otherwise + */ + bool +-drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range) ++drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range) + { +- return !drm_gpuva_find_first(mgr, addr, range); ++ return !drm_gpuva_find_first(gpuvm, addr, range); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_interval_empty); ++EXPORT_SYMBOL_GPL(drm_gpuvm_interval_empty); + + /** + * drm_gpuva_map() - helper to insert a &drm_gpuva according to a + * &drm_gpuva_op_map +- * @mgr: the &drm_gpuva_manager ++ * @gpuvm: the &drm_gpuvm + * @va: the &drm_gpuva to insert + * @op: the &drm_gpuva_op_map to initialize @va with + * +- * Initializes the @va from the @op and inserts it into the given @mgr. ++ * Initializes the @va from the @op and inserts it into the given @gpuvm. + */ + void +-drm_gpuva_map(struct drm_gpuva_manager *mgr, ++drm_gpuva_map(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va, + struct drm_gpuva_op_map *op) + { + drm_gpuva_init_from_op(va, op); +- drm_gpuva_insert(mgr, va); ++ drm_gpuva_insert(gpuvm, va); + } + EXPORT_SYMBOL_GPL(drm_gpuva_map); + +@@ -993,18 +993,18 @@ drm_gpuva_remap(struct drm_gpuva *prev, + struct drm_gpuva_op_remap *op) + { + struct drm_gpuva *curr = op->unmap->va; +- struct drm_gpuva_manager *mgr = curr->mgr; ++ struct drm_gpuvm *gpuvm = curr->vm; + + drm_gpuva_remove(curr); + + if (op->prev) { + drm_gpuva_init_from_op(prev, op->prev); +- drm_gpuva_insert(mgr, prev); ++ drm_gpuva_insert(gpuvm, prev); + } + + if (op->next) { + drm_gpuva_init_from_op(next, op->next); +- drm_gpuva_insert(mgr, next); ++ drm_gpuva_insert(gpuvm, next); + } + } + EXPORT_SYMBOL_GPL(drm_gpuva_remap); +@@ -1024,7 +1024,7 @@ drm_gpuva_unmap(struct drm_gpuva_op_unmap *op) + EXPORT_SYMBOL_GPL(drm_gpuva_unmap); + + static int +-op_map_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++op_map_cb(const struct drm_gpuvm_ops *fn, void *priv, + u64 addr, u64 range, + struct drm_gem_object *obj, u64 offset) + { +@@ -1040,7 +1040,7 @@ op_map_cb(const struct drm_gpuva_fn_ops *fn, void *priv, + } + + static int +-op_remap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++op_remap_cb(const struct drm_gpuvm_ops *fn, void *priv, + struct drm_gpuva_op_map *prev, + struct drm_gpuva_op_map *next, + struct drm_gpuva_op_unmap *unmap) +@@ -1058,7 +1058,7 @@ op_remap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, + } + + static int +-op_unmap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, ++op_unmap_cb(const struct drm_gpuvm_ops *fn, void *priv, + struct drm_gpuva *va, bool merge) + { + struct drm_gpuva_op op = {}; +@@ -1071,8 +1071,8 @@ op_unmap_cb(const struct drm_gpuva_fn_ops *fn, void *priv, + } + + static int +-__drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, +- const struct drm_gpuva_fn_ops *ops, void *priv, ++__drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, ++ const struct drm_gpuvm_ops *ops, void *priv, + u64 req_addr, u64 req_range, + struct drm_gem_object *req_obj, u64 req_offset) + { +@@ -1080,10 +1080,10 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + u64 req_end = req_addr + req_range; + int ret; + +- if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range))) ++ if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) + return -EINVAL; + +- drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) { ++ drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { + struct drm_gem_object *obj = va->gem.obj; + u64 offset = va->gem.offset; + u64 addr = va->va.addr; +@@ -1213,18 +1213,18 @@ __drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, + } + + static int +-__drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, +- const struct drm_gpuva_fn_ops *ops, void *priv, ++__drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, ++ const struct drm_gpuvm_ops *ops, void *priv, + u64 req_addr, u64 req_range) + { + struct drm_gpuva *va, *next; + u64 req_end = req_addr + req_range; + int ret; + +- if (unlikely(!drm_gpuva_range_valid(mgr, req_addr, req_range))) ++ if (unlikely(!drm_gpuvm_range_valid(gpuvm, req_addr, req_range))) + return -EINVAL; + +- drm_gpuva_for_each_va_range_safe(va, next, mgr, req_addr, req_end) { ++ drm_gpuvm_for_each_va_range_safe(va, next, gpuvm, req_addr, req_end) { + struct drm_gpuva_op_map prev = {}, next = {}; + bool prev_split = false, next_split = false; + struct drm_gem_object *obj = va->gem.obj; +@@ -1271,8 +1271,8 @@ __drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, + } + + /** +- * drm_gpuva_sm_map() - creates the &drm_gpuva_op split/merge steps +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * drm_gpuvm_sm_map() - creates the &drm_gpuva_op split/merge steps ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @req_addr: the start address of the new mapping + * @req_range: the range of the new mapping + * @req_obj: the &drm_gem_object to map +@@ -1280,15 +1280,15 @@ __drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, + * @priv: pointer to a driver private data structure + * + * This function iterates the given range of the GPU VA space. It utilizes the +- * &drm_gpuva_fn_ops to call back into the driver providing the split and merge ++ * &drm_gpuvm_ops to call back into the driver providing the split and merge + * steps. + * + * Drivers may use these callbacks to update the GPU VA space right away within + * the callback. In case the driver decides to copy and store the operations for +- * later processing neither this function nor &drm_gpuva_sm_unmap is allowed to +- * be called before the &drm_gpuva_manager's view of the GPU VA space was ++ * later processing neither this function nor &drm_gpuvm_sm_unmap is allowed to ++ * be called before the &drm_gpuvm's view of the GPU VA space was + * updated with the previous set of operations. To update the +- * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), + * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be + * used. + * +@@ -1303,39 +1303,39 @@ __drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, + * Returns: 0 on success or a negative error code + */ + int +-drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv, ++drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, + u64 req_addr, u64 req_range, + struct drm_gem_object *req_obj, u64 req_offset) + { +- const struct drm_gpuva_fn_ops *ops = mgr->ops; ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; + + if (unlikely(!(ops && ops->sm_step_map && + ops->sm_step_remap && + ops->sm_step_unmap))) + return -EINVAL; + +- return __drm_gpuva_sm_map(mgr, ops, priv, ++ return __drm_gpuvm_sm_map(gpuvm, ops, priv, + req_addr, req_range, + req_obj, req_offset); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_sm_map); ++EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map); + + /** +- * drm_gpuva_sm_unmap() - creates the &drm_gpuva_ops to split on unmap +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * drm_gpuvm_sm_unmap() - creates the &drm_gpuva_ops to split on unmap ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @priv: pointer to a driver private data structure + * @req_addr: the start address of the range to unmap + * @req_range: the range of the mappings to unmap + * + * This function iterates the given range of the GPU VA space. It utilizes the +- * &drm_gpuva_fn_ops to call back into the driver providing the operations to ++ * &drm_gpuvm_ops to call back into the driver providing the operations to + * unmap and, if required, split existent mappings. + * + * Drivers may use these callbacks to update the GPU VA space right away within + * the callback. In case the driver decides to copy and store the operations for +- * later processing neither this function nor &drm_gpuva_sm_map is allowed to be +- * called before the &drm_gpuva_manager's view of the GPU VA space was updated +- * with the previous set of operations. To update the &drm_gpuva_manager's view ++ * later processing neither this function nor &drm_gpuvm_sm_map is allowed to be ++ * called before the &drm_gpuvm's view of the GPU VA space was updated ++ * with the previous set of operations. To update the &drm_gpuvm's view + * of the GPU VA space drm_gpuva_insert(), drm_gpuva_destroy_locked() and/or + * drm_gpuva_destroy_unlocked() should be used. + * +@@ -1348,24 +1348,24 @@ EXPORT_SYMBOL_GPL(drm_gpuva_sm_map); + * Returns: 0 on success or a negative error code + */ + int +-drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv, ++drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, + u64 req_addr, u64 req_range) + { +- const struct drm_gpuva_fn_ops *ops = mgr->ops; ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; + + if (unlikely(!(ops && ops->sm_step_remap && + ops->sm_step_unmap))) + return -EINVAL; + +- return __drm_gpuva_sm_unmap(mgr, ops, priv, ++ return __drm_gpuvm_sm_unmap(gpuvm, ops, priv, + req_addr, req_range); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap); ++EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap); + + static struct drm_gpuva_op * +-gpuva_op_alloc(struct drm_gpuva_manager *mgr) ++gpuva_op_alloc(struct drm_gpuvm *gpuvm) + { +- const struct drm_gpuva_fn_ops *fn = mgr->ops; ++ const struct drm_gpuvm_ops *fn = gpuvm->ops; + struct drm_gpuva_op *op; + + if (fn && fn->op_alloc) +@@ -1380,10 +1380,10 @@ gpuva_op_alloc(struct drm_gpuva_manager *mgr) + } + + static void +-gpuva_op_free(struct drm_gpuva_manager *mgr, ++gpuva_op_free(struct drm_gpuvm *gpuvm, + struct drm_gpuva_op *op) + { +- const struct drm_gpuva_fn_ops *fn = mgr->ops; ++ const struct drm_gpuvm_ops *fn = gpuvm->ops; + + if (fn && fn->op_free) + fn->op_free(op); +@@ -1396,14 +1396,14 @@ drm_gpuva_sm_step(struct drm_gpuva_op *__op, + void *priv) + { + struct { +- struct drm_gpuva_manager *mgr; ++ struct drm_gpuvm *vm; + struct drm_gpuva_ops *ops; + } *args = priv; +- struct drm_gpuva_manager *mgr = args->mgr; ++ struct drm_gpuvm *gpuvm = args->vm; + struct drm_gpuva_ops *ops = args->ops; + struct drm_gpuva_op *op; + +- op = gpuva_op_alloc(mgr); ++ op = gpuva_op_alloc(gpuvm); + if (unlikely(!op)) + goto err; + +@@ -1442,20 +1442,20 @@ drm_gpuva_sm_step(struct drm_gpuva_op *__op, + err_free_prev: + kfree(op->remap.prev); + err_free_op: +- gpuva_op_free(mgr, op); ++ gpuva_op_free(gpuvm, op); + err: + return -ENOMEM; + } + +-static const struct drm_gpuva_fn_ops gpuva_list_ops = { ++static const struct drm_gpuvm_ops gpuvm_list_ops = { + .sm_step_map = drm_gpuva_sm_step, + .sm_step_remap = drm_gpuva_sm_step, + .sm_step_unmap = drm_gpuva_sm_step, + }; + + /** +- * drm_gpuva_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * drm_gpuvm_sm_map_ops_create() - creates the &drm_gpuva_ops to split and merge ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @req_addr: the start address of the new mapping + * @req_range: the range of the new mapping + * @req_obj: the &drm_gem_object to map +@@ -1474,9 +1474,9 @@ static const struct drm_gpuva_fn_ops gpuva_list_ops = { + * map operation requested by the caller. + * + * Note that before calling this function again with another mapping request it +- * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The ++ * is necessary to update the &drm_gpuvm's view of the GPU VA space. The + * previously obtained operations must be either processed or abandoned. To +- * update the &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * update the &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), + * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be + * used. + * +@@ -1486,13 +1486,13 @@ static const struct drm_gpuva_fn_ops gpuva_list_ops = { + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ + struct drm_gpuva_ops * +-drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, + u64 req_addr, u64 req_range, + struct drm_gem_object *req_obj, u64 req_offset) + { + struct drm_gpuva_ops *ops; + struct { +- struct drm_gpuva_manager *mgr; ++ struct drm_gpuvm *vm; + struct drm_gpuva_ops *ops; + } args; + int ret; +@@ -1503,10 +1503,10 @@ drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, + + INIT_LIST_HEAD(&ops->list); + +- args.mgr = mgr; ++ args.vm = gpuvm; + args.ops = ops; + +- ret = __drm_gpuva_sm_map(mgr, &gpuva_list_ops, &args, ++ ret = __drm_gpuvm_sm_map(gpuvm, &gpuvm_list_ops, &args, + req_addr, req_range, + req_obj, req_offset); + if (ret) +@@ -1515,15 +1515,15 @@ drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, + return ops; + + err_free_ops: +- drm_gpuva_ops_free(mgr, ops); ++ drm_gpuva_ops_free(gpuvm, ops); + return ERR_PTR(ret); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_sm_map_ops_create); ++EXPORT_SYMBOL_GPL(drm_gpuvm_sm_map_ops_create); + + /** +- * drm_gpuva_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on ++ * drm_gpuvm_sm_unmap_ops_create() - creates the &drm_gpuva_ops to split on + * unmap +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @req_addr: the start address of the range to unmap + * @req_range: the range of the mappings to unmap + * +@@ -1538,9 +1538,9 @@ EXPORT_SYMBOL_GPL(drm_gpuva_sm_map_ops_create); + * remap operations. + * + * Note that before calling this function again with another range to unmap it +- * is necessary to update the &drm_gpuva_manager's view of the GPU VA space. The ++ * is necessary to update the &drm_gpuvm's view of the GPU VA space. The + * previously obtained operations must be processed or abandoned. To update the +- * &drm_gpuva_manager's view of the GPU VA space drm_gpuva_insert(), ++ * &drm_gpuvm's view of the GPU VA space drm_gpuva_insert(), + * drm_gpuva_destroy_locked() and/or drm_gpuva_destroy_unlocked() should be + * used. + * +@@ -1550,12 +1550,12 @@ EXPORT_SYMBOL_GPL(drm_gpuva_sm_map_ops_create); + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ + struct drm_gpuva_ops * +-drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, + u64 req_addr, u64 req_range) + { + struct drm_gpuva_ops *ops; + struct { +- struct drm_gpuva_manager *mgr; ++ struct drm_gpuvm *vm; + struct drm_gpuva_ops *ops; + } args; + int ret; +@@ -1566,10 +1566,10 @@ drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, + + INIT_LIST_HEAD(&ops->list); + +- args.mgr = mgr; ++ args.vm = gpuvm; + args.ops = ops; + +- ret = __drm_gpuva_sm_unmap(mgr, &gpuva_list_ops, &args, ++ ret = __drm_gpuvm_sm_unmap(gpuvm, &gpuvm_list_ops, &args, + req_addr, req_range); + if (ret) + goto err_free_ops; +@@ -1577,14 +1577,14 @@ drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, + return ops; + + err_free_ops: +- drm_gpuva_ops_free(mgr, ops); ++ drm_gpuva_ops_free(gpuvm, ops); + return ERR_PTR(ret); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap_ops_create); ++EXPORT_SYMBOL_GPL(drm_gpuvm_sm_unmap_ops_create); + + /** +- * drm_gpuva_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * drm_gpuvm_prefetch_ops_create() - creates the &drm_gpuva_ops to prefetch ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @addr: the start address of the range to prefetch + * @range: the range of the mappings to prefetch + * +@@ -1601,7 +1601,7 @@ EXPORT_SYMBOL_GPL(drm_gpuva_sm_unmap_ops_create); + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ + struct drm_gpuva_ops * +-drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { + struct drm_gpuva_ops *ops; +@@ -1616,8 +1616,8 @@ drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, + + INIT_LIST_HEAD(&ops->list); + +- drm_gpuva_for_each_va_range(va, mgr, addr, end) { +- op = gpuva_op_alloc(mgr); ++ drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { ++ op = gpuva_op_alloc(gpuvm); + if (!op) { + ret = -ENOMEM; + goto err_free_ops; +@@ -1631,14 +1631,14 @@ drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, + return ops; + + err_free_ops: +- drm_gpuva_ops_free(mgr, ops); ++ drm_gpuva_ops_free(gpuvm, ops); + return ERR_PTR(ret); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_prefetch_ops_create); ++EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); + + /** +- * drm_gpuva_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM +- * @mgr: the &drm_gpuva_manager representing the GPU VA space ++ * drm_gpuvm_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM ++ * @gpuvm: the &drm_gpuvm representing the GPU VA space + * @obj: the &drm_gem_object to unmap + * + * This function creates a list of operations to perform unmapping for every +@@ -1656,7 +1656,7 @@ EXPORT_SYMBOL_GPL(drm_gpuva_prefetch_ops_create); + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ + struct drm_gpuva_ops * +-drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj) + { + struct drm_gpuva_ops *ops; +@@ -1673,7 +1673,7 @@ drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, + INIT_LIST_HEAD(&ops->list); + + drm_gem_for_each_gpuva(va, obj) { +- op = gpuva_op_alloc(mgr); ++ op = gpuva_op_alloc(gpuvm); + if (!op) { + ret = -ENOMEM; + goto err_free_ops; +@@ -1687,21 +1687,21 @@ drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, + return ops; + + err_free_ops: +- drm_gpuva_ops_free(mgr, ops); ++ drm_gpuva_ops_free(gpuvm, ops); + return ERR_PTR(ret); + } +-EXPORT_SYMBOL_GPL(drm_gpuva_gem_unmap_ops_create); ++EXPORT_SYMBOL_GPL(drm_gpuvm_gem_unmap_ops_create); + + /** + * drm_gpuva_ops_free() - free the given &drm_gpuva_ops +- * @mgr: the &drm_gpuva_manager the ops were created for ++ * @gpuvm: the &drm_gpuvm the ops were created for + * @ops: the &drm_gpuva_ops to free + * + * Frees the given &drm_gpuva_ops structure including all the ops associated + * with it. + */ + void +-drm_gpuva_ops_free(struct drm_gpuva_manager *mgr, ++drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, + struct drm_gpuva_ops *ops) + { + struct drm_gpuva_op *op, *next; +@@ -1715,7 +1715,7 @@ drm_gpuva_ops_free(struct drm_gpuva_manager *mgr, + kfree(op->remap.unmap); + } + +- gpuva_op_free(mgr, op); ++ gpuva_op_free(gpuvm, op); + } + + kfree(ops); +diff --git a/include/drm/drm_gpuva_mgr.h b/include/drm/drm_gpuvm.h +similarity index 78% +rename from include/drm/drm_gpuva_mgr.h +rename to include/drm/drm_gpuvm.h +index ed8d50200cc3a9..c7ed6bf441d4f4 100644 +--- a/include/drm/drm_gpuva_mgr.h ++++ b/include/drm/drm_gpuvm.h +@@ -1,7 +1,7 @@ + /* SPDX-License-Identifier: GPL-2.0-only */ + +-#ifndef __DRM_GPUVA_MGR_H__ +-#define __DRM_GPUVA_MGR_H__ ++#ifndef __DRM_GPUVM_H__ ++#define __DRM_GPUVM_H__ + + /* + * Copyright (c) 2022 Red Hat. +@@ -31,8 +31,8 @@ + + #include + +-struct drm_gpuva_manager; +-struct drm_gpuva_fn_ops; ++struct drm_gpuvm; ++struct drm_gpuvm_ops; + + /** + * enum drm_gpuva_flags - flags for struct drm_gpuva +@@ -62,15 +62,15 @@ enum drm_gpuva_flags { + * struct drm_gpuva - structure to track a GPU VA mapping + * + * This structure represents a GPU VA mapping and is associated with a +- * &drm_gpuva_manager. ++ * &drm_gpuvm. + * + * Typically, this structure is embedded in bigger driver structures. + */ + struct drm_gpuva { + /** +- * @mgr: the &drm_gpuva_manager this object is associated with ++ * @vm: the &drm_gpuvm this object is associated with + */ +- struct drm_gpuva_manager *mgr; ++ struct drm_gpuvm *vm; + + /** + * @flags: the &drm_gpuva_flags for this mapping +@@ -137,20 +137,18 @@ struct drm_gpuva { + } rb; + }; + +-int drm_gpuva_insert(struct drm_gpuva_manager *mgr, struct drm_gpuva *va); ++int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); + void drm_gpuva_remove(struct drm_gpuva *va); + + void drm_gpuva_link(struct drm_gpuva *va); + void drm_gpuva_unlink(struct drm_gpuva *va); + +-struct drm_gpuva *drm_gpuva_find(struct drm_gpuva_manager *mgr, ++struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, + u64 addr, u64 range); +-struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuva_manager *mgr, ++struct drm_gpuva *drm_gpuva_find_first(struct drm_gpuvm *gpuvm, + u64 addr, u64 range); +-struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuva_manager *mgr, u64 start); +-struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuva_manager *mgr, u64 end); +- +-bool drm_gpuva_interval_empty(struct drm_gpuva_manager *mgr, u64 addr, u64 range); ++struct drm_gpuva *drm_gpuva_find_prev(struct drm_gpuvm *gpuvm, u64 start); ++struct drm_gpuva *drm_gpuva_find_next(struct drm_gpuvm *gpuvm, u64 end); + + static inline void drm_gpuva_init(struct drm_gpuva *va, u64 addr, u64 range, + struct drm_gem_object *obj, u64 offset) +@@ -186,7 +184,7 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) + } + + /** +- * struct drm_gpuva_manager - DRM GPU VA Manager ++ * struct drm_gpuvm - DRM GPU VA Manager + * + * The DRM GPU VA Manager keeps track of a GPU's virtual address space by using + * &maple_tree structures. Typically, this structure is embedded in bigger +@@ -197,7 +195,7 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) + * + * There should be one manager instance per GPU virtual address space. + */ +-struct drm_gpuva_manager { ++struct drm_gpuvm { + /** + * @name: the name of the DRM GPU VA space + */ +@@ -237,100 +235,101 @@ struct drm_gpuva_manager { + struct drm_gpuva kernel_alloc_node; + + /** +- * @ops: &drm_gpuva_fn_ops providing the split/merge steps to drivers ++ * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers + */ +- const struct drm_gpuva_fn_ops *ops; ++ const struct drm_gpuvm_ops *ops; + }; + +-void drm_gpuva_manager_init(struct drm_gpuva_manager *mgr, +- const char *name, +- u64 start_offset, u64 range, +- u64 reserve_offset, u64 reserve_range, +- const struct drm_gpuva_fn_ops *ops); +-void drm_gpuva_manager_destroy(struct drm_gpuva_manager *mgr); ++void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, ++ u64 start_offset, u64 range, ++ u64 reserve_offset, u64 reserve_range, ++ const struct drm_gpuvm_ops *ops); ++void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm); ++ ++bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + + static inline struct drm_gpuva * + __drm_gpuva_next(struct drm_gpuva *va) + { +- if (va && !list_is_last(&va->rb.entry, &va->mgr->rb.list)) ++ if (va && !list_is_last(&va->rb.entry, &va->vm->rb.list)) + return list_next_entry(va, rb.entry); + + return NULL; + } + + /** +- * drm_gpuva_for_each_va_range() - iterate over a range of &drm_gpuvas ++ * drm_gpuvm_for_each_va_range() - iterate over a range of &drm_gpuvas + * @va__: &drm_gpuva structure to assign to in each iteration step +- * @mgr__: &drm_gpuva_manager to walk over ++ * @gpuvm__: &drm_gpuvm to walk over + * @start__: starting offset, the first gpuva will overlap this + * @end__: ending offset, the last gpuva will start before this (but may + * overlap) + * +- * This iterator walks over all &drm_gpuvas in the &drm_gpuva_manager that lie ++ * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie + * between @start__ and @end__. It is implemented similarly to list_for_each(), +- * but is using the &drm_gpuva_manager's internal interval tree to accelerate ++ * but is using the &drm_gpuvm's internal interval tree to accelerate + * the search for the starting &drm_gpuva, and hence isn't safe against removal + * of elements. It assumes that @end__ is within (or is the upper limit of) the +- * &drm_gpuva_manager. This iterator does not skip over the &drm_gpuva_manager's ++ * &drm_gpuvm. This iterator does not skip over the &drm_gpuvm's + * @kernel_alloc_node. + */ +-#define drm_gpuva_for_each_va_range(va__, mgr__, start__, end__) \ +- for (va__ = drm_gpuva_find_first((mgr__), (start__), (end__) - (start__)); \ ++#define drm_gpuvm_for_each_va_range(va__, gpuvm__, start__, end__) \ ++ for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)); \ + va__ && (va__->va.addr < (end__)); \ + va__ = __drm_gpuva_next(va__)) + + /** +- * drm_gpuva_for_each_va_range_safe() - safely iterate over a range of ++ * drm_gpuvm_for_each_va_range_safe() - safely iterate over a range of + * &drm_gpuvas + * @va__: &drm_gpuva to assign to in each iteration step + * @next__: another &drm_gpuva to use as temporary storage +- * @mgr__: &drm_gpuva_manager to walk over ++ * @gpuvm__: &drm_gpuvm to walk over + * @start__: starting offset, the first gpuva will overlap this + * @end__: ending offset, the last gpuva will start before this (but may + * overlap) + * +- * This iterator walks over all &drm_gpuvas in the &drm_gpuva_manager that lie ++ * This iterator walks over all &drm_gpuvas in the &drm_gpuvm that lie + * between @start__ and @end__. It is implemented similarly to +- * list_for_each_safe(), but is using the &drm_gpuva_manager's internal interval ++ * list_for_each_safe(), but is using the &drm_gpuvm's internal interval + * tree to accelerate the search for the starting &drm_gpuva, and hence is safe + * against removal of elements. It assumes that @end__ is within (or is the +- * upper limit of) the &drm_gpuva_manager. This iterator does not skip over the +- * &drm_gpuva_manager's @kernel_alloc_node. ++ * upper limit of) the &drm_gpuvm. This iterator does not skip over the ++ * &drm_gpuvm's @kernel_alloc_node. + */ +-#define drm_gpuva_for_each_va_range_safe(va__, next__, mgr__, start__, end__) \ +- for (va__ = drm_gpuva_find_first((mgr__), (start__), (end__) - (start__)), \ ++#define drm_gpuvm_for_each_va_range_safe(va__, next__, gpuvm__, start__, end__) \ ++ for (va__ = drm_gpuva_find_first((gpuvm__), (start__), (end__) - (start__)), \ + next__ = __drm_gpuva_next(va__); \ + va__ && (va__->va.addr < (end__)); \ + va__ = next__, next__ = __drm_gpuva_next(va__)) + + /** +- * drm_gpuva_for_each_va() - iterate over all &drm_gpuvas ++ * drm_gpuvm_for_each_va() - iterate over all &drm_gpuvas + * @va__: &drm_gpuva to assign to in each iteration step +- * @mgr__: &drm_gpuva_manager to walk over ++ * @gpuvm__: &drm_gpuvm to walk over + * + * This iterator walks over all &drm_gpuva structures associated with the given +- * &drm_gpuva_manager. ++ * &drm_gpuvm. + */ +-#define drm_gpuva_for_each_va(va__, mgr__) \ +- list_for_each_entry(va__, &(mgr__)->rb.list, rb.entry) ++#define drm_gpuvm_for_each_va(va__, gpuvm__) \ ++ list_for_each_entry(va__, &(gpuvm__)->rb.list, rb.entry) + + /** +- * drm_gpuva_for_each_va_safe() - safely iterate over all &drm_gpuvas ++ * drm_gpuvm_for_each_va_safe() - safely iterate over all &drm_gpuvas + * @va__: &drm_gpuva to assign to in each iteration step + * @next__: another &drm_gpuva to use as temporary storage +- * @mgr__: &drm_gpuva_manager to walk over ++ * @gpuvm__: &drm_gpuvm to walk over + * + * This iterator walks over all &drm_gpuva structures associated with the given +- * &drm_gpuva_manager. It is implemented with list_for_each_entry_safe(), and ++ * &drm_gpuvm. It is implemented with list_for_each_entry_safe(), and + * hence safe against the removal of elements. + */ +-#define drm_gpuva_for_each_va_safe(va__, next__, mgr__) \ +- list_for_each_entry_safe(va__, next__, &(mgr__)->rb.list, rb.entry) ++#define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \ ++ list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry) + + /** + * enum drm_gpuva_op_type - GPU VA operation type + * +- * Operations to alter the GPU VA mappings tracked by the &drm_gpuva_manager. ++ * Operations to alter the GPU VA mappings tracked by the &drm_gpuvm. + */ + enum drm_gpuva_op_type { + /** +@@ -413,7 +412,7 @@ struct drm_gpuva_op_unmap { + * + * Optionally, if &keep is set, drivers may keep the actual page table + * mappings for this &drm_gpuva, adding the missing page table entries +- * only and update the &drm_gpuva_manager accordingly. ++ * only and update the &drm_gpuvm accordingly. + */ + bool keep; + }; +@@ -584,22 +583,22 @@ struct drm_gpuva_ops { + #define drm_gpuva_next_op(op) list_next_entry(op, entry) + + struct drm_gpuva_ops * +-drm_gpuva_sm_map_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_sm_map_ops_create(struct drm_gpuvm *gpuvm, + u64 addr, u64 range, + struct drm_gem_object *obj, u64 offset); + struct drm_gpuva_ops * +-drm_gpuva_sm_unmap_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_sm_unmap_ops_create(struct drm_gpuvm *gpuvm, + u64 addr, u64 range); + + struct drm_gpuva_ops * +-drm_gpuva_prefetch_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, + u64 addr, u64 range); + + struct drm_gpuva_ops * +-drm_gpuva_gem_unmap_ops_create(struct drm_gpuva_manager *mgr, ++drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj); + +-void drm_gpuva_ops_free(struct drm_gpuva_manager *mgr, ++void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, + struct drm_gpuva_ops *ops); + + static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, +@@ -610,15 +609,15 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, + } + + /** +- * struct drm_gpuva_fn_ops - callbacks for split/merge steps ++ * struct drm_gpuvm_ops - callbacks for split/merge steps + * +- * This structure defines the callbacks used by &drm_gpuva_sm_map and +- * &drm_gpuva_sm_unmap to provide the split/merge steps for map and unmap ++ * This structure defines the callbacks used by &drm_gpuvm_sm_map and ++ * &drm_gpuvm_sm_unmap to provide the split/merge steps for map and unmap + * operations to drivers. + */ +-struct drm_gpuva_fn_ops { ++struct drm_gpuvm_ops { + /** +- * @op_alloc: called when the &drm_gpuva_manager allocates ++ * @op_alloc: called when the &drm_gpuvm allocates + * a struct drm_gpuva_op + * + * Some drivers may want to embed struct drm_gpuva_op into driver +@@ -630,7 +629,7 @@ struct drm_gpuva_fn_ops { + struct drm_gpuva_op *(*op_alloc)(void); + + /** +- * @op_free: called when the &drm_gpuva_manager frees a ++ * @op_free: called when the &drm_gpuvm frees a + * struct drm_gpuva_op + * + * Some drivers may want to embed struct drm_gpuva_op into driver +@@ -642,19 +641,19 @@ struct drm_gpuva_fn_ops { + void (*op_free)(struct drm_gpuva_op *op); + + /** +- * @sm_step_map: called from &drm_gpuva_sm_map to finally insert the ++ * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the + * mapping once all previous steps were completed + * + * The &priv pointer matches the one the driver passed to +- * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. + * +- * Can be NULL if &drm_gpuva_sm_map is used. ++ * Can be NULL if &drm_gpuvm_sm_map is used. + */ + int (*sm_step_map)(struct drm_gpuva_op *op, void *priv); + + /** +- * @sm_step_remap: called from &drm_gpuva_sm_map and +- * &drm_gpuva_sm_unmap to split up an existent mapping ++ * @sm_step_remap: called from &drm_gpuvm_sm_map and ++ * &drm_gpuvm_sm_unmap to split up an existent mapping + * + * This callback is called when existent mapping needs to be split up. + * This is the case when either a newly requested mapping overlaps or +@@ -662,38 +661,38 @@ struct drm_gpuva_fn_ops { + * mapping is requested. + * + * The &priv pointer matches the one the driver passed to +- * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. + * +- * Can be NULL if neither &drm_gpuva_sm_map nor &drm_gpuva_sm_unmap is ++ * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is + * used. + */ + int (*sm_step_remap)(struct drm_gpuva_op *op, void *priv); + + /** +- * @sm_step_unmap: called from &drm_gpuva_sm_map and +- * &drm_gpuva_sm_unmap to unmap an existent mapping ++ * @sm_step_unmap: called from &drm_gpuvm_sm_map and ++ * &drm_gpuvm_sm_unmap to unmap an existent mapping + * + * This callback is called when existent mapping needs to be unmapped. + * This is the case when either a newly requested mapping encloses an + * existent mapping or an unmap of an existent mapping is requested. + * + * The &priv pointer matches the one the driver passed to +- * &drm_gpuva_sm_map or &drm_gpuva_sm_unmap, respectively. ++ * &drm_gpuvm_sm_map or &drm_gpuvm_sm_unmap, respectively. + * +- * Can be NULL if neither &drm_gpuva_sm_map nor &drm_gpuva_sm_unmap is ++ * Can be NULL if neither &drm_gpuvm_sm_map nor &drm_gpuvm_sm_unmap is + * used. + */ + int (*sm_step_unmap)(struct drm_gpuva_op *op, void *priv); + }; + +-int drm_gpuva_sm_map(struct drm_gpuva_manager *mgr, void *priv, ++int drm_gpuvm_sm_map(struct drm_gpuvm *gpuvm, void *priv, + u64 addr, u64 range, + struct drm_gem_object *obj, u64 offset); + +-int drm_gpuva_sm_unmap(struct drm_gpuva_manager *mgr, void *priv, ++int drm_gpuvm_sm_unmap(struct drm_gpuvm *gpuvm, void *priv, + u64 addr, u64 range); + +-void drm_gpuva_map(struct drm_gpuva_manager *mgr, ++void drm_gpuva_map(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va, + struct drm_gpuva_op_map *op); + +@@ -703,4 +702,4 @@ void drm_gpuva_remap(struct drm_gpuva *prev, + + void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op); + +-#endif /* __DRM_GPUVA_MGR_H__ */ ++#endif /* __DRM_GPUVM_H__ */ + +From c4ebeb96eb6c7bf96e0d5380e5c54b33f07dacfe Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 20 Sep 2023 16:42:35 +0200 +Subject: [PATCH 007/150] [MOD] drm/gpuvm: allow building as module +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: +drivers/gpu/drm/nouveau/Kconfig +skipped because there is no gpuvm support of nouveau in 6.1 + +Currently, the DRM GPUVM does not have any core dependencies preventing +a module build. + +Also, new features from subsequent patches require helpers (namely +drm_exec) which can be built as module. + +Reviewed-by: Christian König +Reviewed-by: Dave Airlie +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20230920144343.64830-3-dakr@redhat.com +--- + drivers/gpu/drm/Kconfig | 7 +++++++ + drivers/gpu/drm/Makefile | 2 +- + drivers/gpu/drm/drm_gpuvm.c | 3 +++ + 3 files changed, 11 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index 0e111ebc3c12e4..ddbbe187f04195 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -213,6 +213,13 @@ config DRM_EXEC + help + Execution context for command submissions + ++config DRM_GPUVM ++ tristate ++ depends on DRM ++ help ++ GPU-VM representation providing helpers to manage a GPUs virtual ++ address space ++ + config DRM_BUDDY + tristate + depends on DRM +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index f718d2c83ed215..1f167b96acfbe2 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -12,7 +12,6 @@ drm-y := drm_aperture.o drm_auth.o drm_cache.o \ + drm_crtc.o drm_fourcc.o drm_modes.o drm_displayid.o \ + drm_trace_points.o drm_prime.o \ + drm_vma_manager.o \ +- drm_gpuvm.o \ + drm_modeset_lock.o drm_atomic.o drm_bridge.o \ + drm_framebuffer.o drm_connector.o drm_blend.o \ + drm_encoder.o drm_mode_object.o drm_property.o \ +@@ -43,6 +42,7 @@ obj-$(CONFIG_DRM_PANEL_ORIENTATION_QUIRKS) += drm_panel_orientation_quirks.o + # + # + obj-$(CONFIG_DRM_EXEC) += drm_exec.o ++obj-$(CONFIG_DRM_GPUVM) += drm_gpuvm.o + + obj-$(CONFIG_DRM_BUDDY) += drm_buddy.o + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 89dcc6c34de45e..02ce6baacdad4b 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -1721,3 +1721,6 @@ drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, + kfree(ops); + } + EXPORT_SYMBOL_GPL(drm_gpuva_ops_free); ++ ++MODULE_DESCRIPTION("DRM GPUVM"); ++MODULE_LICENSE("GPL"); + +From a4e398ed25f209bfbcdeb12eb34ed8e4c17c93d7 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:31 +0100 +Subject: [PATCH 008/150] [MOD] drm/gpuvm: convert WARN() to drm_WARN() + variants +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: +drivers/gpu/drm/nouveau/nouveau_uvmm.c +skipped since 6.1 does not support gpuvm on nv + +Use drm_WARN() and drm_WARN_ON() variants to indicate drivers the +context the failing VM resides in. + +Acked-by: Christian König +Reviewed-by: Boris Brezillon +Reviewed-by: Thomas Hellström +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-2-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 30 ++++++++++++++++-------------- + include/drm/drm_gpuvm.h | 7 +++++++ + 2 files changed, 23 insertions(+), 14 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 02ce6baacdad4b..1aee12c2c825ad 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -614,11 +614,11 @@ static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, + static void __drm_gpuva_remove(struct drm_gpuva *va); + + static bool +-drm_gpuvm_check_overflow(u64 addr, u64 range) ++drm_gpuvm_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) + { + u64 end; + +- return WARN(check_add_overflow(addr, range, &end), ++ return drm_WARN(gpuvm->drm, check_add_overflow(addr, range, &end), + "GPUVA address limited to %zu bytes.\n", sizeof(end)); + } + +@@ -647,7 +647,7 @@ static bool + drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { +- return !drm_gpuvm_check_overflow(addr, range) && ++ return !drm_gpuvm_check_overflow(gpuvm, addr, range) && + drm_gpuvm_in_mm_range(gpuvm, addr, range) && + !drm_gpuvm_in_kernel_node(gpuvm, addr, range); + } +@@ -656,6 +656,7 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + * drm_gpuvm_init() - initialize a &drm_gpuvm + * @gpuvm: pointer to the &drm_gpuvm to initialize + * @name: the name of the GPU VA space ++ * @drm: the &drm_device this VM resides in + * @start_offset: the start offset of the GPU VA space + * @range: the size of the GPU VA space + * @reserve_offset: the start of the kernel reserved GPU VA area +@@ -668,8 +669,8 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + * &name is expected to be managed by the surrounding driver structures. + */ + void +-drm_gpuvm_init(struct drm_gpuvm *gpuvm, +- const char *name, ++drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, ++ struct drm_device *drm, + u64 start_offset, u64 range, + u64 reserve_offset, u64 reserve_range, + const struct drm_gpuvm_ops *ops) +@@ -677,20 +678,20 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, + gpuvm->rb.tree = RB_ROOT_CACHED; + INIT_LIST_HEAD(&gpuvm->rb.list); + +- drm_gpuvm_check_overflow(start_offset, range); +- gpuvm->mm_start = start_offset; +- gpuvm->mm_range = range; +- + gpuvm->name = name ? name : "unknown"; + gpuvm->ops = ops; ++ gpuvm->drm = drm; + +- memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); ++ drm_gpuvm_check_overflow(gpuvm, start_offset, range); ++ gpuvm->mm_start = start_offset; ++ gpuvm->mm_range = range; + ++ memset(&gpuvm->kernel_alloc_node, 0, sizeof(struct drm_gpuva)); + if (reserve_range) { + gpuvm->kernel_alloc_node.va.addr = reserve_offset; + gpuvm->kernel_alloc_node.va.range = reserve_range; + +- if (likely(!drm_gpuvm_check_overflow(reserve_offset, ++ if (likely(!drm_gpuvm_check_overflow(gpuvm, reserve_offset, + reserve_range))) + __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); + } +@@ -712,8 +713,8 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) + if (gpuvm->kernel_alloc_node.va.range) + __drm_gpuva_remove(&gpuvm->kernel_alloc_node); + +- WARN(!RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), +- "GPUVA tree is not empty, potentially leaking memory."); ++ drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), ++ "GPUVA tree is not empty, potentially leaking memory.\n"); + } + EXPORT_SYMBOL_GPL(drm_gpuvm_destroy); + +@@ -795,7 +796,8 @@ drm_gpuva_remove(struct drm_gpuva *va) + struct drm_gpuvm *gpuvm = va->vm; + + if (unlikely(va == &gpuvm->kernel_alloc_node)) { +- WARN(1, "Can't destroy kernel reserved node.\n"); ++ drm_WARN(gpuvm->drm, 1, ++ "Can't destroy kernel reserved node.\n"); + return; + } + +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index c7ed6bf441d4f4..af8f67ba173ffb 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -29,6 +29,7 @@ + #include + #include + ++#include + #include + + struct drm_gpuvm; +@@ -201,6 +202,11 @@ struct drm_gpuvm { + */ + const char *name; + ++ /** ++ * @drm: the &drm_device this VM lives in ++ */ ++ struct drm_device *drm; ++ + /** + * @mm_start: start of the VA space + */ +@@ -241,6 +247,7 @@ struct drm_gpuvm { + }; + + void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, ++ struct drm_device *drm, + u64 start_offset, u64 range, + u64 reserve_offset, u64 reserve_range, + const struct drm_gpuvm_ops *ops); + +From 2e6ded033bde774e387255bc4936d52c9009345b Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:32 +0100 +Subject: [PATCH 009/150] drm/gpuvm: don't always WARN in + drm_gpuvm_check_overflow() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Don't always WARN in drm_gpuvm_check_overflow() and separate it into a +drm_gpuvm_check_overflow() and a dedicated +drm_gpuvm_warn_check_overflow() variant. + +This avoids printing warnings due to invalid userspace requests. + +Acked-by: Christian König +Reviewed-by: Thomas Hellström +Reviewed-by: Boris Brezillon +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-3-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 20 +++++++++++++------- + 1 file changed, 13 insertions(+), 7 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 1aee12c2c825ad..1cbeb4169f9c0e 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -614,12 +614,18 @@ static int __drm_gpuva_insert(struct drm_gpuvm *gpuvm, + static void __drm_gpuva_remove(struct drm_gpuva *va); + + static bool +-drm_gpuvm_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) ++drm_gpuvm_check_overflow(u64 addr, u64 range) + { + u64 end; + +- return drm_WARN(gpuvm->drm, check_add_overflow(addr, range, &end), +- "GPUVA address limited to %zu bytes.\n", sizeof(end)); ++ return check_add_overflow(addr, range, &end); ++} ++ ++static bool ++drm_gpuvm_warn_check_overflow(struct drm_gpuvm *gpuvm, u64 addr, u64 range) ++{ ++ return drm_WARN(gpuvm->drm, drm_gpuvm_check_overflow(addr, range), ++ "GPUVA address limited to %zu bytes.\n", sizeof(addr)); + } + + static bool +@@ -647,7 +653,7 @@ static bool + drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { +- return !drm_gpuvm_check_overflow(gpuvm, addr, range) && ++ return !drm_gpuvm_check_overflow(addr, range) && + drm_gpuvm_in_mm_range(gpuvm, addr, range) && + !drm_gpuvm_in_kernel_node(gpuvm, addr, range); + } +@@ -682,7 +688,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + gpuvm->ops = ops; + gpuvm->drm = drm; + +- drm_gpuvm_check_overflow(gpuvm, start_offset, range); ++ drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); + gpuvm->mm_start = start_offset; + gpuvm->mm_range = range; + +@@ -691,8 +697,8 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + gpuvm->kernel_alloc_node.va.addr = reserve_offset; + gpuvm->kernel_alloc_node.va.range = reserve_range; + +- if (likely(!drm_gpuvm_check_overflow(gpuvm, reserve_offset, +- reserve_range))) ++ if (likely(!drm_gpuvm_warn_check_overflow(gpuvm, reserve_offset, ++ reserve_range))) + __drm_gpuva_insert(gpuvm, &gpuvm->kernel_alloc_node); + } + } + +From 748b2e3f5c43feb0a25d44e022d111fa65c9c2a8 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:33 +0100 +Subject: [PATCH 010/150] drm/gpuvm: export drm_gpuvm_range_valid() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Drivers may use this function to validate userspace requests in advance, +hence export it. + +Acked-by: Christian König +Reviewed-by: Thomas Hellström +Reviewed-by: Boris Brezillon +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-4-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 14 +++++++++++++- + include/drm/drm_gpuvm.h | 1 + + 2 files changed, 14 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 1cbeb4169f9c0e..dd46d14fd3e605 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -649,7 +649,18 @@ drm_gpuvm_in_kernel_node(struct drm_gpuvm *gpuvm, u64 addr, u64 range) + return krange && addr < kend && kstart < end; + } + +-static bool ++/** ++ * drm_gpuvm_range_valid() - checks whether the given range is valid for the ++ * given &drm_gpuvm ++ * @gpuvm: the GPUVM to check the range for ++ * @addr: the base address ++ * @range: the range starting from the base address ++ * ++ * Checks whether the range is within the GPUVM's managed boundaries. ++ * ++ * Returns: true for a valid range, false otherwise ++ */ ++bool + drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + u64 addr, u64 range) + { +@@ -657,6 +668,7 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + drm_gpuvm_in_mm_range(gpuvm, addr, range) && + !drm_gpuvm_in_kernel_node(gpuvm, addr, range); + } ++EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid); + + /** + * drm_gpuvm_init() - initialize a &drm_gpuvm +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index af8f67ba173ffb..6d035e4157baf3 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -253,6 +253,7 @@ void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + const struct drm_gpuvm_ops *ops); + void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm); + ++bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + + static inline struct drm_gpuva * + +From 5e3b895399031594549c352e1e8571f7e6f2ac37 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:35 +0100 +Subject: [PATCH 011/150] [MOD] drm/gpuvm: add common dma-resv per struct + drm_gpuvm +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +hb: +drivers/gpu/drm/nouveau/nouveau_uvmm.c +skipped + +Provide a common dma-resv for GEM objects not being used outside of this +GPU-VM. This is used in a subsequent patch to generalize dma-resv, +external and evicted object handling and GEM validation. + +Acked-by: Christian König +Reviewed-by: Boris Brezillon +Reviewed-by: Thomas Hellström +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-6-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 53 +++++++++++++++++++++++++++++++++++++ + include/drm/drm_gpuvm.h | 33 +++++++++++++++++++++++ + 2 files changed, 86 insertions(+) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index dd46d14fd3e605..594ebcab5a4b4b 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -61,6 +61,15 @@ + * contained within struct drm_gpuva already. Hence, for inserting &drm_gpuva + * entries from within dma-fence signalling critical sections it is enough to + * pre-allocate the &drm_gpuva structures. ++ * ++ * &drm_gem_objects which are private to a single VM can share a common ++ * &dma_resv in order to improve locking efficiency (e.g. with &drm_exec). ++ * For this purpose drivers must pass a &drm_gem_object to drm_gpuvm_init(), in ++ * the following called 'resv object', which serves as the container of the ++ * GPUVM's shared &dma_resv. This resv object can be a driver specific ++ * &drm_gem_object, such as the &drm_gem_object containing the root page table, ++ * but it can also be a 'dummy' object, which can be allocated with ++ * drm_gpuvm_resv_object_alloc(). + */ + + /** +@@ -670,11 +679,49 @@ drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, + } + EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid); + ++static void ++drm_gpuvm_gem_object_free(struct drm_gem_object *obj) ++{ ++ drm_gem_object_release(obj); ++ kfree(obj); ++} ++ ++static const struct drm_gem_object_funcs drm_gpuvm_object_funcs = { ++ .free = drm_gpuvm_gem_object_free, ++}; ++ ++/** ++ * drm_gpuvm_resv_object_alloc() - allocate a dummy &drm_gem_object ++ * @drm: the drivers &drm_device ++ * ++ * Allocates a dummy &drm_gem_object which can be passed to drm_gpuvm_init() in ++ * order to serve as root GEM object providing the &drm_resv shared across ++ * &drm_gem_objects local to a single GPUVM. ++ * ++ * Returns: the &drm_gem_object on success, NULL on failure ++ */ ++struct drm_gem_object * ++drm_gpuvm_resv_object_alloc(struct drm_device *drm) ++{ ++ struct drm_gem_object *obj; ++ ++ obj = kzalloc(sizeof(*obj), GFP_KERNEL); ++ if (!obj) ++ return NULL; ++ ++ obj->funcs = &drm_gpuvm_object_funcs; ++ drm_gem_private_object_init(drm, obj, 0); ++ ++ return obj; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc); ++ + /** + * drm_gpuvm_init() - initialize a &drm_gpuvm + * @gpuvm: pointer to the &drm_gpuvm to initialize + * @name: the name of the GPU VA space + * @drm: the &drm_device this VM resides in ++ * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv + * @start_offset: the start offset of the GPU VA space + * @range: the size of the GPU VA space + * @reserve_offset: the start of the kernel reserved GPU VA area +@@ -689,6 +736,7 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_range_valid); + void + drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + struct drm_device *drm, ++ struct drm_gem_object *r_obj, + u64 start_offset, u64 range, + u64 reserve_offset, u64 reserve_range, + const struct drm_gpuvm_ops *ops) +@@ -699,6 +747,9 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + gpuvm->name = name ? name : "unknown"; + gpuvm->ops = ops; + gpuvm->drm = drm; ++ gpuvm->r_obj = r_obj; ++ ++ drm_gem_object_get(r_obj); + + drm_gpuvm_warn_check_overflow(gpuvm, start_offset, range); + gpuvm->mm_start = start_offset; +@@ -733,6 +784,8 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) + + drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), + "GPUVA tree is not empty, potentially leaking memory.\n"); ++ ++ drm_gem_object_put(gpuvm->r_obj); + } + EXPORT_SYMBOL_GPL(drm_gpuvm_destroy); + +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 6d035e4157baf3..efa6f1f4643510 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -244,10 +244,16 @@ struct drm_gpuvm { + * @ops: &drm_gpuvm_ops providing the split/merge steps to drivers + */ + const struct drm_gpuvm_ops *ops; ++ ++ /** ++ * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv. ++ */ ++ struct drm_gem_object *r_obj; + }; + + void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + struct drm_device *drm, ++ struct drm_gem_object *r_obj, + u64 start_offset, u64 range, + u64 reserve_offset, u64 reserve_range, + const struct drm_gpuvm_ops *ops); +@@ -256,6 +262,33 @@ void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm); + bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + ++struct drm_gem_object * ++drm_gpuvm_resv_object_alloc(struct drm_device *drm); ++ ++/** ++ * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv ++ * @gpuvm__: the &drm_gpuvm ++ * ++ * Returns: a pointer to the &drm_gpuvm's shared &dma_resv ++ */ ++#define drm_gpuvm_resv(gpuvm__) ((gpuvm__)->r_obj->resv) ++ ++/** ++ * drm_gpuvm_resv_obj() - returns the &drm_gem_object holding the &drm_gpuvm's ++ * &dma_resv ++ * @gpuvm__: the &drm_gpuvm ++ * ++ * Returns: a pointer to the &drm_gem_object holding the &drm_gpuvm's shared ++ * &dma_resv ++ */ ++#define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj) ++ ++#define drm_gpuvm_resv_held(gpuvm__) \ ++ dma_resv_held(drm_gpuvm_resv(gpuvm__)) ++ ++#define drm_gpuvm_resv_assert_held(gpuvm__) \ ++ dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) ++ + static inline struct drm_gpuva * + __drm_gpuva_next(struct drm_gpuva *va) + { + +From 03d15b6cf2101178a89b6961882b63bfc4714d06 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:37 +0100 +Subject: [PATCH 012/150] [MOD] drm/gpuvm: add drm_gpuvm_flags to drm_gpuvm +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: +drivers/gpu/drm/nouveau/nouveau_uvmm.c +skipped + +Introduce flags for struct drm_gpuvm, this required by subsequent +commits. + +Acked-by: Christian König +Reviewed-by: Boris Brezillon +Reviewed-by: Thomas Hellström +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-8-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 3 +++ + include/drm/drm_gpuvm.h | 16 ++++++++++++++++ + 2 files changed, 19 insertions(+) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 594ebcab5a4b4b..0a010b52e33dd0 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -720,6 +720,7 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc); + * drm_gpuvm_init() - initialize a &drm_gpuvm + * @gpuvm: pointer to the &drm_gpuvm to initialize + * @name: the name of the GPU VA space ++ * @flags: the &drm_gpuvm_flags for this GPUVM + * @drm: the &drm_device this VM resides in + * @r_obj: the resv &drm_gem_object providing the GPUVM's common &dma_resv + * @start_offset: the start offset of the GPU VA space +@@ -735,6 +736,7 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_resv_object_alloc); + */ + void + drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, ++ enum drm_gpuvm_flags flags, + struct drm_device *drm, + struct drm_gem_object *r_obj, + u64 start_offset, u64 range, +@@ -745,6 +747,7 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + INIT_LIST_HEAD(&gpuvm->rb.list); + + gpuvm->name = name ? name : "unknown"; ++ gpuvm->flags = flags; + gpuvm->ops = ops; + gpuvm->drm = drm; + gpuvm->r_obj = r_obj; +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index efa6f1f4643510..924c1225d19c2f 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -184,6 +184,16 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) + return va->flags & DRM_GPUVA_INVALIDATED; + } + ++/** ++ * enum drm_gpuvm_flags - flags for struct drm_gpuvm ++ */ ++enum drm_gpuvm_flags { ++ /** ++ * @DRM_GPUVM_USERBITS: user defined bits ++ */ ++ DRM_GPUVM_USERBITS = BIT(0), ++}; ++ + /** + * struct drm_gpuvm - DRM GPU VA Manager + * +@@ -202,6 +212,11 @@ struct drm_gpuvm { + */ + const char *name; + ++ /** ++ * @flags: the &drm_gpuvm_flags of this GPUVM ++ */ ++ enum drm_gpuvm_flags flags; ++ + /** + * @drm: the &drm_device this VM lives in + */ +@@ -252,6 +267,7 @@ struct drm_gpuvm { + }; + + void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, ++ enum drm_gpuvm_flags flags, + struct drm_device *drm, + struct drm_gem_object *r_obj, + u64 start_offset, u64 range, + +From 29625740149aa301a9d4793116bf11e39c3dbfba Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:39 +0100 +Subject: [PATCH 013/150] [MOD] drm/gpuvm: reference count drm_gpuvm structures +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: +drivers/gpu/drm/nouveau/nouveau_uvmm.c +skipped + +Implement reference counting for struct drm_gpuvm. + +Acked-by: Christian König +Reviewed-by: Thomas Hellström +Reviewed-by: Boris Brezillon +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-10-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 56 +++++++++++++++++++++++++++++-------- + include/drm/drm_gpuvm.h | 31 +++++++++++++++++++- + 2 files changed, 75 insertions(+), 12 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 0a010b52e33dd0..10bd15822b9b1b 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -746,6 +746,8 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + gpuvm->rb.tree = RB_ROOT_CACHED; + INIT_LIST_HEAD(&gpuvm->rb.list); + ++ kref_init(&gpuvm->kref); ++ + gpuvm->name = name ? name : "unknown"; + gpuvm->flags = flags; + gpuvm->ops = ops; +@@ -770,15 +772,8 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + } + EXPORT_SYMBOL_GPL(drm_gpuvm_init); + +-/** +- * drm_gpuvm_destroy() - cleanup a &drm_gpuvm +- * @gpuvm: pointer to the &drm_gpuvm to clean up +- * +- * Note that it is a bug to call this function on a manager that still +- * holds GPU VA mappings. +- */ +-void +-drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) ++static void ++drm_gpuvm_fini(struct drm_gpuvm *gpuvm) + { + gpuvm->name = NULL; + +@@ -790,7 +785,35 @@ drm_gpuvm_destroy(struct drm_gpuvm *gpuvm) + + drm_gem_object_put(gpuvm->r_obj); + } +-EXPORT_SYMBOL_GPL(drm_gpuvm_destroy); ++ ++static void ++drm_gpuvm_free(struct kref *kref) ++{ ++ struct drm_gpuvm *gpuvm = container_of(kref, struct drm_gpuvm, kref); ++ ++ drm_gpuvm_fini(gpuvm); ++ ++ if (drm_WARN_ON(gpuvm->drm, !gpuvm->ops->vm_free)) ++ return; ++ ++ gpuvm->ops->vm_free(gpuvm); ++} ++ ++/** ++ * drm_gpuvm_put() - drop a struct drm_gpuvm reference ++ * @gpuvm: the &drm_gpuvm to release the reference of ++ * ++ * This releases a reference to @gpuvm. ++ * ++ * This function may be called from atomic context. ++ */ ++void ++drm_gpuvm_put(struct drm_gpuvm *gpuvm) ++{ ++ if (gpuvm) ++ kref_put(&gpuvm->kref, drm_gpuvm_free); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_put); + + static int + __drm_gpuva_insert(struct drm_gpuvm *gpuvm, +@@ -839,11 +862,21 @@ drm_gpuva_insert(struct drm_gpuvm *gpuvm, + { + u64 addr = va->va.addr; + u64 range = va->va.range; ++ int ret; + + if (unlikely(!drm_gpuvm_range_valid(gpuvm, addr, range))) + return -EINVAL; + +- return __drm_gpuva_insert(gpuvm, va); ++ ret = __drm_gpuva_insert(gpuvm, va); ++ if (likely(!ret)) ++ /* Take a reference of the GPUVM for the successfully inserted ++ * drm_gpuva. We can't take the reference in ++ * __drm_gpuva_insert() itself, since we don't want to increse ++ * the reference count for the GPUVM's kernel_alloc_node. ++ */ ++ drm_gpuvm_get(gpuvm); ++ ++ return ret; + } + EXPORT_SYMBOL_GPL(drm_gpuva_insert); + +@@ -876,6 +909,7 @@ drm_gpuva_remove(struct drm_gpuva *va) + } + + __drm_gpuva_remove(va); ++ drm_gpuvm_put(va->vm); + } + EXPORT_SYMBOL_GPL(drm_gpuva_remove); + +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 924c1225d19c2f..8bfa30b9df14d2 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -247,6 +247,11 @@ struct drm_gpuvm { + struct list_head list; + } rb; + ++ /** ++ * @kref: reference count of this object ++ */ ++ struct kref kref; ++ + /** + * @kernel_alloc_node: + * +@@ -273,7 +278,23 @@ void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + u64 start_offset, u64 range, + u64 reserve_offset, u64 reserve_range, + const struct drm_gpuvm_ops *ops); +-void drm_gpuvm_destroy(struct drm_gpuvm *gpuvm); ++ ++/** ++ * drm_gpuvm_get() - acquire a struct drm_gpuvm reference ++ * @gpuvm: the &drm_gpuvm to acquire the reference of ++ * ++ * This function acquires an additional reference to @gpuvm. It is illegal to ++ * call this without already holding a reference. No locks required. ++ */ ++static inline struct drm_gpuvm * ++drm_gpuvm_get(struct drm_gpuvm *gpuvm) ++{ ++ kref_get(&gpuvm->kref); ++ ++ return gpuvm; ++} ++ ++void drm_gpuvm_put(struct drm_gpuvm *gpuvm); + + bool drm_gpuvm_range_valid(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); +@@ -673,6 +694,14 @@ static inline void drm_gpuva_init_from_op(struct drm_gpuva *va, + * operations to drivers. + */ + struct drm_gpuvm_ops { ++ /** ++ * @vm_free: called when the last reference of a struct drm_gpuvm is ++ * dropped ++ * ++ * This callback is mandatory. ++ */ ++ void (*vm_free)(struct drm_gpuvm *gpuvm); ++ + /** + * @op_alloc: called when the &drm_gpuvm allocates + * a struct drm_gpuva_op + +From 3848e3a7d5208cd1bc4bf5a7860b3ab8df2fd5d6 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:40 +0100 +Subject: [PATCH 014/150] [MOD] drm/gpuvm: add an abstraction for a VM / BO + combination +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: +drivers/gpu/drm/nouveau/nouveau_uvmm.c +skipped + +Add an abstraction layer between the drm_gpuva mappings of a particular +drm_gem_object and this GEM object itself. The abstraction represents a +combination of a drm_gem_object and drm_gpuvm. The drm_gem_object holds +a list of drm_gpuvm_bo structures (the structure representing this +abstraction), while each drm_gpuvm_bo contains list of mappings of this +GEM object. + +This has multiple advantages: + +1) We can use the drm_gpuvm_bo structure to attach it to various lists + of the drm_gpuvm. This is useful for tracking external and evicted + objects per VM, which is introduced in subsequent patches. + +2) Finding mappings of a certain drm_gem_object mapped in a certain + drm_gpuvm becomes much cheaper. + +3) Drivers can derive and extend the structure to easily represent + driver specific states of a BO for a certain GPUVM. + +The idea of this abstraction was taken from amdgpu, hence the credit for +this idea goes to the developers of amdgpu. + +Cc: Christian König +Acked-by: Christian König +Reviewed-by: Thomas Hellström +Reviewed-by: Boris Brezillon +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-11-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 340 +++++++++++++++++++++++++++++++----- + include/drm/drm_gem.h | 32 ++-- + include/drm/drm_gpuvm.h | 185 +++++++++++++++++++- + 3 files changed, 489 insertions(+), 68 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 10bd15822b9b1b..02e7cfcb80631d 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -70,6 +70,18 @@ + * &drm_gem_object, such as the &drm_gem_object containing the root page table, + * but it can also be a 'dummy' object, which can be allocated with + * drm_gpuvm_resv_object_alloc(). ++ * ++ * In order to connect a struct drm_gpuva its backing &drm_gem_object each ++ * &drm_gem_object maintains a list of &drm_gpuvm_bo structures, and each ++ * &drm_gpuvm_bo contains a list of &drm_gpuva structures. ++ * ++ * A &drm_gpuvm_bo is an abstraction that represents a combination of a ++ * &drm_gpuvm and a &drm_gem_object. Every such combination should be unique. ++ * This is ensured by the API through drm_gpuvm_bo_obtain() and ++ * drm_gpuvm_bo_obtain_prealloc() which first look into the corresponding ++ * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this ++ * particular combination. If not existent a new instance is created and linked ++ * to the &drm_gem_object. + */ + + /** +@@ -395,21 +407,28 @@ + /** + * DOC: Locking + * +- * Generally, the GPU VA manager does not take care of locking itself, it is +- * the drivers responsibility to take care about locking. Drivers might want to +- * protect the following operations: inserting, removing and iterating +- * &drm_gpuva objects as well as generating all kinds of operations, such as +- * split / merge or prefetch. +- * +- * The GPU VA manager also does not take care of the locking of the backing +- * &drm_gem_object buffers GPU VA lists by itself; drivers are responsible to +- * enforce mutual exclusion using either the GEMs dma_resv lock or alternatively +- * a driver specific external lock. For the latter see also +- * drm_gem_gpuva_set_lock(). +- * +- * However, the GPU VA manager contains lockdep checks to ensure callers of its +- * API hold the corresponding lock whenever the &drm_gem_objects GPU VA list is +- * accessed by functions such as drm_gpuva_link() or drm_gpuva_unlink(). ++ * In terms of managing &drm_gpuva entries DRM GPUVM does not take care of ++ * locking itself, it is the drivers responsibility to take care about locking. ++ * Drivers might want to protect the following operations: inserting, removing ++ * and iterating &drm_gpuva objects as well as generating all kinds of ++ * operations, such as split / merge or prefetch. ++ * ++ * DRM GPUVM also does not take care of the locking of the backing ++ * &drm_gem_object buffers GPU VA lists and &drm_gpuvm_bo abstractions by ++ * itself; drivers are responsible to enforce mutual exclusion using either the ++ * GEMs dma_resv lock or alternatively a driver specific external lock. For the ++ * latter see also drm_gem_gpuva_set_lock(). ++ * ++ * However, DRM GPUVM contains lockdep checks to ensure callers of its API hold ++ * the corresponding lock whenever the &drm_gem_objects GPU VA list is accessed ++ * by functions such as drm_gpuva_link() or drm_gpuva_unlink(), but also ++ * drm_gpuvm_bo_obtain() and drm_gpuvm_bo_put(). ++ * ++ * The latter is required since on creation and destruction of a &drm_gpuvm_bo ++ * the &drm_gpuvm_bo is attached / removed from the &drm_gem_objects gpuva list. ++ * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and ++ * &drm_gem_object must be able to observe previous creations and destructions ++ * of &drm_gpuvm_bos in order to keep instances unique. + */ + + /** +@@ -439,6 +458,7 @@ + * { + * struct drm_gpuva_ops *ops; + * struct drm_gpuva_op *op ++ * struct drm_gpuvm_bo *vm_bo; + * + * driver_lock_va_space(); + * ops = drm_gpuvm_sm_map_ops_create(gpuvm, addr, range, +@@ -446,6 +466,10 @@ + * if (IS_ERR(ops)) + * return PTR_ERR(ops); + * ++ * vm_bo = drm_gpuvm_bo_obtain(gpuvm, obj); ++ * if (IS_ERR(vm_bo)) ++ * return PTR_ERR(vm_bo); ++ * + * drm_gpuva_for_each_op(op, ops) { + * struct drm_gpuva *va; + * +@@ -458,7 +482,7 @@ + * + * driver_vm_map(); + * drm_gpuva_map(gpuvm, va, &op->map); +- * drm_gpuva_link(va); ++ * drm_gpuva_link(va, vm_bo); + * + * break; + * case DRM_GPUVA_OP_REMAP: { +@@ -485,11 +509,11 @@ + * driver_vm_remap(); + * drm_gpuva_remap(prev, next, &op->remap); + * +- * drm_gpuva_unlink(va); + * if (prev) +- * drm_gpuva_link(prev); ++ * drm_gpuva_link(prev, va->vm_bo); + * if (next) +- * drm_gpuva_link(next); ++ * drm_gpuva_link(next, va->vm_bo); ++ * drm_gpuva_unlink(va); + * + * break; + * } +@@ -505,6 +529,7 @@ + * break; + * } + * } ++ * drm_gpuvm_bo_put(vm_bo); + * driver_unlock_va_space(); + * + * return 0; +@@ -514,6 +539,7 @@ + * + * struct driver_context { + * struct drm_gpuvm *gpuvm; ++ * struct drm_gpuvm_bo *vm_bo; + * struct drm_gpuva *new_va; + * struct drm_gpuva *prev_va; + * struct drm_gpuva *next_va; +@@ -534,6 +560,7 @@ + * struct drm_gem_object *obj, u64 offset) + * { + * struct driver_context ctx; ++ * struct drm_gpuvm_bo *vm_bo; + * struct drm_gpuva_ops *ops; + * struct drm_gpuva_op *op; + * int ret = 0; +@@ -543,16 +570,23 @@ + * ctx.new_va = kzalloc(sizeof(*ctx.new_va), GFP_KERNEL); + * ctx.prev_va = kzalloc(sizeof(*ctx.prev_va), GFP_KERNEL); + * ctx.next_va = kzalloc(sizeof(*ctx.next_va), GFP_KERNEL); +- * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va) { ++ * ctx.vm_bo = drm_gpuvm_bo_create(gpuvm, obj); ++ * if (!ctx.new_va || !ctx.prev_va || !ctx.next_va || !vm_bo) { + * ret = -ENOMEM; + * goto out; + * } + * ++ * // Typically protected with a driver specific GEM gpuva lock ++ * // used in the fence signaling path for drm_gpuva_link() and ++ * // drm_gpuva_unlink(), hence pre-allocate. ++ * ctx.vm_bo = drm_gpuvm_bo_obtain_prealloc(ctx.vm_bo); ++ * + * driver_lock_va_space(); + * ret = drm_gpuvm_sm_map(gpuvm, &ctx, addr, range, obj, offset); + * driver_unlock_va_space(); + * + * out: ++ * drm_gpuvm_bo_put(ctx.vm_bo); + * kfree(ctx.new_va); + * kfree(ctx.prev_va); + * kfree(ctx.next_va); +@@ -565,7 +599,7 @@ + * + * drm_gpuva_map(ctx->vm, ctx->new_va, &op->map); + * +- * drm_gpuva_link(ctx->new_va); ++ * drm_gpuva_link(ctx->new_va, ctx->vm_bo); + * + * // prevent the new GPUVA from being freed in + * // driver_mapping_create() +@@ -577,22 +611,23 @@ + * int driver_gpuva_remap(struct drm_gpuva_op *op, void *__ctx) + * { + * struct driver_context *ctx = __ctx; ++ * struct drm_gpuva *va = op->remap.unmap->va; + * + * drm_gpuva_remap(ctx->prev_va, ctx->next_va, &op->remap); + * +- * drm_gpuva_unlink(op->remap.unmap->va); +- * kfree(op->remap.unmap->va); +- * + * if (op->remap.prev) { +- * drm_gpuva_link(ctx->prev_va); ++ * drm_gpuva_link(ctx->prev_va, va->vm_bo); + * ctx->prev_va = NULL; + * } + * + * if (op->remap.next) { +- * drm_gpuva_link(ctx->next_va); ++ * drm_gpuva_link(ctx->next_va, va->vm_bo); + * ctx->next_va = NULL; + * } + * ++ * drm_gpuva_unlink(va); ++ * kfree(va); ++ * + * return 0; + * } + * +@@ -815,6 +850,199 @@ drm_gpuvm_put(struct drm_gpuvm *gpuvm) + } + EXPORT_SYMBOL_GPL(drm_gpuvm_put); + ++/** ++ * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo ++ * @gpuvm: The &drm_gpuvm the @obj is mapped in. ++ * @obj: The &drm_gem_object being mapped in the @gpuvm. ++ * ++ * If provided by the driver, this function uses the &drm_gpuvm_ops ++ * vm_bo_alloc() callback to allocate. ++ * ++ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure ++ */ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj) ++{ ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; ++ struct drm_gpuvm_bo *vm_bo; ++ ++ if (ops && ops->vm_bo_alloc) ++ vm_bo = ops->vm_bo_alloc(); ++ else ++ vm_bo = kzalloc(sizeof(*vm_bo), GFP_KERNEL); ++ ++ if (unlikely(!vm_bo)) ++ return NULL; ++ ++ vm_bo->vm = drm_gpuvm_get(gpuvm); ++ vm_bo->obj = obj; ++ drm_gem_object_get(obj); ++ ++ kref_init(&vm_bo->kref); ++ INIT_LIST_HEAD(&vm_bo->list.gpuva); ++ INIT_LIST_HEAD(&vm_bo->list.entry.gem); ++ ++ return vm_bo; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create); ++ ++static void ++drm_gpuvm_bo_destroy(struct kref *kref) ++{ ++ struct drm_gpuvm_bo *vm_bo = container_of(kref, struct drm_gpuvm_bo, ++ kref); ++ struct drm_gpuvm *gpuvm = vm_bo->vm; ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; ++ struct drm_gem_object *obj = vm_bo->obj; ++ bool lock = !drm_gpuvm_resv_protected(gpuvm); ++ ++ if (!lock) ++ drm_gpuvm_resv_assert_held(gpuvm); ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ list_del(&vm_bo->list.entry.gem); ++ ++ if (ops && ops->vm_bo_free) ++ ops->vm_bo_free(vm_bo); ++ else ++ kfree(vm_bo); ++ ++ drm_gpuvm_put(gpuvm); ++ drm_gem_object_put(obj); ++} ++ ++/** ++ * drm_gpuvm_bo_put() - drop a struct drm_gpuvm_bo reference ++ * @vm_bo: the &drm_gpuvm_bo to release the reference of ++ * ++ * This releases a reference to @vm_bo. ++ * ++ * If the reference count drops to zero, the &gpuvm_bo is destroyed, which ++ * includes removing it from the GEMs gpuva list. Hence, if a call to this ++ * function can potentially let the reference count drop to zero the caller must ++ * hold the dma-resv or driver specific GEM gpuva lock. ++ * ++ * This function may only be called from non-atomic context. ++ */ ++void ++drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo) ++{ ++ might_sleep(); ++ ++ if (vm_bo) ++ kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put); ++ ++static struct drm_gpuvm_bo * ++__drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj) ++{ ++ struct drm_gpuvm_bo *vm_bo; ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ drm_gem_for_each_gpuvm_bo(vm_bo, obj) ++ if (vm_bo->vm == gpuvm) ++ return vm_bo; ++ ++ return NULL; ++} ++ ++/** ++ * drm_gpuvm_bo_find() - find the &drm_gpuvm_bo for the given ++ * &drm_gpuvm and &drm_gem_object ++ * @gpuvm: The &drm_gpuvm the @obj is mapped in. ++ * @obj: The &drm_gem_object being mapped in the @gpuvm. ++ * ++ * Find the &drm_gpuvm_bo representing the combination of the given ++ * &drm_gpuvm and &drm_gem_object. If found, increases the reference ++ * count of the &drm_gpuvm_bo accordingly. ++ * ++ * Returns: a pointer to the &drm_gpuvm_bo on success, NULL on failure ++ */ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj) ++{ ++ struct drm_gpuvm_bo *vm_bo = __drm_gpuvm_bo_find(gpuvm, obj); ++ ++ return vm_bo ? drm_gpuvm_bo_get(vm_bo) : NULL; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_find); ++ ++/** ++ * drm_gpuvm_bo_obtain() - obtains and instance of the &drm_gpuvm_bo for the ++ * given &drm_gpuvm and &drm_gem_object ++ * @gpuvm: The &drm_gpuvm the @obj is mapped in. ++ * @obj: The &drm_gem_object being mapped in the @gpuvm. ++ * ++ * Find the &drm_gpuvm_bo representing the combination of the given ++ * &drm_gpuvm and &drm_gem_object. If found, increases the reference ++ * count of the &drm_gpuvm_bo accordingly. If not found, allocates a new ++ * &drm_gpuvm_bo. ++ * ++ * A new &drm_gpuvm_bo is added to the GEMs gpuva list. ++ * ++ * Returns: a pointer to the &drm_gpuvm_bo on success, an ERR_PTR on failure ++ */ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj) ++{ ++ struct drm_gpuvm_bo *vm_bo; ++ ++ vm_bo = drm_gpuvm_bo_find(gpuvm, obj); ++ if (vm_bo) ++ return vm_bo; ++ ++ vm_bo = drm_gpuvm_bo_create(gpuvm, obj); ++ if (!vm_bo) ++ return ERR_PTR(-ENOMEM); ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ list_add_tail(&vm_bo->list.entry.gem, &obj->gpuva.list); ++ ++ return vm_bo; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain); ++ ++/** ++ * drm_gpuvm_bo_obtain_prealloc() - obtains and instance of the &drm_gpuvm_bo ++ * for the given &drm_gpuvm and &drm_gem_object ++ * @__vm_bo: A pre-allocated struct drm_gpuvm_bo. ++ * ++ * Find the &drm_gpuvm_bo representing the combination of the given ++ * &drm_gpuvm and &drm_gem_object. If found, increases the reference ++ * count of the found &drm_gpuvm_bo accordingly, while the @__vm_bo reference ++ * count is decreased. If not found @__vm_bo is returned without further ++ * increase of the reference count. ++ * ++ * A new &drm_gpuvm_bo is added to the GEMs gpuva list. ++ * ++ * Returns: a pointer to the found &drm_gpuvm_bo or @__vm_bo if no existing ++ * &drm_gpuvm_bo was found ++ */ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo) ++{ ++ struct drm_gpuvm *gpuvm = __vm_bo->vm; ++ struct drm_gem_object *obj = __vm_bo->obj; ++ struct drm_gpuvm_bo *vm_bo; ++ ++ vm_bo = drm_gpuvm_bo_find(gpuvm, obj); ++ if (vm_bo) { ++ drm_gpuvm_bo_put(__vm_bo); ++ return vm_bo; ++ } ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ list_add_tail(&__vm_bo->list.entry.gem, &obj->gpuva.list); ++ ++ return __vm_bo; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc); ++ + static int + __drm_gpuva_insert(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va) +@@ -916,24 +1144,33 @@ EXPORT_SYMBOL_GPL(drm_gpuva_remove); + /** + * drm_gpuva_link() - link a &drm_gpuva + * @va: the &drm_gpuva to link ++ * @vm_bo: the &drm_gpuvm_bo to add the &drm_gpuva to + * +- * This adds the given &va to the GPU VA list of the &drm_gem_object it is +- * associated with. ++ * This adds the given &va to the GPU VA list of the &drm_gpuvm_bo and the ++ * &drm_gpuvm_bo to the &drm_gem_object it is associated with. ++ * ++ * For every &drm_gpuva entry added to the &drm_gpuvm_bo an additional ++ * reference of the latter is taken. + * + * This function expects the caller to protect the GEM's GPUVA list against +- * concurrent access using the GEMs dma_resv lock. ++ * concurrent access using either the GEMs dma_resv lock or a driver specific ++ * lock set through drm_gem_gpuva_set_lock(). + */ + void +-drm_gpuva_link(struct drm_gpuva *va) ++drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo) + { + struct drm_gem_object *obj = va->gem.obj; ++ struct drm_gpuvm *gpuvm = va->vm; + + if (unlikely(!obj)) + return; + +- drm_gem_gpuva_assert_lock_held(obj); ++ drm_WARN_ON(gpuvm->drm, obj != vm_bo->obj); + +- list_add_tail(&va->gem.entry, &obj->gpuva.list); ++ va->vm_bo = drm_gpuvm_bo_get(vm_bo); ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ list_add_tail(&va->gem.entry, &vm_bo->list.gpuva); + } + EXPORT_SYMBOL_GPL(drm_gpuva_link); + +@@ -944,20 +1181,31 @@ EXPORT_SYMBOL_GPL(drm_gpuva_link); + * This removes the given &va from the GPU VA list of the &drm_gem_object it is + * associated with. + * ++ * This removes the given &va from the GPU VA list of the &drm_gpuvm_bo and ++ * the &drm_gpuvm_bo from the &drm_gem_object it is associated with in case ++ * this call unlinks the last &drm_gpuva from the &drm_gpuvm_bo. ++ * ++ * For every &drm_gpuva entry removed from the &drm_gpuvm_bo a reference of ++ * the latter is dropped. ++ * + * This function expects the caller to protect the GEM's GPUVA list against +- * concurrent access using the GEMs dma_resv lock. ++ * concurrent access using either the GEMs dma_resv lock or a driver specific ++ * lock set through drm_gem_gpuva_set_lock(). + */ + void + drm_gpuva_unlink(struct drm_gpuva *va) + { + struct drm_gem_object *obj = va->gem.obj; ++ struct drm_gpuvm_bo *vm_bo = va->vm_bo; + + if (unlikely(!obj)) + return; + + drm_gem_gpuva_assert_lock_held(obj); +- + list_del_init(&va->gem.entry); ++ ++ va->vm_bo = NULL; ++ drm_gpuvm_bo_put(vm_bo); + } + EXPORT_SYMBOL_GPL(drm_gpuva_unlink); + +@@ -1102,10 +1350,10 @@ drm_gpuva_remap(struct drm_gpuva *prev, + struct drm_gpuva *next, + struct drm_gpuva_op_remap *op) + { +- struct drm_gpuva *curr = op->unmap->va; +- struct drm_gpuvm *gpuvm = curr->vm; ++ struct drm_gpuva *va = op->unmap->va; ++ struct drm_gpuvm *gpuvm = va->vm; + +- drm_gpuva_remove(curr); ++ drm_gpuva_remove(va); + + if (op->prev) { + drm_gpuva_init_from_op(prev, op->prev); +@@ -1747,9 +1995,8 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, + EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); + + /** +- * drm_gpuvm_gem_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM +- * @gpuvm: the &drm_gpuvm representing the GPU VA space +- * @obj: the &drm_gem_object to unmap ++ * drm_gpuvm_bo_unmap_ops_create() - creates the &drm_gpuva_ops to unmap a GEM ++ * @vm_bo: the &drm_gpuvm_bo abstraction + * + * This function creates a list of operations to perform unmapping for every + * GPUVA attached to a GEM. +@@ -1766,15 +2013,14 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_prefetch_ops_create); + * Returns: a pointer to the &drm_gpuva_ops on success, an ERR_PTR on failure + */ + struct drm_gpuva_ops * +-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, +- struct drm_gem_object *obj) ++drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo) + { + struct drm_gpuva_ops *ops; + struct drm_gpuva_op *op; + struct drm_gpuva *va; + int ret; + +- drm_gem_gpuva_assert_lock_held(obj); ++ drm_gem_gpuva_assert_lock_held(vm_bo->obj); + + ops = kzalloc(sizeof(*ops), GFP_KERNEL); + if (!ops) +@@ -1782,8 +2028,8 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, + + INIT_LIST_HEAD(&ops->list); + +- drm_gem_for_each_gpuva(va, obj) { +- op = gpuva_op_alloc(gpuvm); ++ drm_gpuvm_bo_for_each_va(va, vm_bo) { ++ op = gpuva_op_alloc(vm_bo->vm); + if (!op) { + ret = -ENOMEM; + goto err_free_ops; +@@ -1797,10 +2043,10 @@ drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, + return ops; + + err_free_ops: +- drm_gpuva_ops_free(gpuvm, ops); ++ drm_gpuva_ops_free(vm_bo->vm, ops); + return ERR_PTR(ret); + } +-EXPORT_SYMBOL_GPL(drm_gpuvm_gem_unmap_ops_create); ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_unmap_ops_create); + + /** + * drm_gpuva_ops_free() - free the given &drm_gpuva_ops +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index f6bb5e8ec3eda5..2b27a656b2679d 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -520,7 +520,7 @@ unsigned long drm_gem_lru_scan(struct drm_gem_lru *lru, + * drm_gem_gpuva_init() - initialize the gpuva list of a GEM object + * @obj: the &drm_gem_object + * +- * This initializes the &drm_gem_object's &drm_gpuva list. ++ * This initializes the &drm_gem_object's &drm_gpuvm_bo list. + * + * Calling this function is only necessary for drivers intending to support the + * &drm_driver_feature DRIVER_GEM_GPUVA. +@@ -532,28 +532,28 @@ static inline void drm_gem_gpuva_init(struct drm_gem_object *obj) + } + + /** +- * drm_gem_for_each_gpuva() - iternator to walk over a list of gpuvas +- * @entry__: &drm_gpuva structure to assign to in each iteration step +- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with ++ * drm_gem_for_each_gpuvm_bo() - iterator to walk over a list of &drm_gpuvm_bo ++ * @entry__: &drm_gpuvm_bo structure to assign to in each iteration step ++ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with + * +- * This iterator walks over all &drm_gpuva structures associated with the +- * &drm_gpuva_manager. ++ * This iterator walks over all &drm_gpuvm_bo structures associated with the ++ * &drm_gem_object. + */ +-#define drm_gem_for_each_gpuva(entry__, obj__) \ +- list_for_each_entry(entry__, &(obj__)->gpuva.list, gem.entry) ++#define drm_gem_for_each_gpuvm_bo(entry__, obj__) \ ++ list_for_each_entry(entry__, &(obj__)->gpuva.list, list.entry.gem) + + /** +- * drm_gem_for_each_gpuva_safe() - iternator to safely walk over a list of +- * gpuvas +- * @entry__: &drm_gpuva structure to assign to in each iteration step +- * @next__: &next &drm_gpuva to store the next step +- * @obj__: the &drm_gem_object the &drm_gpuvas to walk are associated with ++ * drm_gem_for_each_gpuvm_bo_safe() - iterator to safely walk over a list of ++ * &drm_gpuvm_bo ++ * @entry__: &drm_gpuvm_bostructure to assign to in each iteration step ++ * @next__: &next &drm_gpuvm_bo to store the next step ++ * @obj__: the &drm_gem_object the &drm_gpuvm_bo to walk are associated with + * +- * This iterator walks over all &drm_gpuva structures associated with the ++ * This iterator walks over all &drm_gpuvm_bo structures associated with the + * &drm_gem_object. It is implemented with list_for_each_entry_safe(), hence + * it is save against removal of elements. + */ +-#define drm_gem_for_each_gpuva_safe(entry__, next__, obj__) \ +- list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, gem.entry) ++#define drm_gem_for_each_gpuvm_bo_safe(entry__, next__, obj__) \ ++ list_for_each_entry_safe(entry__, next__, &(obj__)->gpuva.list, list.entry.gem) + + #endif /* __DRM_GEM_H__ */ +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 8bfa30b9df14d2..0363c2017da2eb 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -25,6 +25,7 @@ + * OTHER DEALINGS IN THE SOFTWARE. + */ + ++#include + #include + #include + #include +@@ -33,6 +34,7 @@ + #include + + struct drm_gpuvm; ++struct drm_gpuvm_bo; + struct drm_gpuvm_ops; + + /** +@@ -73,6 +75,12 @@ struct drm_gpuva { + */ + struct drm_gpuvm *vm; + ++ /** ++ * @vm_bo: the &drm_gpuvm_bo abstraction for the mapped ++ * &drm_gem_object ++ */ ++ struct drm_gpuvm_bo *vm_bo; ++ + /** + * @flags: the &drm_gpuva_flags for this mapping + */ +@@ -108,7 +116,7 @@ struct drm_gpuva { + struct drm_gem_object *obj; + + /** +- * @entry: the &list_head to attach this object to a &drm_gem_object ++ * @entry: the &list_head to attach this object to a &drm_gpuvm_bo + */ + struct list_head entry; + } gem; +@@ -141,7 +149,7 @@ struct drm_gpuva { + int drm_gpuva_insert(struct drm_gpuvm *gpuvm, struct drm_gpuva *va); + void drm_gpuva_remove(struct drm_gpuva *va); + +-void drm_gpuva_link(struct drm_gpuva *va); ++void drm_gpuva_link(struct drm_gpuva *va, struct drm_gpuvm_bo *vm_bo); + void drm_gpuva_unlink(struct drm_gpuva *va); + + struct drm_gpuva *drm_gpuva_find(struct drm_gpuvm *gpuvm, +@@ -188,10 +196,16 @@ static inline bool drm_gpuva_invalidated(struct drm_gpuva *va) + * enum drm_gpuvm_flags - flags for struct drm_gpuvm + */ + enum drm_gpuvm_flags { ++ /** ++ * @DRM_GPUVM_RESV_PROTECTED: GPUVM is protected externally by the ++ * GPUVM's &dma_resv lock ++ */ ++ DRM_GPUVM_RESV_PROTECTED = BIT(0), ++ + /** + * @DRM_GPUVM_USERBITS: user defined bits + */ +- DRM_GPUVM_USERBITS = BIT(0), ++ DRM_GPUVM_USERBITS = BIT(1), + }; + + /** +@@ -302,6 +316,19 @@ bool drm_gpuvm_interval_empty(struct drm_gpuvm *gpuvm, u64 addr, u64 range); + struct drm_gem_object * + drm_gpuvm_resv_object_alloc(struct drm_device *drm); + ++/** ++ * drm_gpuvm_resv_protected() - indicates whether &DRM_GPUVM_RESV_PROTECTED is ++ * set ++ * @gpuvm: the &drm_gpuvm ++ * ++ * Returns: true if &DRM_GPUVM_RESV_PROTECTED is set, false otherwise. ++ */ ++static inline bool ++drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm) ++{ ++ return gpuvm->flags & DRM_GPUVM_RESV_PROTECTED; ++} ++ + /** + * drm_gpuvm_resv() - returns the &drm_gpuvm's &dma_resv + * @gpuvm__: the &drm_gpuvm +@@ -320,6 +347,12 @@ drm_gpuvm_resv_object_alloc(struct drm_device *drm); + */ + #define drm_gpuvm_resv_obj(gpuvm__) ((gpuvm__)->r_obj) + ++#define drm_gpuvm_resv_held(gpuvm__) \ ++ dma_resv_held(drm_gpuvm_resv(gpuvm__)) ++ ++#define drm_gpuvm_resv_assert_held(gpuvm__) \ ++ dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) ++ + #define drm_gpuvm_resv_held(gpuvm__) \ + dma_resv_held(drm_gpuvm_resv(gpuvm__)) + +@@ -404,6 +437,125 @@ __drm_gpuva_next(struct drm_gpuva *va) + #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \ + list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry) + ++/** ++ * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and ++ * &drm_gem_object combination ++ * ++ * This structure is an abstraction representing a &drm_gpuvm and ++ * &drm_gem_object combination. It serves as an indirection to accelerate ++ * iterating all &drm_gpuvas within a &drm_gpuvm backed by the same ++ * &drm_gem_object. ++ * ++ * Furthermore it is used cache evicted GEM objects for a certain GPU-VM to ++ * accelerate validation. ++ * ++ * Typically, drivers want to create an instance of a struct drm_gpuvm_bo once ++ * a GEM object is mapped first in a GPU-VM and release the instance once the ++ * last mapping of the GEM object in this GPU-VM is unmapped. ++ */ ++struct drm_gpuvm_bo { ++ /** ++ * @vm: The &drm_gpuvm the @obj is mapped in. This is a reference ++ * counted pointer. ++ */ ++ struct drm_gpuvm *vm; ++ ++ /** ++ * @obj: The &drm_gem_object being mapped in @vm. This is a reference ++ * counted pointer. ++ */ ++ struct drm_gem_object *obj; ++ ++ /** ++ * @kref: The reference count for this &drm_gpuvm_bo. ++ */ ++ struct kref kref; ++ ++ /** ++ * @list: Structure containing all &list_heads. ++ */ ++ struct { ++ /** ++ * @gpuva: The list of linked &drm_gpuvas. ++ * ++ * It is safe to access entries from this list as long as the ++ * GEM's gpuva lock is held. See also struct drm_gem_object. ++ */ ++ struct list_head gpuva; ++ ++ /** ++ * @entry: Structure containing all &list_heads serving as ++ * entry. ++ */ ++ struct { ++ /** ++ * @gem: List entry to attach to the &drm_gem_objects ++ * gpuva list. ++ */ ++ struct list_head gem; ++ } entry; ++ } list; ++}; ++ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj); ++ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_obtain(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj); ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *vm_bo); ++ ++/** ++ * drm_gpuvm_bo_get() - acquire a struct drm_gpuvm_bo reference ++ * @vm_bo: the &drm_gpuvm_bo to acquire the reference of ++ * ++ * This function acquires an additional reference to @vm_bo. It is illegal to ++ * call this without already holding a reference. No locks required. ++ */ ++static inline struct drm_gpuvm_bo * ++drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) ++{ ++ kref_get(&vm_bo->kref); ++ return vm_bo; ++} ++ ++void drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); ++ ++struct drm_gpuvm_bo * ++drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj); ++ ++/** ++ * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva ++ * @va__: &drm_gpuva structure to assign to in each iteration step ++ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the ++ * &drm_gpuvm_bo. ++ * ++ * The caller must hold the GEM's gpuva lock. ++ */ ++#define drm_gpuvm_bo_for_each_va(va__, vm_bo__) \ ++ list_for_each_entry(va__, &(vm_bo)->list.gpuva, gem.entry) ++ ++/** ++ * drm_gpuvm_bo_for_each_va_safe() - iterator to safely walk over a list of ++ * &drm_gpuva ++ * @va__: &drm_gpuva structure to assign to in each iteration step ++ * @next__: &next &drm_gpuva to store the next step ++ * @vm_bo__: the &drm_gpuvm_bo the &drm_gpuva to walk are associated with ++ * ++ * This iterator walks over all &drm_gpuva structures associated with the ++ * &drm_gpuvm_bo. It is implemented with list_for_each_entry_safe(), hence ++ * it is save against removal of elements. ++ * ++ * The caller must hold the GEM's gpuva lock. ++ */ ++#define drm_gpuvm_bo_for_each_va_safe(va__, next__, vm_bo__) \ ++ list_for_each_entry_safe(va__, next__, &(vm_bo)->list.gpuva, gem.entry) ++ + /** + * enum drm_gpuva_op_type - GPU VA operation type + * +@@ -673,8 +825,7 @@ drm_gpuvm_prefetch_ops_create(struct drm_gpuvm *gpuvm, + u64 addr, u64 range); + + struct drm_gpuva_ops * +-drm_gpuvm_gem_unmap_ops_create(struct drm_gpuvm *gpuvm, +- struct drm_gem_object *obj); ++drm_gpuvm_bo_unmap_ops_create(struct drm_gpuvm_bo *vm_bo); + + void drm_gpuva_ops_free(struct drm_gpuvm *gpuvm, + struct drm_gpuva_ops *ops); +@@ -726,6 +877,30 @@ struct drm_gpuvm_ops { + */ + void (*op_free)(struct drm_gpuva_op *op); + ++ /** ++ * @vm_bo_alloc: called when the &drm_gpuvm allocates ++ * a struct drm_gpuvm_bo ++ * ++ * Some drivers may want to embed struct drm_gpuvm_bo into driver ++ * specific structures. By implementing this callback drivers can ++ * allocate memory accordingly. ++ * ++ * This callback is optional. ++ */ ++ struct drm_gpuvm_bo *(*vm_bo_alloc)(void); ++ ++ /** ++ * @vm_bo_free: called when the &drm_gpuvm frees a ++ * struct drm_gpuvm_bo ++ * ++ * Some drivers may want to embed struct drm_gpuvm_bo into driver ++ * specific structures. By implementing this callback drivers can ++ * free the previously allocated memory accordingly. ++ * ++ * This callback is optional. ++ */ ++ void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo); ++ + /** + * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the + * mapping once all previous steps were completed + +From b73f2c7f0161460b3b90f5527fd88d8bba94530c Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 8 Nov 2023 01:12:41 +0100 +Subject: [PATCH 015/150] drm/gpuvm: track/lock/validate external/evicted + objects +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Currently the DRM GPUVM offers common infrastructure to track GPU VA +allocations and mappings, generically connect GPU VA mappings to their +backing buffers and perform more complex mapping operations on the GPU VA +space. + +However, there are more design patterns commonly used by drivers, which +can potentially be generalized in order to make the DRM GPUVM represent +a basis for GPU-VM implementations. In this context, this patch aims +at generalizing the following elements. + +1) Provide a common dma-resv for GEM objects not being used outside of + this GPU-VM. + +2) Provide tracking of external GEM objects (GEM objects which are + shared with other GPU-VMs). + +3) Provide functions to efficiently lock all GEM objects dma-resv the + GPU-VM contains mappings of. + +4) Provide tracking of evicted GEM objects the GPU-VM contains mappings + of, such that validation of evicted GEM objects is accelerated. + +5) Provide some convinience functions for common patterns. + +Big thanks to Boris Brezillon for his help to figure out locking for +drivers updating the GPU VA space within the fence signalling path. + +Acked-by: Christian König +Reviewed-by: Boris Brezillon +Reviewed-by: Thomas Hellström +Suggested-by: Matthew Brost +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231108001259.15123-12-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 633 ++++++++++++++++++++++++++++++++++++ + include/drm/drm_gpuvm.h | 250 ++++++++++++++ + 2 files changed, 883 insertions(+) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 02e7cfcb80631d..92815da084ce04 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -82,6 +82,21 @@ + * &drm_gem_object list of &drm_gpuvm_bos for an existing instance of this + * particular combination. If not existent a new instance is created and linked + * to the &drm_gem_object. ++ * ++ * &drm_gpuvm_bo structures, since unique for a given &drm_gpuvm, are also used ++ * as entry for the &drm_gpuvm's lists of external and evicted objects. Those ++ * lists are maintained in order to accelerate locking of dma-resv locks and ++ * validation of evicted objects bound in a &drm_gpuvm. For instance, all ++ * &drm_gem_object's &dma_resv of a given &drm_gpuvm can be locked by calling ++ * drm_gpuvm_exec_lock(). Once locked drivers can call drm_gpuvm_validate() in ++ * order to validate all evicted &drm_gem_objects. It is also possible to lock ++ * additional &drm_gem_objects by providing the corresponding parameters to ++ * drm_gpuvm_exec_lock() as well as open code the &drm_exec loop while making ++ * use of helper functions such as drm_gpuvm_prepare_range() or ++ * drm_gpuvm_prepare_objects(). ++ * ++ * Every bound &drm_gem_object is treated as external object when its &dma_resv ++ * structure is different than the &drm_gpuvm's common &dma_resv structure. + */ + + /** +@@ -429,6 +444,20 @@ + * Subsequent calls to drm_gpuvm_bo_obtain() for the same &drm_gpuvm and + * &drm_gem_object must be able to observe previous creations and destructions + * of &drm_gpuvm_bos in order to keep instances unique. ++ * ++ * The &drm_gpuvm's lists for keeping track of external and evicted objects are ++ * protected against concurrent insertion / removal and iteration internally. ++ * ++ * However, drivers still need ensure to protect concurrent calls to functions ++ * iterating those lists, namely drm_gpuvm_prepare_objects() and ++ * drm_gpuvm_validate(). ++ * ++ * Alternatively, drivers can set the &DRM_GPUVM_RESV_PROTECTED flag to indicate ++ * that the corresponding &dma_resv locks are held in order to protect the ++ * lists. If &DRM_GPUVM_RESV_PROTECTED is set, internal locking is disabled and ++ * the corresponding lockdep checks are enabled. This is an optimization for ++ * drivers which are capable of taking the corresponding &dma_resv locks and ++ * hence do not require internal locking. + */ + + /** +@@ -641,6 +670,201 @@ + * } + */ + ++/** ++ * get_next_vm_bo_from_list() - get the next vm_bo element ++ * @__gpuvm: the &drm_gpuvm ++ * @__list_name: the name of the list we're iterating on ++ * @__local_list: a pointer to the local list used to store already iterated items ++ * @__prev_vm_bo: the previous element we got from get_next_vm_bo_from_list() ++ * ++ * This helper is here to provide lockless list iteration. Lockless as in, the ++ * iterator releases the lock immediately after picking the first element from ++ * the list, so list insertion deletion can happen concurrently. ++ * ++ * Elements popped from the original list are kept in a local list, so removal ++ * and is_empty checks can still happen while we're iterating the list. ++ */ ++#define get_next_vm_bo_from_list(__gpuvm, __list_name, __local_list, __prev_vm_bo) \ ++ ({ \ ++ struct drm_gpuvm_bo *__vm_bo = NULL; \ ++ \ ++ drm_gpuvm_bo_put(__prev_vm_bo); \ ++ \ ++ spin_lock(&(__gpuvm)->__list_name.lock); \ ++ if (!(__gpuvm)->__list_name.local_list) \ ++ (__gpuvm)->__list_name.local_list = __local_list; \ ++ else \ ++ drm_WARN_ON((__gpuvm)->drm, \ ++ (__gpuvm)->__list_name.local_list != __local_list); \ ++ \ ++ while (!list_empty(&(__gpuvm)->__list_name.list)) { \ ++ __vm_bo = list_first_entry(&(__gpuvm)->__list_name.list, \ ++ struct drm_gpuvm_bo, \ ++ list.entry.__list_name); \ ++ if (kref_get_unless_zero(&__vm_bo->kref)) { \ ++ list_move_tail(&(__vm_bo)->list.entry.__list_name, \ ++ __local_list); \ ++ break; \ ++ } else { \ ++ list_del_init(&(__vm_bo)->list.entry.__list_name); \ ++ __vm_bo = NULL; \ ++ } \ ++ } \ ++ spin_unlock(&(__gpuvm)->__list_name.lock); \ ++ \ ++ __vm_bo; \ ++ }) ++ ++/** ++ * for_each_vm_bo_in_list() - internal vm_bo list iterator ++ * @__gpuvm: the &drm_gpuvm ++ * @__list_name: the name of the list we're iterating on ++ * @__local_list: a pointer to the local list used to store already iterated items ++ * @__vm_bo: the struct drm_gpuvm_bo to assign in each iteration step ++ * ++ * This helper is here to provide lockless list iteration. Lockless as in, the ++ * iterator releases the lock immediately after picking the first element from the ++ * list, hence list insertion and deletion can happen concurrently. ++ * ++ * It is not allowed to re-assign the vm_bo pointer from inside this loop. ++ * ++ * Typical use: ++ * ++ * struct drm_gpuvm_bo *vm_bo; ++ * LIST_HEAD(my_local_list); ++ * ++ * ret = 0; ++ * for_each_vm_bo_in_list(gpuvm, , &my_local_list, vm_bo) { ++ * ret = do_something_with_vm_bo(..., vm_bo); ++ * if (ret) ++ * break; ++ * } ++ * // Drop ref in case we break out of the loop. ++ * drm_gpuvm_bo_put(vm_bo); ++ * restore_vm_bo_list(gpuvm, , &my_local_list); ++ * ++ * ++ * Only used for internal list iterations, not meant to be exposed to the outside ++ * world. ++ */ ++#define for_each_vm_bo_in_list(__gpuvm, __list_name, __local_list, __vm_bo) \ ++ for (__vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ ++ __local_list, NULL); \ ++ __vm_bo; \ ++ __vm_bo = get_next_vm_bo_from_list(__gpuvm, __list_name, \ ++ __local_list, __vm_bo)) ++ ++static void ++__restore_vm_bo_list(struct drm_gpuvm *gpuvm, spinlock_t *lock, ++ struct list_head *list, struct list_head **local_list) ++{ ++ /* Merge back the two lists, moving local list elements to the ++ * head to preserve previous ordering, in case it matters. ++ */ ++ spin_lock(lock); ++ if (*local_list) { ++ list_splice(*local_list, list); ++ *local_list = NULL; ++ } ++ spin_unlock(lock); ++} ++ ++/** ++ * restore_vm_bo_list() - move vm_bo elements back to their original list ++ * @__gpuvm: the &drm_gpuvm ++ * @__list_name: the name of the list we're iterating on ++ * ++ * When we're done iterating a vm_bo list, we should call restore_vm_bo_list() ++ * to restore the original state and let new iterations take place. ++ */ ++#define restore_vm_bo_list(__gpuvm, __list_name) \ ++ __restore_vm_bo_list((__gpuvm), &(__gpuvm)->__list_name.lock, \ ++ &(__gpuvm)->__list_name.list, \ ++ &(__gpuvm)->__list_name.local_list) ++ ++static void ++cond_spin_lock(spinlock_t *lock, bool cond) ++{ ++ if (cond) ++ spin_lock(lock); ++} ++ ++static void ++cond_spin_unlock(spinlock_t *lock, bool cond) ++{ ++ if (cond) ++ spin_unlock(lock); ++} ++ ++static void ++__drm_gpuvm_bo_list_add(struct drm_gpuvm *gpuvm, spinlock_t *lock, ++ struct list_head *entry, struct list_head *list) ++{ ++ cond_spin_lock(lock, !!lock); ++ if (list_empty(entry)) ++ list_add_tail(entry, list); ++ cond_spin_unlock(lock, !!lock); ++} ++ ++/** ++ * drm_gpuvm_bo_list_add() - insert a vm_bo into the given list ++ * @__vm_bo: the &drm_gpuvm_bo ++ * @__list_name: the name of the list to insert into ++ * @__lock: whether to lock with the internal spinlock ++ * ++ * Inserts the given @__vm_bo into the list specified by @__list_name. ++ */ ++#define drm_gpuvm_bo_list_add(__vm_bo, __list_name, __lock) \ ++ __drm_gpuvm_bo_list_add((__vm_bo)->vm, \ ++ __lock ? &(__vm_bo)->vm->__list_name.lock : \ ++ NULL, \ ++ &(__vm_bo)->list.entry.__list_name, \ ++ &(__vm_bo)->vm->__list_name.list) ++ ++static void ++__drm_gpuvm_bo_list_del(struct drm_gpuvm *gpuvm, spinlock_t *lock, ++ struct list_head *entry, bool init) ++{ ++ cond_spin_lock(lock, !!lock); ++ if (init) { ++ if (!list_empty(entry)) ++ list_del_init(entry); ++ } else { ++ list_del(entry); ++ } ++ cond_spin_unlock(lock, !!lock); ++} ++ ++/** ++ * drm_gpuvm_bo_list_del_init() - remove a vm_bo from the given list ++ * @__vm_bo: the &drm_gpuvm_bo ++ * @__list_name: the name of the list to insert into ++ * @__lock: whether to lock with the internal spinlock ++ * ++ * Removes the given @__vm_bo from the list specified by @__list_name. ++ */ ++#define drm_gpuvm_bo_list_del_init(__vm_bo, __list_name, __lock) \ ++ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ ++ __lock ? &(__vm_bo)->vm->__list_name.lock : \ ++ NULL, \ ++ &(__vm_bo)->list.entry.__list_name, \ ++ true) ++ ++/** ++ * drm_gpuvm_bo_list_del() - remove a vm_bo from the given list ++ * @__vm_bo: the &drm_gpuvm_bo ++ * @__list_name: the name of the list to insert into ++ * @__lock: whether to lock with the internal spinlock ++ * ++ * Removes the given @__vm_bo from the list specified by @__list_name. ++ */ ++#define drm_gpuvm_bo_list_del(__vm_bo, __list_name, __lock) \ ++ __drm_gpuvm_bo_list_del((__vm_bo)->vm, \ ++ __lock ? &(__vm_bo)->vm->__list_name.lock : \ ++ NULL, \ ++ &(__vm_bo)->list.entry.__list_name, \ ++ false) ++ + #define to_drm_gpuva(__node) container_of((__node), struct drm_gpuva, rb.node) + + #define GPUVA_START(node) ((node)->va.addr) +@@ -781,6 +1005,12 @@ drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, + gpuvm->rb.tree = RB_ROOT_CACHED; + INIT_LIST_HEAD(&gpuvm->rb.list); + ++ INIT_LIST_HEAD(&gpuvm->extobj.list); ++ spin_lock_init(&gpuvm->extobj.lock); ++ ++ INIT_LIST_HEAD(&gpuvm->evict.list); ++ spin_lock_init(&gpuvm->evict.lock); ++ + kref_init(&gpuvm->kref); + + gpuvm->name = name ? name : "unknown"; +@@ -818,6 +1048,11 @@ drm_gpuvm_fini(struct drm_gpuvm *gpuvm) + drm_WARN(gpuvm->drm, !RB_EMPTY_ROOT(&gpuvm->rb.tree.rb_root), + "GPUVA tree is not empty, potentially leaking memory.\n"); + ++ drm_WARN(gpuvm->drm, !list_empty(&gpuvm->extobj.list), ++ "Extobj list should be empty.\n"); ++ drm_WARN(gpuvm->drm, !list_empty(&gpuvm->evict.list), ++ "Evict list should be empty.\n"); ++ + drm_gem_object_put(gpuvm->r_obj); + } + +@@ -850,6 +1085,343 @@ drm_gpuvm_put(struct drm_gpuvm *gpuvm) + } + EXPORT_SYMBOL_GPL(drm_gpuvm_put); + ++static int ++__drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences) ++{ ++ struct drm_gpuvm_bo *vm_bo; ++ LIST_HEAD(extobjs); ++ int ret = 0; ++ ++ for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) { ++ ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences); ++ if (ret) ++ break; ++ } ++ /* Drop ref in case we break out of the loop. */ ++ drm_gpuvm_bo_put(vm_bo); ++ restore_vm_bo_list(gpuvm, extobj); ++ ++ return ret; ++} ++ ++static int ++drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences) ++{ ++ struct drm_gpuvm_bo *vm_bo; ++ int ret = 0; ++ ++ drm_gpuvm_resv_assert_held(gpuvm); ++ list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) { ++ ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences); ++ if (ret) ++ break; ++ ++ if (vm_bo->evicted) ++ drm_gpuvm_bo_list_add(vm_bo, evict, false); ++ } ++ ++ return ret; ++} ++ ++/** ++ * drm_gpuvm_prepare_objects() - prepare all assoiciated BOs ++ * @gpuvm: the &drm_gpuvm ++ * @exec: the &drm_exec locking context ++ * @num_fences: the amount of &dma_fences to reserve ++ * ++ * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given ++ * &drm_gpuvm contains mappings of. ++ * ++ * Using this function directly, it is the drivers responsibility to call ++ * drm_exec_init() and drm_exec_fini() accordingly. ++ * ++ * Note: This function is safe against concurrent insertion and removal of ++ * external objects, however it is not safe against concurrent usage itself. ++ * ++ * Drivers need to make sure to protect this case with either an outer VM lock ++ * or by calling drm_gpuvm_prepare_vm() before this function within the ++ * drm_exec_until_all_locked() loop, such that the GPUVM's dma-resv lock ensures ++ * mutual exclusion. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences) ++{ ++ if (drm_gpuvm_resv_protected(gpuvm)) ++ return drm_gpuvm_prepare_objects_locked(gpuvm, exec, ++ num_fences); ++ else ++ return __drm_gpuvm_prepare_objects(gpuvm, exec, num_fences); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects); ++ ++/** ++ * drm_gpuvm_prepare_range() - prepare all BOs mapped within a given range ++ * @gpuvm: the &drm_gpuvm ++ * @exec: the &drm_exec locking context ++ * @addr: the start address within the VA space ++ * @range: the range to iterate within the VA space ++ * @num_fences: the amount of &dma_fences to reserve ++ * ++ * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr ++ * and @addr + @range. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, ++ u64 addr, u64 range, unsigned int num_fences) ++{ ++ struct drm_gpuva *va; ++ u64 end = addr + range; ++ int ret; ++ ++ drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { ++ struct drm_gem_object *obj = va->gem.obj; ++ ++ ret = drm_exec_prepare_obj(exec, obj, num_fences); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_range); ++ ++/** ++ * drm_gpuvm_exec_lock() - lock all dma-resv of all assoiciated BOs ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * ++ * Acquires all dma-resv locks of all &drm_gem_objects the given ++ * &drm_gpuvm contains mappings of. ++ * ++ * Addionally, when calling this function with struct drm_gpuvm_exec::extra ++ * being set the driver receives the given @fn callback to lock additional ++ * dma-resv in the context of the &drm_gpuvm_exec instance. Typically, drivers ++ * would call drm_exec_prepare_obj() from within this callback. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec) ++{ ++ struct drm_gpuvm *gpuvm = vm_exec->vm; ++ struct drm_exec *exec = &vm_exec->exec; ++ unsigned int num_fences = vm_exec->num_fences; ++ int ret; ++ ++ drm_exec_init(exec, vm_exec->flags); ++ ++ drm_exec_until_all_locked(exec) { ++ ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences); ++ drm_exec_retry_on_contention(exec); ++ if (ret) ++ goto err; ++ ++ ret = drm_gpuvm_prepare_objects(gpuvm, exec, num_fences); ++ drm_exec_retry_on_contention(exec); ++ if (ret) ++ goto err; ++ ++ if (vm_exec->extra.fn) { ++ ret = vm_exec->extra.fn(vm_exec); ++ drm_exec_retry_on_contention(exec); ++ if (ret) ++ goto err; ++ } ++ } ++ ++ return 0; ++ ++err: ++ drm_exec_fini(exec); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock); ++ ++static int ++fn_lock_array(struct drm_gpuvm_exec *vm_exec) ++{ ++ struct { ++ struct drm_gem_object **objs; ++ unsigned int num_objs; ++ } *args = vm_exec->extra.priv; ++ ++ return drm_exec_prepare_array(&vm_exec->exec, args->objs, ++ args->num_objs, vm_exec->num_fences); ++} ++ ++/** ++ * drm_gpuvm_exec_lock_array() - lock all dma-resv of all assoiciated BOs ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * @objs: additional &drm_gem_objects to lock ++ * @num_objs: the number of additional &drm_gem_objects to lock ++ * ++ * Acquires all dma-resv locks of all &drm_gem_objects the given &drm_gpuvm ++ * contains mappings of, plus the ones given through @objs. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, ++ struct drm_gem_object **objs, ++ unsigned int num_objs) ++{ ++ struct { ++ struct drm_gem_object **objs; ++ unsigned int num_objs; ++ } args; ++ ++ args.objs = objs; ++ args.num_objs = num_objs; ++ ++ vm_exec->extra.fn = fn_lock_array; ++ vm_exec->extra.priv = &args; ++ ++ return drm_gpuvm_exec_lock(vm_exec); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_array); ++ ++/** ++ * drm_gpuvm_exec_lock_range() - prepare all BOs mapped within a given range ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * @addr: the start address within the VA space ++ * @range: the range to iterate within the VA space ++ * ++ * Acquires all dma-resv locks of all &drm_gem_objects mapped between @addr and ++ * @addr + @range. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, ++ u64 addr, u64 range) ++{ ++ struct drm_gpuvm *gpuvm = vm_exec->vm; ++ struct drm_exec *exec = &vm_exec->exec; ++ int ret; ++ ++ drm_exec_init(exec, vm_exec->flags); ++ ++ drm_exec_until_all_locked(exec) { ++ ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range, ++ vm_exec->num_fences); ++ drm_exec_retry_on_contention(exec); ++ if (ret) ++ goto err; ++ } ++ ++ return ret; ++ ++err: ++ drm_exec_fini(exec); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_exec_lock_range); ++ ++static int ++__drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) ++{ ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; ++ struct drm_gpuvm_bo *vm_bo; ++ LIST_HEAD(evict); ++ int ret = 0; ++ ++ for_each_vm_bo_in_list(gpuvm, evict, &evict, vm_bo) { ++ ret = ops->vm_bo_validate(vm_bo, exec); ++ if (ret) ++ break; ++ } ++ /* Drop ref in case we break out of the loop. */ ++ drm_gpuvm_bo_put(vm_bo); ++ restore_vm_bo_list(gpuvm, evict); ++ ++ return ret; ++} ++ ++static int ++drm_gpuvm_validate_locked(struct drm_gpuvm *gpuvm, struct drm_exec *exec) ++{ ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; ++ struct drm_gpuvm_bo *vm_bo, *next; ++ int ret = 0; ++ ++ drm_gpuvm_resv_assert_held(gpuvm); ++ ++ list_for_each_entry_safe(vm_bo, next, &gpuvm->evict.list, ++ list.entry.evict) { ++ ret = ops->vm_bo_validate(vm_bo, exec); ++ if (ret) ++ break; ++ ++ dma_resv_assert_held(vm_bo->obj->resv); ++ if (!vm_bo->evicted) ++ drm_gpuvm_bo_list_del_init(vm_bo, evict, false); ++ } ++ ++ return ret; ++} ++ ++/** ++ * drm_gpuvm_validate() - validate all BOs marked as evicted ++ * @gpuvm: the &drm_gpuvm to validate evicted BOs ++ * @exec: the &drm_exec instance used for locking the GPUVM ++ * ++ * Calls the &drm_gpuvm_ops::vm_bo_validate callback for all evicted buffer ++ * objects being mapped in the given &drm_gpuvm. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec) ++{ ++ const struct drm_gpuvm_ops *ops = gpuvm->ops; ++ ++ if (unlikely(!ops || !ops->vm_bo_validate)) ++ return -EOPNOTSUPP; ++ ++ if (drm_gpuvm_resv_protected(gpuvm)) ++ return drm_gpuvm_validate_locked(gpuvm, exec); ++ else ++ return __drm_gpuvm_validate(gpuvm, exec); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_validate); ++ ++/** ++ * drm_gpuvm_resv_add_fence - add fence to private and all extobj ++ * dma-resv ++ * @gpuvm: the &drm_gpuvm to add a fence to ++ * @exec: the &drm_exec locking context ++ * @fence: fence to add ++ * @private_usage: private dma-resv usage ++ * @extobj_usage: extobj dma-resv usage ++ */ ++void ++drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ struct dma_fence *fence, ++ enum dma_resv_usage private_usage, ++ enum dma_resv_usage extobj_usage) ++{ ++ struct drm_gem_object *obj; ++ unsigned long index; ++ ++ drm_exec_for_each_locked_object(exec, index, obj) { ++ dma_resv_assert_held(obj->resv); ++ dma_resv_add_fence(obj->resv, fence, ++ drm_gpuvm_is_extobj(gpuvm, obj) ? ++ extobj_usage : private_usage); ++ } ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_resv_add_fence); ++ + /** + * drm_gpuvm_bo_create() - create a new instance of struct drm_gpuvm_bo + * @gpuvm: The &drm_gpuvm the @obj is mapped in. +@@ -883,6 +1455,9 @@ drm_gpuvm_bo_create(struct drm_gpuvm *gpuvm, + INIT_LIST_HEAD(&vm_bo->list.gpuva); + INIT_LIST_HEAD(&vm_bo->list.entry.gem); + ++ INIT_LIST_HEAD(&vm_bo->list.entry.extobj); ++ INIT_LIST_HEAD(&vm_bo->list.entry.evict); ++ + return vm_bo; + } + EXPORT_SYMBOL_GPL(drm_gpuvm_bo_create); +@@ -900,6 +1475,9 @@ drm_gpuvm_bo_destroy(struct kref *kref) + if (!lock) + drm_gpuvm_resv_assert_held(gpuvm); + ++ drm_gpuvm_bo_list_del(vm_bo, extobj, lock); ++ drm_gpuvm_bo_list_del(vm_bo, evict, lock); ++ + drm_gem_gpuva_assert_lock_held(obj); + list_del(&vm_bo->list.entry.gem); + +@@ -1043,6 +1621,61 @@ drm_gpuvm_bo_obtain_prealloc(struct drm_gpuvm_bo *__vm_bo) + } + EXPORT_SYMBOL_GPL(drm_gpuvm_bo_obtain_prealloc); + ++/** ++ * drm_gpuvm_bo_extobj_add() - adds the &drm_gpuvm_bo to its &drm_gpuvm's ++ * extobj list ++ * @vm_bo: The &drm_gpuvm_bo to add to its &drm_gpuvm's the extobj list. ++ * ++ * Adds the given @vm_bo to its &drm_gpuvm's extobj list if not on the list ++ * already and if the corresponding &drm_gem_object is an external object, ++ * actually. ++ */ ++void ++drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo) ++{ ++ struct drm_gpuvm *gpuvm = vm_bo->vm; ++ bool lock = !drm_gpuvm_resv_protected(gpuvm); ++ ++ if (!lock) ++ drm_gpuvm_resv_assert_held(gpuvm); ++ ++ if (drm_gpuvm_is_extobj(gpuvm, vm_bo->obj)) ++ drm_gpuvm_bo_list_add(vm_bo, extobj, lock); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_extobj_add); ++ ++/** ++ * drm_gpuvm_bo_evict() - add / remove a &drm_gpuvm_bo to / from the &drm_gpuvms ++ * evicted list ++ * @vm_bo: the &drm_gpuvm_bo to add or remove ++ * @evict: indicates whether the object is evicted ++ * ++ * Adds a &drm_gpuvm_bo to or removes it from the &drm_gpuvms evicted list. ++ */ ++void ++drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict) ++{ ++ struct drm_gpuvm *gpuvm = vm_bo->vm; ++ struct drm_gem_object *obj = vm_bo->obj; ++ bool lock = !drm_gpuvm_resv_protected(gpuvm); ++ ++ dma_resv_assert_held(obj->resv); ++ vm_bo->evicted = evict; ++ ++ /* Can't add external objects to the evicted list directly if not using ++ * internal spinlocks, since in this case the evicted list is protected ++ * with the VM's common dma-resv lock. ++ */ ++ if (drm_gpuvm_is_extobj(gpuvm, obj) && !lock) ++ return; ++ ++ if (evict) ++ drm_gpuvm_bo_list_add(vm_bo, evict, lock); ++ else ++ drm_gpuvm_bo_list_del_init(vm_bo, evict, lock); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_bo_evict); ++ + static int + __drm_gpuva_insert(struct drm_gpuvm *gpuvm, + struct drm_gpuva *va) +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 0363c2017da2eb..74baa9d5da21df 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -32,6 +32,7 @@ + + #include + #include ++#include + + struct drm_gpuvm; + struct drm_gpuvm_bo; +@@ -283,6 +284,50 @@ struct drm_gpuvm { + * @r_obj: Resv GEM object; representing the GPUVM's common &dma_resv. + */ + struct drm_gem_object *r_obj; ++ ++ /** ++ * @extobj: structure holding the extobj list ++ */ ++ struct { ++ /** ++ * @list: &list_head storing &drm_gpuvm_bos serving as ++ * external object ++ */ ++ struct list_head list; ++ ++ /** ++ * @local_list: pointer to the local list temporarily storing ++ * entries from the external object list ++ */ ++ struct list_head *local_list; ++ ++ /** ++ * @lock: spinlock to protect the extobj list ++ */ ++ spinlock_t lock; ++ } extobj; ++ ++ /** ++ * @evict: structure holding the evict list and evict list lock ++ */ ++ struct { ++ /** ++ * @list: &list_head storing &drm_gpuvm_bos currently being ++ * evicted ++ */ ++ struct list_head list; ++ ++ /** ++ * @local_list: pointer to the local list temporarily storing ++ * entries from the evicted object list ++ */ ++ struct list_head *local_list; ++ ++ /** ++ * @lock: spinlock to protect the evict list ++ */ ++ spinlock_t lock; ++ } evict; + }; + + void drm_gpuvm_init(struct drm_gpuvm *gpuvm, const char *name, +@@ -359,6 +404,22 @@ drm_gpuvm_resv_protected(struct drm_gpuvm *gpuvm) + #define drm_gpuvm_resv_assert_held(gpuvm__) \ + dma_resv_assert_held(drm_gpuvm_resv(gpuvm__)) + ++/** ++ * drm_gpuvm_is_extobj() - indicates whether the given &drm_gem_object is an ++ * external object ++ * @gpuvm: the &drm_gpuvm to check ++ * @obj: the &drm_gem_object to check ++ * ++ * Returns: true if the &drm_gem_object &dma_resv differs from the ++ * &drm_gpuvms &dma_resv, false otherwise ++ */ ++static inline bool ++drm_gpuvm_is_extobj(struct drm_gpuvm *gpuvm, ++ struct drm_gem_object *obj) ++{ ++ return obj && obj->resv != drm_gpuvm_resv(gpuvm); ++} ++ + static inline struct drm_gpuva * + __drm_gpuva_next(struct drm_gpuva *va) + { +@@ -437,6 +498,144 @@ __drm_gpuva_next(struct drm_gpuva *va) + #define drm_gpuvm_for_each_va_safe(va__, next__, gpuvm__) \ + list_for_each_entry_safe(va__, next__, &(gpuvm__)->rb.list, rb.entry) + ++/** ++ * struct drm_gpuvm_exec - &drm_gpuvm abstraction of &drm_exec ++ * ++ * This structure should be created on the stack as &drm_exec should be. ++ * ++ * Optionally, @extra can be set in order to lock additional &drm_gem_objects. ++ */ ++struct drm_gpuvm_exec { ++ /** ++ * @exec: the &drm_exec structure ++ */ ++ struct drm_exec exec; ++ ++ /** ++ * @flags: the flags for the struct drm_exec ++ */ ++ uint32_t flags; ++ ++ /** ++ * @vm: the &drm_gpuvm to lock its DMA reservations ++ */ ++ struct drm_gpuvm *vm; ++ ++ /** ++ * @num_fences: the number of fences to reserve for the &dma_resv of the ++ * locked &drm_gem_objects ++ */ ++ unsigned int num_fences; ++ ++ /** ++ * @extra: Callback and corresponding private data for the driver to ++ * lock arbitrary additional &drm_gem_objects. ++ */ ++ struct { ++ /** ++ * @fn: The driver callback to lock additional &drm_gem_objects. ++ */ ++ int (*fn)(struct drm_gpuvm_exec *vm_exec); ++ ++ /** ++ * @priv: driver private data for the @fn callback ++ */ ++ void *priv; ++ } extra; ++}; ++ ++/** ++ * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv ++ * @gpuvm: the &drm_gpuvm ++ * @exec: the &drm_exec context ++ * @num_fences: the amount of &dma_fences to reserve ++ * ++ * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object. ++ * ++ * Using this function directly, it is the drivers responsibility to call ++ * drm_exec_init() and drm_exec_fini() accordingly. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static inline int ++drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences) ++{ ++ return drm_exec_prepare_obj(exec, gpuvm->r_obj, num_fences); ++} ++ ++int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences); ++ ++int drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ u64 addr, u64 range, ++ unsigned int num_fences); ++ ++int drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec); ++ ++int drm_gpuvm_exec_lock_array(struct drm_gpuvm_exec *vm_exec, ++ struct drm_gem_object **objs, ++ unsigned int num_objs); ++ ++int drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, ++ u64 addr, u64 range); ++ ++/** ++ * drm_gpuvm_exec_unlock() - lock all dma-resv of all assoiciated BOs ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * ++ * Releases all dma-resv locks of all &drm_gem_objects previously acquired ++ * through drm_gpuvm_exec_lock() or its variants. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++static inline void ++drm_gpuvm_exec_unlock(struct drm_gpuvm_exec *vm_exec) ++{ ++ drm_exec_fini(&vm_exec->exec); ++} ++ ++int drm_gpuvm_validate(struct drm_gpuvm *gpuvm, struct drm_exec *exec); ++void drm_gpuvm_resv_add_fence(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ struct dma_fence *fence, ++ enum dma_resv_usage private_usage, ++ enum dma_resv_usage extobj_usage); ++ ++/** ++ * drm_gpuvm_exec_resv_add_fence() ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * @fence: fence to add ++ * @private_usage: private dma-resv usage ++ * @extobj_usage: extobj dma-resv usage ++ * ++ * See drm_gpuvm_resv_add_fence(). ++ */ ++static inline void ++drm_gpuvm_exec_resv_add_fence(struct drm_gpuvm_exec *vm_exec, ++ struct dma_fence *fence, ++ enum dma_resv_usage private_usage, ++ enum dma_resv_usage extobj_usage) ++{ ++ drm_gpuvm_resv_add_fence(vm_exec->vm, &vm_exec->exec, fence, ++ private_usage, extobj_usage); ++} ++ ++/** ++ * drm_gpuvm_exec_validate() ++ * @vm_exec: the &drm_gpuvm_exec wrapper ++ * ++ * See drm_gpuvm_validate(). ++ */ ++static inline int ++drm_gpuvm_exec_validate(struct drm_gpuvm_exec *vm_exec) ++{ ++ return drm_gpuvm_validate(vm_exec->vm, &vm_exec->exec); ++} ++ + /** + * struct drm_gpuvm_bo - structure representing a &drm_gpuvm and + * &drm_gem_object combination +@@ -466,6 +665,12 @@ struct drm_gpuvm_bo { + */ + struct drm_gem_object *obj; + ++ /** ++ * @evicted: Indicates whether the &drm_gem_object is evicted; field ++ * protected by the &drm_gem_object's dma-resv lock. ++ */ ++ bool evicted; ++ + /** + * @kref: The reference count for this &drm_gpuvm_bo. + */ +@@ -493,6 +698,18 @@ struct drm_gpuvm_bo { + * gpuva list. + */ + struct list_head gem; ++ ++ /** ++ * @evict: List entry to attach to the &drm_gpuvms ++ * extobj list. ++ */ ++ struct list_head extobj; ++ ++ /** ++ * @evict: List entry to attach to the &drm_gpuvms evict ++ * list. ++ */ ++ struct list_head evict; + } entry; + } list; + }; +@@ -527,6 +744,27 @@ struct drm_gpuvm_bo * + drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, + struct drm_gem_object *obj); + ++void drm_gpuvm_bo_evict(struct drm_gpuvm_bo *vm_bo, bool evict); ++ ++/** ++ * drm_gpuvm_bo_gem_evict() ++ * @obj: the &drm_gem_object ++ * @evict: indicates whether @obj is evicted ++ * ++ * See drm_gpuvm_bo_evict(). ++ */ ++static inline void ++drm_gpuvm_bo_gem_evict(struct drm_gem_object *obj, bool evict) ++{ ++ struct drm_gpuvm_bo *vm_bo; ++ ++ drm_gem_gpuva_assert_lock_held(obj); ++ drm_gem_for_each_gpuvm_bo(vm_bo, obj) ++ drm_gpuvm_bo_evict(vm_bo, evict); ++} ++ ++void drm_gpuvm_bo_extobj_add(struct drm_gpuvm_bo *vm_bo); ++ + /** + * drm_gpuvm_bo_for_each_va() - iterator to walk over a list of &drm_gpuva + * @va__: &drm_gpuva structure to assign to in each iteration step +@@ -901,6 +1139,18 @@ struct drm_gpuvm_ops { + */ + void (*vm_bo_free)(struct drm_gpuvm_bo *vm_bo); + ++ /** ++ * @vm_bo_validate: called from drm_gpuvm_validate() ++ * ++ * Drivers receive this callback for every evicted &drm_gem_object being ++ * mapped in the corresponding &drm_gpuvm. ++ * ++ * Typically, drivers would call their driver specific variant of ++ * ttm_bo_validate() from within this callback. ++ */ ++ int (*vm_bo_validate)(struct drm_gpuvm_bo *vm_bo, ++ struct drm_exec *exec); ++ + /** + * @sm_step_map: called from &drm_gpuvm_sm_map to finally insert the + * mapping once all previous steps were completed + +From f4d9314a9e087e97a467b06cce5bf588fbd4a0d7 Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Wed, 29 Nov 2023 23:08:00 +0100 +Subject: [PATCH 016/150] drm/gpuvm: fall back to drm_exec_lock_obj() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Fall back to drm_exec_lock_obj() if num_fences is zero for the +drm_gpuvm_prepare_* function family. + +Otherwise dma_resv_reserve_fences() would actually allocate slots even +though num_fences is zero. + +Cc: Christian König +Acked-by: Donald Robson +Signed-off-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231129220835.297885-2-dakr@redhat.com +--- + drivers/gpu/drm/drm_gpuvm.c | 43 ++++++++++++++++++++++++++++++++----- + include/drm/drm_gpuvm.h | 23 +++----------------- + 2 files changed, 41 insertions(+), 25 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 92815da084ce04..12ab620a8cdb72 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -1085,6 +1085,37 @@ drm_gpuvm_put(struct drm_gpuvm *gpuvm) + } + EXPORT_SYMBOL_GPL(drm_gpuvm_put); + ++static int ++exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj, ++ unsigned int num_fences) ++{ ++ return num_fences ? drm_exec_prepare_obj(exec, obj, num_fences) : ++ drm_exec_lock_obj(exec, obj); ++} ++ ++/** ++ * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv ++ * @gpuvm: the &drm_gpuvm ++ * @exec: the &drm_exec context ++ * @num_fences: the amount of &dma_fences to reserve ++ * ++ * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object; if ++ * @num_fences is zero drm_exec_lock_obj() is called instead. ++ * ++ * Using this function directly, it is the drivers responsibility to call ++ * drm_exec_init() and drm_exec_fini() accordingly. ++ * ++ * Returns: 0 on success, negative error code on failure. ++ */ ++int ++drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences) ++{ ++ return exec_prepare_obj(exec, gpuvm->r_obj, num_fences); ++} ++EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_vm); ++ + static int + __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, + struct drm_exec *exec, +@@ -1095,7 +1126,7 @@ __drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, + int ret = 0; + + for_each_vm_bo_in_list(gpuvm, extobj, &extobjs, vm_bo) { +- ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences); ++ ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); + if (ret) + break; + } +@@ -1116,7 +1147,7 @@ drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, + + drm_gpuvm_resv_assert_held(gpuvm); + list_for_each_entry(vm_bo, &gpuvm->extobj.list, list.entry.extobj) { +- ret = drm_exec_prepare_obj(exec, vm_bo->obj, num_fences); ++ ret = exec_prepare_obj(exec, vm_bo->obj, num_fences); + if (ret) + break; + +@@ -1134,7 +1165,8 @@ drm_gpuvm_prepare_objects_locked(struct drm_gpuvm *gpuvm, + * @num_fences: the amount of &dma_fences to reserve + * + * Calls drm_exec_prepare_obj() for all &drm_gem_objects the given +- * &drm_gpuvm contains mappings of. ++ * &drm_gpuvm contains mappings of; if @num_fences is zero drm_exec_lock_obj() ++ * is called instead. + * + * Using this function directly, it is the drivers responsibility to call + * drm_exec_init() and drm_exec_fini() accordingly. +@@ -1171,7 +1203,8 @@ EXPORT_SYMBOL_GPL(drm_gpuvm_prepare_objects); + * @num_fences: the amount of &dma_fences to reserve + * + * Calls drm_exec_prepare_obj() for all &drm_gem_objects mapped between @addr +- * and @addr + @range. ++ * and @addr + @range; if @num_fences is zero drm_exec_lock_obj() is called ++ * instead. + * + * Returns: 0 on success, negative error code on failure. + */ +@@ -1186,7 +1219,7 @@ drm_gpuvm_prepare_range(struct drm_gpuvm *gpuvm, struct drm_exec *exec, + drm_gpuvm_for_each_va_range(va, gpuvm, addr, end) { + struct drm_gem_object *obj = va->gem.obj; + +- ret = drm_exec_prepare_obj(exec, obj, num_fences); ++ ret = exec_prepare_obj(exec, obj, num_fences); + if (ret) + return ret; + } +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 74baa9d5da21df..9aaacd2b21b09e 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -544,26 +544,9 @@ struct drm_gpuvm_exec { + } extra; + }; + +-/** +- * drm_gpuvm_prepare_vm() - prepare the GPUVMs common dma-resv +- * @gpuvm: the &drm_gpuvm +- * @exec: the &drm_exec context +- * @num_fences: the amount of &dma_fences to reserve +- * +- * Calls drm_exec_prepare_obj() for the GPUVMs dummy &drm_gem_object. +- * +- * Using this function directly, it is the drivers responsibility to call +- * drm_exec_init() and drm_exec_fini() accordingly. +- * +- * Returns: 0 on success, negative error code on failure. +- */ +-static inline int +-drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, +- struct drm_exec *exec, +- unsigned int num_fences) +-{ +- return drm_exec_prepare_obj(exec, gpuvm->r_obj, num_fences); +-} ++int drm_gpuvm_prepare_vm(struct drm_gpuvm *gpuvm, ++ struct drm_exec *exec, ++ unsigned int num_fences); + + int drm_gpuvm_prepare_objects(struct drm_gpuvm *gpuvm, + struct drm_exec *exec, + +From 6a1749557da60f7e7943843ee441dc97514be34e Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Mon, 4 Dec 2023 16:14:06 +0100 +Subject: [PATCH 017/150] drm/gpuvm: Let drm_gpuvm_bo_put() report when the + vm_bo object is destroyed + +Some users need to release resources attached to the vm_bo object when +it's destroyed. In Panthor's case, we need to release the pin ref so +BO pages can be returned to the system when all GPU mappings are gone. + +This could be done through a custom drm_gpuvm::vm_bo_free() hook, but +this has all sort of locking implications that would force us to expose +a drm_gem_shmem_unpin_locked() helper, not to mention the fact that +having a ::vm_bo_free() implementation without a ::vm_bo_alloc() one +seems odd. So let's keep things simple, and extend drm_gpuvm_bo_put() +to report when the object is destroyed. + +Signed-off-by: Boris Brezillon +Reviewed-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231204151406.1977285-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/drm_gpuvm.c | 8 ++++++-- + include/drm/drm_gpuvm.h | 2 +- + 2 files changed, 7 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 12ab620a8cdb72..383e6ad2791dc6 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -1535,14 +1535,18 @@ drm_gpuvm_bo_destroy(struct kref *kref) + * hold the dma-resv or driver specific GEM gpuva lock. + * + * This function may only be called from non-atomic context. ++ * ++ * Returns: true if vm_bo was destroyed, false otherwise. + */ +-void ++bool + drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo) + { + might_sleep(); + + if (vm_bo) +- kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy); ++ return !!kref_put(&vm_bo->kref, drm_gpuvm_bo_destroy); ++ ++ return false; + } + EXPORT_SYMBOL_GPL(drm_gpuvm_bo_put); + +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 9aaacd2b21b09e..7921eb322b83fe 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -721,7 +721,7 @@ drm_gpuvm_bo_get(struct drm_gpuvm_bo *vm_bo) + return vm_bo; + } + +-void drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); ++bool drm_gpuvm_bo_put(struct drm_gpuvm_bo *vm_bo); + + struct drm_gpuvm_bo * + drm_gpuvm_bo_find(struct drm_gpuvm *gpuvm, + +From 937d2c782e312fb82be73bcc8aa36342e47dbe28 Mon Sep 17 00:00:00 2001 +From: Rob Clark +Date: Mon, 20 Nov 2023 16:38:50 -0800 +Subject: [PATCH 018/150] [MOD] drm/exec: Pass in initial # of objects +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +HB: skipped +drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c +drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c +drivers/gpu/drm/amd/amdgpu/amdgpu_csa.c +drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c +drivers/gpu/drm/amd/amdgpu/amdgpu_mes.c +drivers/gpu/drm/amd/amdgpu/amdgpu_umsch_mm.c +drivers/gpu/drm/amd/amdkfd/kfd_svm.c +drivers/gpu/drm/imagination/pvr_job.c +drivers/gpu/drm/nouveau/nouveau_uvmm.c + +In cases where the # is known ahead of time, it is silly to do the table +resize dance. + +Signed-off-by: Rob Clark +Reviewed-by: Christian König +Patchwork: https://patchwork.freedesktop.org/patch/568338/ +--- + drivers/gpu/drm/drm_exec.c | 13 ++++++++++--- + drivers/gpu/drm/drm_gpuvm.c | 4 ++-- + include/drm/drm_exec.h | 2 +- + 3 files changed, 13 insertions(+), 6 deletions(-) + +diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c +index ff69cf0fb42aab..303013c23f3385 100644 +--- a/drivers/gpu/drm/drm_exec.c ++++ b/drivers/gpu/drm/drm_exec.c +@@ -69,16 +69,23 @@ static void drm_exec_unlock_all(struct drm_exec *exec) + * drm_exec_init - initialize a drm_exec object + * @exec: the drm_exec object to initialize + * @flags: controls locking behavior, see DRM_EXEC_* defines ++ * @nr: the initial # of objects + * + * Initialize the object and make sure that we can track locked objects. ++ * ++ * If nr is non-zero then it is used as the initial objects table size. ++ * In either case, the table will grow (be re-allocated) on demand. + */ +-void drm_exec_init(struct drm_exec *exec, uint32_t flags) ++void drm_exec_init(struct drm_exec *exec, uint32_t flags, unsigned nr) + { ++ if (!nr) ++ nr = PAGE_SIZE / sizeof(void *); ++ + exec->flags = flags; +- exec->objects = kmalloc(PAGE_SIZE, GFP_KERNEL); ++ exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL); + + /* If allocation here fails, just delay that till the first use */ +- exec->max_objects = exec->objects ? PAGE_SIZE / sizeof(void *) : 0; ++ exec->max_objects = exec->objects ? nr : 0; + exec->num_objects = 0; + exec->contended = DRM_EXEC_DUMMY; + exec->prelocked = NULL; +diff --git a/drivers/gpu/drm/drm_gpuvm.c b/drivers/gpu/drm/drm_gpuvm.c +index 383e6ad2791dc6..9387823c2dde4e 100644 +--- a/drivers/gpu/drm/drm_gpuvm.c ++++ b/drivers/gpu/drm/drm_gpuvm.c +@@ -1250,7 +1250,7 @@ drm_gpuvm_exec_lock(struct drm_gpuvm_exec *vm_exec) + unsigned int num_fences = vm_exec->num_fences; + int ret; + +- drm_exec_init(exec, vm_exec->flags); ++ drm_exec_init(exec, vm_exec->flags, 0); + + drm_exec_until_all_locked(exec) { + ret = drm_gpuvm_prepare_vm(gpuvm, exec, num_fences); +@@ -1341,7 +1341,7 @@ drm_gpuvm_exec_lock_range(struct drm_gpuvm_exec *vm_exec, + struct drm_exec *exec = &vm_exec->exec; + int ret; + +- drm_exec_init(exec, vm_exec->flags); ++ drm_exec_init(exec, vm_exec->flags, 0); + + drm_exec_until_all_locked(exec) { + ret = drm_gpuvm_prepare_range(gpuvm, exec, addr, range, +diff --git a/include/drm/drm_exec.h b/include/drm/drm_exec.h +index 73205afec16211..ae52623e43c8ab 100644 +--- a/include/drm/drm_exec.h ++++ b/include/drm/drm_exec.h +@@ -108,7 +108,7 @@ static inline bool drm_exec_is_contended(struct drm_exec *exec) + return !!exec->contended; + } + +-void drm_exec_init(struct drm_exec *exec, uint32_t flags); ++void drm_exec_init(struct drm_exec *exec, uint32_t flags, unsigned nr); + void drm_exec_fini(struct drm_exec *exec); + bool drm_exec_cleanup(struct drm_exec *exec); + int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj); + +From 0637782810329256aa6cf59bfd18aeb3966a949d Mon Sep 17 00:00:00 2001 +From: ChunyouTang +Date: Sat, 19 Nov 2022 14:41:31 +0800 +Subject: [PATCH 019/150] drm/gem-shmem: When drm_gem_object_init failed, + should release object + +when goto err_free, the object had init, so it should be release when fail. + +Signed-off-by: ChunyouTang +Signed-off-by: Thomas Zimmermann +Link: https://patchwork.freedesktop.org/patch/msgid/20221119064131.364-1-tangchunyou@163.com +--- + drivers/gpu/drm/drm_gem.c | 19 ++++++++++++++++--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 4 +++- + include/drm/drm_gem.h | 1 + + 3 files changed, 20 insertions(+), 4 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c +index fab07f0a46398b..4e58e500c2a5fd 100644 +--- a/drivers/gpu/drm/drm_gem.c ++++ b/drivers/gpu/drm/drm_gem.c +@@ -172,6 +172,20 @@ void drm_gem_private_object_init(struct drm_device *dev, + } + EXPORT_SYMBOL(drm_gem_private_object_init); + ++/** ++ * drm_gem_private_object_fini - Finalize a failed drm_gem_object ++ * @obj: drm_gem_object ++ * ++ * Uninitialize an already allocated GEM object when it initialized failed ++ */ ++void drm_gem_private_object_fini(struct drm_gem_object *obj) ++{ ++ WARN_ON(obj->dma_buf); ++ ++ dma_resv_fini(&obj->_resv); ++} ++EXPORT_SYMBOL(drm_gem_private_object_fini); ++ + /** + * drm_gem_object_handle_free - release resources bound to userspace handles + * @obj: GEM object to clean up. +@@ -933,12 +947,11 @@ drm_gem_release(struct drm_device *dev, struct drm_file *file_private) + void + drm_gem_object_release(struct drm_gem_object *obj) + { +- WARN_ON(obj->dma_buf); +- + if (obj->filp) + fput(obj->filp); + +- dma_resv_fini(&obj->_resv); ++ drm_gem_private_object_fini(obj); ++ + drm_gem_free_mmap_offset(obj); + drm_gem_lru_remove(obj); + } +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index e33f06bb66eb43..63715caeea8c53 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -79,8 +79,10 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private) + } else { + ret = drm_gem_object_init(dev, obj, size); + } +- if (ret) ++ if (ret) { ++ drm_gem_private_object_fini(obj); + goto err_free; ++ } + + ret = drm_gem_create_mmap_offset(obj); + if (ret) +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 2b27a656b2679d..58b69fa1cf5dbf 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -423,6 +423,7 @@ int drm_gem_object_init(struct drm_device *dev, + struct drm_gem_object *obj, size_t size); + void drm_gem_private_object_init(struct drm_device *dev, + struct drm_gem_object *obj, size_t size); ++void drm_gem_private_object_fini(struct drm_gem_object *obj); + void drm_gem_vm_open(struct vm_area_struct *vma); + void drm_gem_vm_close(struct vm_area_struct *vma); + int drm_gem_mmap_obj(struct drm_gem_object *obj, unsigned long obj_size, + +From 683499d7e1b33d52ceb3024c7017055d4893248c Mon Sep 17 00:00:00 2001 +From: Siddh Raman Pant +Date: Mon, 9 Jan 2023 18:30:33 +0530 +Subject: [PATCH 020/150] drm: Remove usage of deprecated DRM_DEBUG_PRIME + +drm_print.h says DRM_DEBUG_PRIME is deprecated in favor of +drm_dbg_prime(). + +Signed-off-by: Siddh Raman Pant +Reviewed-by: Simon Ser +Signed-off-by: Simon Ser +Link: https://patchwork.freedesktop.org/patch/msgid/cd663b1bc42189e55898cddecdb3b73c591b341a.1673269059.git.code@siddh.me +--- + drivers/gpu/drm/drm_gem_dma_helper.c | 4 ++-- + drivers/gpu/drm/drm_gem_shmem_helper.c | 2 +- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_dma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c +index f6901ff97bbb5b..2daad6c6bfb280 100644 +--- a/drivers/gpu/drm/drm_gem_dma_helper.c ++++ b/drivers/gpu/drm/drm_gem_dma_helper.c +@@ -477,8 +477,8 @@ drm_gem_dma_prime_import_sg_table(struct drm_device *dev, + dma_obj->dma_addr = sg_dma_address(sgt->sgl); + dma_obj->sgt = sgt; + +- DRM_DEBUG_PRIME("dma_addr = %pad, size = %zu\n", &dma_obj->dma_addr, +- attach->dmabuf->size); ++ drm_dbg_prime(dev, "dma_addr = %pad, size = %zu\n", &dma_obj->dma_addr, ++ attach->dmabuf->size); + + return &dma_obj->base; + } +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 63715caeea8c53..71231321935e37 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -789,7 +789,7 @@ drm_gem_shmem_prime_import_sg_table(struct drm_device *dev, + + shmem->sgt = sgt; + +- DRM_DEBUG_PRIME("size = %zu\n", size); ++ drm_dbg_prime(dev, "size = %zu\n", size); + + return &shmem->base; + } + +From 82bc06a6529cd9c8ec6a27d0117b76cfa6b561f2 Mon Sep 17 00:00:00 2001 +From: Rob Clark +Date: Mon, 23 Jan 2023 07:48:31 -0800 +Subject: [PATCH 021/150] drm/shmem: Cleanup drm_gem_shmem_create_with_handle() + +Once we create the handle, the handle owns the reference. Currently +nothing was doing anything with the shmem ptr after the handle was +created, but let's change drm_gem_shmem_create_with_handle() to not +return the pointer, so-as to not encourage problematic use of this +function in the future. As a bonus, it makes the code a bit cleaner. + +Signed-off-by: Rob Clark +Reviewed-by: Steven Price +Signed-off-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20230123154831.3191821-1-robdclark@gmail.com +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 13 ++++--------- + 1 file changed, 4 insertions(+), 9 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 71231321935e37..1825779a0289fb 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -415,7 +415,7 @@ void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, + } + EXPORT_SYMBOL(drm_gem_shmem_vunmap); + +-static struct drm_gem_shmem_object * ++static int + drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + struct drm_device *dev, size_t size, + uint32_t *handle) +@@ -425,7 +425,7 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + + shmem = drm_gem_shmem_create(dev, size); + if (IS_ERR(shmem)) +- return shmem; ++ return PTR_ERR(shmem); + + /* + * Allocate an id of idr table where the obj is registered +@@ -434,10 +434,8 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + ret = drm_gem_handle_create(file_priv, &shmem->base, handle); + /* drop reference from allocate - handle holds it now. */ + drm_gem_object_put(&shmem->base); +- if (ret) +- return ERR_PTR(ret); + +- return shmem; ++ return ret; + } + + /* Update madvise status, returns true if not purged, else +@@ -520,7 +518,6 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, + struct drm_mode_create_dumb *args) + { + u32 min_pitch = DIV_ROUND_UP(args->width * args->bpp, 8); +- struct drm_gem_shmem_object *shmem; + + if (!args->pitch || !args->size) { + args->pitch = min_pitch; +@@ -533,9 +530,7 @@ int drm_gem_shmem_dumb_create(struct drm_file *file, struct drm_device *dev, + args->size = PAGE_ALIGN(args->pitch * args->height); + } + +- shmem = drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle); +- +- return PTR_ERR_OR_ZERO(shmem); ++ return drm_gem_shmem_create_with_handle(file, dev, args->size, &args->handle); + } + EXPORT_SYMBOL_GPL(drm_gem_shmem_dumb_create); + + +From f7798e2c371cd6f48a9518748d711766e78a7e89 Mon Sep 17 00:00:00 2001 +From: Asahi Lina +Date: Sun, 5 Feb 2023 21:51:24 +0900 +Subject: [PATCH 022/150] drm/shmem-helper: Fix locking for + drm_gem_shmem_get_pages_sgt() + +Other functions touching shmem->sgt take the pages lock, so do that here +too. drm_gem_shmem_get_pages() & co take the same lock, so move to the +_locked() variants to avoid recursive locking. + +Discovered while auditing locking to write the Rust abstractions. + +Fixes: 2194a63a818d ("drm: Add library for shmem backed GEM objects") +Fixes: 4fa3d66f132b ("drm/shmem: Do dma_unmap_sg before purging pages") +Signed-off-by: Asahi Lina +Reviewed-by: Javier Martinez Canillas +Signed-off-by: Javier Martinez Canillas +Link: https://patchwork.freedesktop.org/patch/msgid/20230205125124.2260-1-lina@asahilina.net +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 1825779a0289fb..2c647f4889cda9 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -753,7 +753,7 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) + + return sgt; + } +-EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages_sgt); ++EXPORT_SYMBOL(drm_gem_shmem_get_pages_sgt); + + /** + * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from + +From 1537c88e2730ff87e4643764f79b3612c9220a13 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Wed, 16 Nov 2022 23:56:33 +0300 +Subject: [PATCH 023/150] drm/shmem-helper: Switch to use drm_* debug helpers + +Ease debugging of a multi-GPU system by using drm_WARN_*() and +drm_dbg_kms() helpers that print out DRM device name corresponding +to shmem GEM. + +Reviewed-by: Thomas Zimmermann +Suggested-by: Thomas Zimmermann +Signed-off-by: Dmitry Osipenko +Link: https://lore.kernel.org/all/20230108210445.3948344-6-dmitry.osipenko@collabora.com/ +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 38 +++++++++++++++----------- + 1 file changed, 22 insertions(+), 16 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 2c647f4889cda9..a822a5fc512b74 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -141,7 +141,7 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- WARN_ON(shmem->vmap_use_count); ++ drm_WARN_ON(obj->dev, shmem->vmap_use_count); + + if (obj->import_attach) { + drm_prime_gem_destroy(obj, shmem->sgt); +@@ -156,7 +156,7 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + drm_gem_shmem_put_pages(shmem); + } + +- WARN_ON(shmem->pages_use_count); ++ drm_WARN_ON(obj->dev, shmem->pages_use_count); + + drm_gem_object_release(obj); + mutex_destroy(&shmem->pages_lock); +@@ -175,7 +175,8 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) + + pages = drm_gem_get_pages(obj); + if (IS_ERR(pages)) { +- DRM_DEBUG_KMS("Failed to get pages (%ld)\n", PTR_ERR(pages)); ++ drm_dbg_kms(obj->dev, "Failed to get pages (%ld)\n", ++ PTR_ERR(pages)); + shmem->pages_use_count = 0; + return PTR_ERR(pages); + } +@@ -207,9 +208,10 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) + */ + int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) + { ++ struct drm_gem_object *obj = &shmem->base; + int ret; + +- WARN_ON(shmem->base.import_attach); ++ drm_WARN_ON(obj->dev, obj->import_attach); + + ret = mutex_lock_interruptible(&shmem->pages_lock); + if (ret) +@@ -225,7 +227,7 @@ static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- if (WARN_ON_ONCE(!shmem->pages_use_count)) ++ if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + return; + + if (--shmem->pages_use_count > 0) +@@ -268,7 +270,9 @@ EXPORT_SYMBOL(drm_gem_shmem_put_pages); + */ + int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem) + { +- WARN_ON(shmem->base.import_attach); ++ struct drm_gem_object *obj = &shmem->base; ++ ++ drm_WARN_ON(obj->dev, obj->import_attach); + + return drm_gem_shmem_get_pages(shmem); + } +@@ -283,7 +287,9 @@ EXPORT_SYMBOL(drm_gem_shmem_pin); + */ + void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem) + { +- WARN_ON(shmem->base.import_attach); ++ struct drm_gem_object *obj = &shmem->base; ++ ++ drm_WARN_ON(obj->dev, obj->import_attach); + + drm_gem_shmem_put_pages(shmem); + } +@@ -303,7 +309,7 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + if (obj->import_attach) { + ret = dma_buf_vmap(obj->import_attach->dmabuf, map); + if (!ret) { +- if (WARN_ON(map->is_iomem)) { ++ if (drm_WARN_ON(obj->dev, map->is_iomem)) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); + ret = -EIO; + goto err_put_pages; +@@ -328,7 +334,7 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + } + + if (ret) { +- DRM_DEBUG_KMS("Failed to vmap pages, error %d\n", ret); ++ drm_dbg_kms(obj->dev, "Failed to vmap pages, error %d\n", ret); + goto err_put_pages; + } + +@@ -378,7 +384,7 @@ static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, + { + struct drm_gem_object *obj = &shmem->base; + +- if (WARN_ON_ONCE(!shmem->vmap_use_count)) ++ if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) + return; + + if (--shmem->vmap_use_count > 0) +@@ -461,7 +467,7 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + +- WARN_ON(!drm_gem_shmem_is_purgeable(shmem)); ++ drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); + sg_free_table(shmem->sgt); +@@ -550,7 +556,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + mutex_lock(&shmem->pages_lock); + + if (page_offset >= num_pages || +- WARN_ON_ONCE(!shmem->pages) || ++ drm_WARN_ON_ONCE(obj->dev, !shmem->pages) || + shmem->madv < 0) { + ret = VM_FAULT_SIGBUS; + } else { +@@ -569,7 +575,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + +- WARN_ON(shmem->base.import_attach); ++ drm_WARN_ON(obj->dev, obj->import_attach); + + mutex_lock(&shmem->pages_lock); + +@@ -578,7 +584,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + * mmap'd, vm_open() just grabs an additional reference for the new + * mm the vma is getting copied into (ie. on fork()). + */ +- if (!WARN_ON_ONCE(!shmem->pages_use_count)) ++ if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + shmem->pages_use_count++; + + mutex_unlock(&shmem->pages_lock); +@@ -681,7 +687,7 @@ struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- WARN_ON(shmem->base.import_attach); ++ drm_WARN_ON(obj->dev, obj->import_attach); + + return drm_prime_pages_to_sg(obj->dev, shmem->pages, obj->size >> PAGE_SHIFT); + } +@@ -696,7 +702,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + if (shmem->sgt) + return shmem->sgt; + +- WARN_ON(obj->import_attach); ++ drm_WARN_ON(obj->dev, obj->import_attach); + + ret = drm_gem_shmem_get_pages_locked(shmem); + if (ret) + +From a78704e8602da3cc89b15d8e5e2b3ba25e2aa6e4 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Thu, 5 May 2022 02:06:00 +0300 +Subject: [PATCH 024/150] drm/shmem-helper: Don't use vmap_use_count for + dma-bufs + +DMA-buf core has its own refcounting of vmaps, use it instead of drm-shmem +counting. This change prepares drm-shmem for addition of memory shrinker +support where drm-shmem will use a single dma-buf reservation lock for +all operations performed over dma-bufs. + +Reviewed-by: Thomas Zimmermann +Signed-off-by: Dmitry Osipenko +Link: https://lore.kernel.org/all/20230108210445.3948344-7-dmitry.osipenko@collabora.com/ +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 29 +++++++++++++------------- + 1 file changed, 15 insertions(+), 14 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index a822a5fc512b74..9ce84fa8077043 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -301,24 +301,22 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + struct drm_gem_object *obj = &shmem->base; + int ret = 0; + +- if (shmem->vmap_use_count++ > 0) { +- iosys_map_set_vaddr(map, shmem->vaddr); +- return 0; +- } +- + if (obj->import_attach) { + ret = dma_buf_vmap(obj->import_attach->dmabuf, map); + if (!ret) { + if (drm_WARN_ON(obj->dev, map->is_iomem)) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); +- ret = -EIO; +- goto err_put_pages; ++ return -EIO; + } +- shmem->vaddr = map->vaddr; + } + } else { + pgprot_t prot = PAGE_KERNEL; + ++ if (shmem->vmap_use_count++ > 0) { ++ iosys_map_set_vaddr(map, shmem->vaddr); ++ return 0; ++ } ++ + ret = drm_gem_shmem_get_pages(shmem); + if (ret) + goto err_zero_use; +@@ -384,15 +382,15 @@ static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, + { + struct drm_gem_object *obj = &shmem->base; + +- if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) +- return; +- +- if (--shmem->vmap_use_count > 0) +- return; +- + if (obj->import_attach) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); + } else { ++ if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) ++ return; ++ ++ if (--shmem->vmap_use_count > 0) ++ return; ++ + vunmap(shmem->vaddr); + drm_gem_shmem_put_pages(shmem); + } +@@ -663,6 +661,9 @@ EXPORT_SYMBOL_GPL(drm_gem_shmem_mmap); + void drm_gem_shmem_print_info(const struct drm_gem_shmem_object *shmem, + struct drm_printer *p, unsigned int indent) + { ++ if (shmem->base.import_attach) ++ return; ++ + drm_printf_indent(p, indent, "pages_use_count=%u\n", shmem->pages_use_count); + drm_printf_indent(p, indent, "vmap_use_count=%u\n", shmem->vmap_use_count); + drm_printf_indent(p, indent, "vaddr=%p\n", shmem->vaddr); + +From 241be751ed244ae4e83a96c28c14f2b81b2bb338 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Mon, 21 Mar 2022 03:35:41 +0300 +Subject: [PATCH 025/150] drm/shmem-helper: Switch to reservation lock + +Replace all drm-shmem locks with a GEM reservation lock. This makes locks +consistent with dma-buf locking convention where importers are responsible +for holding reservation lock for all operations performed over dma-bufs, +preventing deadlock between dma-buf importers and exporters. + +Suggested-by: Daniel Vetter +Acked-by: Thomas Zimmermann +Signed-off-by: Dmitry Osipenko +Link: https://lore.kernel.org/all/20230108210445.3948344-8-dmitry.osipenko@collabora.com/ +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 185 +++++++----------- + drivers/gpu/drm/lima/lima_gem.c | 8 +- + drivers/gpu/drm/panfrost/panfrost_drv.c | 7 +- + .../gpu/drm/panfrost/panfrost_gem_shrinker.c | 6 +- + drivers/gpu/drm/panfrost/panfrost_mmu.c | 19 +- + include/drm/drm_gem_shmem_helper.h | 14 +- + 6 files changed, 91 insertions(+), 148 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 9ce84fa8077043..17d08f6e11acd9 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -88,8 +88,6 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private) + if (ret) + goto err_release; + +- mutex_init(&shmem->pages_lock); +- mutex_init(&shmem->vmap_lock); + INIT_LIST_HEAD(&shmem->madv_list); + + if (!private) { +@@ -141,11 +139,13 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- drm_WARN_ON(obj->dev, shmem->vmap_use_count); +- + if (obj->import_attach) { + drm_prime_gem_destroy(obj, shmem->sgt); + } else { ++ dma_resv_lock(shmem->base.resv, NULL); ++ ++ drm_WARN_ON(obj->dev, shmem->vmap_use_count); ++ + if (shmem->sgt) { + dma_unmap_sgtable(obj->dev->dev, shmem->sgt, + DMA_BIDIRECTIONAL, 0); +@@ -154,18 +154,18 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + } + if (shmem->pages) + drm_gem_shmem_put_pages(shmem); +- } + +- drm_WARN_ON(obj->dev, shmem->pages_use_count); ++ drm_WARN_ON(obj->dev, shmem->pages_use_count); ++ ++ dma_resv_unlock(shmem->base.resv); ++ } + + drm_gem_object_release(obj); +- mutex_destroy(&shmem->pages_lock); +- mutex_destroy(&shmem->vmap_lock); + kfree(shmem); + } + EXPORT_SYMBOL_GPL(drm_gem_shmem_free); + +-static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) ++static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct page **pages; +@@ -197,35 +197,16 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) + } + + /* +- * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object ++ * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object + * @shmem: shmem GEM object + * +- * This function makes sure that backing pages exists for the shmem GEM object +- * and increases the use count. +- * +- * Returns: +- * 0 on success or a negative error code on failure. ++ * This function decreases the use count and puts the backing pages when use drops to zero. + */ +-int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) ++void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; +- int ret; + +- drm_WARN_ON(obj->dev, obj->import_attach); +- +- ret = mutex_lock_interruptible(&shmem->pages_lock); +- if (ret) +- return ret; +- ret = drm_gem_shmem_get_pages_locked(shmem); +- mutex_unlock(&shmem->pages_lock); +- +- return ret; +-} +-EXPORT_SYMBOL(drm_gem_shmem_get_pages); +- +-static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) +-{ +- struct drm_gem_object *obj = &shmem->base; ++ dma_resv_assert_held(shmem->base.resv); + + if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + return; +@@ -243,19 +224,6 @@ static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) + shmem->pages_mark_accessed_on_put); + shmem->pages = NULL; + } +- +-/* +- * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object +- * @shmem: shmem GEM object +- * +- * This function decreases the use count and puts the backing pages when use drops to zero. +- */ +-void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) +-{ +- mutex_lock(&shmem->pages_lock); +- drm_gem_shmem_put_pages_locked(shmem); +- mutex_unlock(&shmem->pages_lock); +-} + EXPORT_SYMBOL(drm_gem_shmem_put_pages); + + /** +@@ -272,6 +240,8 @@ int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + ++ dma_resv_assert_held(shmem->base.resv); ++ + drm_WARN_ON(obj->dev, obj->import_attach); + + return drm_gem_shmem_get_pages(shmem); +@@ -289,14 +259,31 @@ void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + ++ dma_resv_assert_held(shmem->base.resv); ++ + drm_WARN_ON(obj->dev, obj->import_attach); + + drm_gem_shmem_put_pages(shmem); + } + EXPORT_SYMBOL(drm_gem_shmem_unpin); + +-static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++/* ++ * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object ++ * @shmem: shmem GEM object ++ * @map: Returns the kernel virtual address of the SHMEM GEM object's backing ++ * store. ++ * ++ * This function makes sure that a contiguous kernel virtual address mapping ++ * exists for the buffer backing the shmem GEM object. It hides the differences ++ * between dma-buf imported and natively allocated objects. ++ * ++ * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). ++ * ++ * Returns: ++ * 0 on success or a negative error code on failure. ++ */ ++int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + int ret = 0; +@@ -312,6 +299,8 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + } else { + pgprot_t prot = PAGE_KERNEL; + ++ dma_resv_assert_held(shmem->base.resv); ++ + if (shmem->vmap_use_count++ > 0) { + iosys_map_set_vaddr(map, shmem->vaddr); + return 0; +@@ -346,45 +335,30 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + + return ret; + } ++EXPORT_SYMBOL(drm_gem_shmem_vmap); + + /* +- * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object ++ * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object + * @shmem: shmem GEM object +- * @map: Returns the kernel virtual address of the SHMEM GEM object's backing +- * store. +- * +- * This function makes sure that a contiguous kernel virtual address mapping +- * exists for the buffer backing the shmem GEM object. It hides the differences +- * between dma-buf imported and natively allocated objects. ++ * @map: Kernel virtual address where the SHMEM GEM object was mapped + * +- * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). ++ * This function cleans up a kernel virtual address mapping acquired by ++ * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to ++ * zero. + * +- * Returns: +- * 0 on success or a negative error code on failure. ++ * This function hides the differences between dma-buf imported and natively ++ * allocated objects. + */ +-int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) +-{ +- int ret; +- +- ret = mutex_lock_interruptible(&shmem->vmap_lock); +- if (ret) +- return ret; +- ret = drm_gem_shmem_vmap_locked(shmem, map); +- mutex_unlock(&shmem->vmap_lock); +- +- return ret; +-} +-EXPORT_SYMBOL(drm_gem_shmem_vmap); +- +-static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + + if (obj->import_attach) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); + } else { ++ dma_resv_assert_held(shmem->base.resv); ++ + if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) + return; + +@@ -397,26 +371,6 @@ static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, + + shmem->vaddr = NULL; + } +- +-/* +- * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object +- * @shmem: shmem GEM object +- * @map: Kernel virtual address where the SHMEM GEM object was mapped +- * +- * This function cleans up a kernel virtual address mapping acquired by +- * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to +- * zero. +- * +- * This function hides the differences between dma-buf imported and natively +- * allocated objects. +- */ +-void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) +-{ +- mutex_lock(&shmem->vmap_lock); +- drm_gem_shmem_vunmap_locked(shmem, map); +- mutex_unlock(&shmem->vmap_lock); +-} + EXPORT_SYMBOL(drm_gem_shmem_vunmap); + + static int +@@ -447,24 +401,24 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + */ + int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) + { +- mutex_lock(&shmem->pages_lock); ++ dma_resv_assert_held(shmem->base.resv); + + if (shmem->madv >= 0) + shmem->madv = madv; + + madv = shmem->madv; + +- mutex_unlock(&shmem->pages_lock); +- + return (madv >= 0); + } + EXPORT_SYMBOL(drm_gem_shmem_madvise); + +-void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) ++void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + ++ dma_resv_assert_held(shmem->base.resv); ++ + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); +@@ -472,7 +426,7 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + kfree(shmem->sgt); + shmem->sgt = NULL; + +- drm_gem_shmem_put_pages_locked(shmem); ++ drm_gem_shmem_put_pages(shmem); + + shmem->madv = -1; + +@@ -488,17 +442,6 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + + invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 0, (loff_t)-1); + } +-EXPORT_SYMBOL(drm_gem_shmem_purge_locked); +- +-bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) +-{ +- if (!mutex_trylock(&shmem->pages_lock)) +- return false; +- drm_gem_shmem_purge_locked(shmem); +- mutex_unlock(&shmem->pages_lock); +- +- return true; +-} + EXPORT_SYMBOL(drm_gem_shmem_purge); + + /** +@@ -551,7 +494,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + /* We don't use vmf->pgoff since that has the fake offset */ + page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + +- mutex_lock(&shmem->pages_lock); ++ dma_resv_lock(shmem->base.resv, NULL); + + if (page_offset >= num_pages || + drm_WARN_ON_ONCE(obj->dev, !shmem->pages) || +@@ -563,7 +506,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page)); + } + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + return ret; + } +@@ -575,7 +518,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + + drm_WARN_ON(obj->dev, obj->import_attach); + +- mutex_lock(&shmem->pages_lock); ++ dma_resv_lock(shmem->base.resv, NULL); + + /* + * We should have already pinned the pages when the buffer was first +@@ -585,7 +528,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + shmem->pages_use_count++; + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + drm_gem_vm_open(vma); + } +@@ -595,7 +538,10 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + ++ dma_resv_lock(shmem->base.resv, NULL); + drm_gem_shmem_put_pages(shmem); ++ dma_resv_unlock(shmem->base.resv); ++ + drm_gem_vm_close(vma); + } + +@@ -639,7 +585,10 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct + return ret; + } + ++ dma_resv_lock(shmem->base.resv, NULL); + ret = drm_gem_shmem_get_pages(shmem); ++ dma_resv_unlock(shmem->base.resv); ++ + if (ret) + return ret; + +@@ -705,7 +654,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + + drm_WARN_ON(obj->dev, obj->import_attach); + +- ret = drm_gem_shmem_get_pages_locked(shmem); ++ ret = drm_gem_shmem_get_pages(shmem); + if (ret) + return ERR_PTR(ret); + +@@ -727,7 +676,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + sg_free_table(sgt); + kfree(sgt); + err_put_pages: +- drm_gem_shmem_put_pages_locked(shmem); ++ drm_gem_shmem_put_pages(shmem); + return ERR_PTR(ret); + } + +@@ -752,11 +701,11 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) + int ret; + struct sg_table *sgt; + +- ret = mutex_lock_interruptible(&shmem->pages_lock); ++ ret = dma_resv_lock_interruptible(shmem->base.resv, NULL); + if (ret) + return ERR_PTR(ret); + sgt = drm_gem_shmem_get_pages_sgt_locked(shmem); +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + return sgt; + } +diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c +index 0f1ca0b0db495c..5008f0c2428f93 100644 +--- a/drivers/gpu/drm/lima/lima_gem.c ++++ b/drivers/gpu/drm/lima/lima_gem.c +@@ -34,7 +34,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + + new_size = min(new_size, bo->base.base.size); + +- mutex_lock(&bo->base.pages_lock); ++ dma_resv_lock(bo->base.base.resv, NULL); + + if (bo->base.pages) { + pages = bo->base.pages; +@@ -42,7 +42,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, + sizeof(*pages), GFP_KERNEL | __GFP_ZERO); + if (!pages) { +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + return -ENOMEM; + } + +@@ -56,13 +56,13 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + struct page *page = shmem_read_mapping_page(mapping, i); + + if (IS_ERR(page)) { +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + return PTR_ERR(page); + } + pages[i] = page; + } + +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + + ret = sg_alloc_table_from_pages(&sgt, pages, i, 0, + new_size, GFP_KERNEL); +diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c +index 919e6cc0498281..b93589f535e08f 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_drv.c ++++ b/drivers/gpu/drm/panfrost/panfrost_drv.c +@@ -414,6 +414,10 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + + bo = to_panfrost_bo(gem_obj); + ++ ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); ++ if (ret) ++ goto out_put_object; ++ + mutex_lock(&pfdev->shrinker_lock); + mutex_lock(&bo->mappings.lock); + if (args->madv == PANFROST_MADV_DONTNEED) { +@@ -451,7 +455,8 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + out_unlock_mappings: + mutex_unlock(&bo->mappings.lock); + mutex_unlock(&pfdev->shrinker_lock); +- ++ dma_resv_unlock(bo->base.base.resv); ++out_put_object: + drm_gem_object_put(gem_obj); + return ret; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +index bf0170782f2585..6a71a2555f85ca 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c ++++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +@@ -48,14 +48,14 @@ static bool panfrost_gem_purge(struct drm_gem_object *obj) + if (!mutex_trylock(&bo->mappings.lock)) + return false; + +- if (!mutex_trylock(&shmem->pages_lock)) ++ if (!dma_resv_trylock(shmem->base.resv)) + goto unlock_mappings; + + panfrost_gem_teardown_mappings_locked(bo); +- drm_gem_shmem_purge_locked(&bo->base); ++ drm_gem_shmem_purge(&bo->base); + ret = true; + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + unlock_mappings: + mutex_unlock(&bo->mappings.lock); +diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c +index e961fa27702ce6..c0123d09f699c7 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_mmu.c ++++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c +@@ -443,6 +443,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + struct panfrost_gem_mapping *bomapping; + struct panfrost_gem_object *bo; + struct address_space *mapping; ++ struct drm_gem_object *obj; + pgoff_t page_offset; + struct sg_table *sgt; + struct page **pages; +@@ -465,15 +466,16 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + page_offset = addr >> PAGE_SHIFT; + page_offset -= bomapping->mmnode.start; + +- mutex_lock(&bo->base.pages_lock); ++ obj = &bo->base.base; ++ ++ dma_resv_lock(obj->resv, NULL); + + if (!bo->base.pages) { + bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, + sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); + if (!bo->sgts) { +- mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_bo; ++ goto err_unlock; + } + + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, +@@ -481,9 +483,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + if (!pages) { + kvfree(bo->sgts); + bo->sgts = NULL; +- mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_bo; ++ goto err_unlock; + } + bo->base.pages = pages; + bo->base.pages_use_count = 1; +@@ -491,7 +492,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + pages = bo->base.pages; + if (pages[page_offset]) { + /* Pages are already mapped, bail out. */ +- mutex_unlock(&bo->base.pages_lock); + goto out; + } + } +@@ -502,15 +502,12 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { + pages[i] = shmem_read_mapping_page(mapping, i); + if (IS_ERR(pages[i])) { +- mutex_unlock(&bo->base.pages_lock); + ret = PTR_ERR(pages[i]); + pages[i] = NULL; + goto err_pages; + } + } + +- mutex_unlock(&bo->base.pages_lock); +- + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + ret = sg_alloc_table_from_pages(sgt, pages + page_offset, + NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); +@@ -529,6 +526,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); + + out: ++ dma_resv_unlock(obj->resv); ++ + panfrost_gem_mapping_put(bomapping); + + return 0; +@@ -537,6 +536,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + sg_free_table(sgt); + err_pages: + drm_gem_shmem_put_pages(&bo->base); ++err_unlock: ++ dma_resv_unlock(obj->resv); + err_bo: + panfrost_gem_mapping_put(bomapping); + return ret; +diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h +index a2201b2488c56c..3b055d238584cb 100644 +--- a/include/drm/drm_gem_shmem_helper.h ++++ b/include/drm/drm_gem_shmem_helper.h +@@ -26,11 +26,6 @@ struct drm_gem_shmem_object { + */ + struct drm_gem_object base; + +- /** +- * @pages_lock: Protects the page table and use count +- */ +- struct mutex pages_lock; +- + /** + * @pages: Page table + */ +@@ -79,11 +74,6 @@ struct drm_gem_shmem_object { + */ + struct sg_table *sgt; + +- /** +- * @vmap_lock: Protects the vmap address and use count +- */ +- struct mutex vmap_lock; +- + /** + * @vaddr: Kernel virtual address of the backing memory + */ +@@ -109,7 +99,6 @@ struct drm_gem_shmem_object { + struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size); + void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem); + +-int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem); + int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem); +@@ -128,8 +117,7 @@ static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem + !shmem->base.dma_buf && !shmem->base.import_attach; + } + +-void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); +-bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); ++void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); + + struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); + struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem); + +From 4ca12ef6fa680caf60e8facc07578a1fde765b4c Mon Sep 17 00:00:00 2001 +From: Asahi Lina +Date: Mon, 27 Feb 2023 18:04:21 +0900 +Subject: [PATCH 026/150] drm/shmem-helper: Revert accidental non-GPL export + +The referenced commit added a wrapper for drm_gem_shmem_get_pages_sgt(), +but in the process it accidentally changed the export type from GPL to +non-GPL. Switch it back to GPL. + +Reported-by: Dmitry Osipenko +Fixes: ddddedaa0db9 ("drm/shmem-helper: Fix locking for drm_gem_shmem_get_pages_sgt()") +Signed-off-by: Asahi Lina +Signed-off-by: Thomas Zimmermann +Link: https://patchwork.freedesktop.org/patch/msgid/20230227-shmem-export-fix-v1-1-8880b2c25e81@asahilina.net +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 17d08f6e11acd9..accd4d1e27f379 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -709,7 +709,7 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) + + return sgt; + } +-EXPORT_SYMBOL(drm_gem_shmem_get_pages_sgt); ++EXPORT_SYMBOL_GPL(drm_gem_shmem_get_pages_sgt); + + /** + * drm_gem_shmem_prime_import_sg_table - Produce a shmem GEM object from + +From 08f8586783e7c17770a85d4295460840a0bea3bf Mon Sep 17 00:00:00 2001 +From: Thomas Zimmermann +Date: Tue, 28 Feb 2023 16:26:12 +0100 +Subject: [PATCH 027/150] Revert "drm/shmem-helper: Switch to reservation lock" + +This reverts commit 67b7836d4458790f1261e31fe0ce3250989784f0. + +The locking appears incomplete. A caller of SHMEM helper's pin +function never acquires the dma-buf reservation lock. So we get + + WARNING: CPU: 3 PID: 967 at drivers/gpu/drm/drm_gem_shmem_helper.c:243 drm_gem_shmem_pin+0x42/0x90 [drm_shmem_helper] + +Signed-off-by: Thomas Zimmermann +Acked-by: Dmitry Osipenko +Link: https://patchwork.freedesktop.org/patch/msgid/20230228152612.19971-1-tzimmermann@suse.de +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 185 +++++++++++------- + drivers/gpu/drm/lima/lima_gem.c | 8 +- + drivers/gpu/drm/panfrost/panfrost_drv.c | 7 +- + .../gpu/drm/panfrost/panfrost_gem_shrinker.c | 6 +- + drivers/gpu/drm/panfrost/panfrost_mmu.c | 19 +- + include/drm/drm_gem_shmem_helper.h | 14 +- + 6 files changed, 148 insertions(+), 91 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index accd4d1e27f379..1308c534a7ad38 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -88,6 +88,8 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private) + if (ret) + goto err_release; + ++ mutex_init(&shmem->pages_lock); ++ mutex_init(&shmem->vmap_lock); + INIT_LIST_HEAD(&shmem->madv_list); + + if (!private) { +@@ -139,13 +141,11 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + ++ drm_WARN_ON(obj->dev, shmem->vmap_use_count); ++ + if (obj->import_attach) { + drm_prime_gem_destroy(obj, shmem->sgt); + } else { +- dma_resv_lock(shmem->base.resv, NULL); +- +- drm_WARN_ON(obj->dev, shmem->vmap_use_count); +- + if (shmem->sgt) { + dma_unmap_sgtable(obj->dev->dev, shmem->sgt, + DMA_BIDIRECTIONAL, 0); +@@ -154,18 +154,18 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + } + if (shmem->pages) + drm_gem_shmem_put_pages(shmem); +- +- drm_WARN_ON(obj->dev, shmem->pages_use_count); +- +- dma_resv_unlock(shmem->base.resv); + } + ++ drm_WARN_ON(obj->dev, shmem->pages_use_count); ++ + drm_gem_object_release(obj); ++ mutex_destroy(&shmem->pages_lock); ++ mutex_destroy(&shmem->vmap_lock); + kfree(shmem); + } + EXPORT_SYMBOL_GPL(drm_gem_shmem_free); + +-static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) ++static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct page **pages; +@@ -197,16 +197,35 @@ static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) + } + + /* +- * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object ++ * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object + * @shmem: shmem GEM object + * +- * This function decreases the use count and puts the backing pages when use drops to zero. ++ * This function makes sure that backing pages exists for the shmem GEM object ++ * and increases the use count. ++ * ++ * Returns: ++ * 0 on success or a negative error code on failure. + */ +-void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) ++int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; ++ int ret; + +- dma_resv_assert_held(shmem->base.resv); ++ drm_WARN_ON(obj->dev, obj->import_attach); ++ ++ ret = mutex_lock_interruptible(&shmem->pages_lock); ++ if (ret) ++ return ret; ++ ret = drm_gem_shmem_get_pages_locked(shmem); ++ mutex_unlock(&shmem->pages_lock); ++ ++ return ret; ++} ++EXPORT_SYMBOL(drm_gem_shmem_get_pages); ++ ++static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) ++{ ++ struct drm_gem_object *obj = &shmem->base; + + if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + return; +@@ -224,6 +243,19 @@ void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) + shmem->pages_mark_accessed_on_put); + shmem->pages = NULL; + } ++ ++/* ++ * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object ++ * @shmem: shmem GEM object ++ * ++ * This function decreases the use count and puts the backing pages when use drops to zero. ++ */ ++void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) ++{ ++ mutex_lock(&shmem->pages_lock); ++ drm_gem_shmem_put_pages_locked(shmem); ++ mutex_unlock(&shmem->pages_lock); ++} + EXPORT_SYMBOL(drm_gem_shmem_put_pages); + + /** +@@ -240,8 +272,6 @@ int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- dma_resv_assert_held(shmem->base.resv); +- + drm_WARN_ON(obj->dev, obj->import_attach); + + return drm_gem_shmem_get_pages(shmem); +@@ -259,31 +289,14 @@ void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- dma_resv_assert_held(shmem->base.resv); +- + drm_WARN_ON(obj->dev, obj->import_attach); + + drm_gem_shmem_put_pages(shmem); + } + EXPORT_SYMBOL(drm_gem_shmem_unpin); + +-/* +- * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object +- * @shmem: shmem GEM object +- * @map: Returns the kernel virtual address of the SHMEM GEM object's backing +- * store. +- * +- * This function makes sure that a contiguous kernel virtual address mapping +- * exists for the buffer backing the shmem GEM object. It hides the differences +- * between dma-buf imported and natively allocated objects. +- * +- * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). +- * +- * Returns: +- * 0 on success or a negative error code on failure. +- */ +-int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + int ret = 0; +@@ -299,8 +312,6 @@ int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, + } else { + pgprot_t prot = PAGE_KERNEL; + +- dma_resv_assert_held(shmem->base.resv); +- + if (shmem->vmap_use_count++ > 0) { + iosys_map_set_vaddr(map, shmem->vaddr); + return 0; +@@ -335,30 +346,45 @@ int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, + + return ret; + } +-EXPORT_SYMBOL(drm_gem_shmem_vmap); + + /* +- * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object ++ * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object + * @shmem: shmem GEM object +- * @map: Kernel virtual address where the SHMEM GEM object was mapped ++ * @map: Returns the kernel virtual address of the SHMEM GEM object's backing ++ * store. + * +- * This function cleans up a kernel virtual address mapping acquired by +- * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to +- * zero. ++ * This function makes sure that a contiguous kernel virtual address mapping ++ * exists for the buffer backing the shmem GEM object. It hides the differences ++ * between dma-buf imported and natively allocated objects. + * +- * This function hides the differences between dma-buf imported and natively +- * allocated objects. ++ * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). ++ * ++ * Returns: ++ * 0 on success or a negative error code on failure. + */ +-void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) ++{ ++ int ret; ++ ++ ret = mutex_lock_interruptible(&shmem->vmap_lock); ++ if (ret) ++ return ret; ++ ret = drm_gem_shmem_vmap_locked(shmem, map); ++ mutex_unlock(&shmem->vmap_lock); ++ ++ return ret; ++} ++EXPORT_SYMBOL(drm_gem_shmem_vmap); ++ ++static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + + if (obj->import_attach) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); + } else { +- dma_resv_assert_held(shmem->base.resv); +- + if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) + return; + +@@ -371,6 +397,26 @@ void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, + + shmem->vaddr = NULL; + } ++ ++/* ++ * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object ++ * @shmem: shmem GEM object ++ * @map: Kernel virtual address where the SHMEM GEM object was mapped ++ * ++ * This function cleans up a kernel virtual address mapping acquired by ++ * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to ++ * zero. ++ * ++ * This function hides the differences between dma-buf imported and natively ++ * allocated objects. ++ */ ++void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) ++{ ++ mutex_lock(&shmem->vmap_lock); ++ drm_gem_shmem_vunmap_locked(shmem, map); ++ mutex_unlock(&shmem->vmap_lock); ++} + EXPORT_SYMBOL(drm_gem_shmem_vunmap); + + static int +@@ -401,24 +447,24 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + */ + int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) + { +- dma_resv_assert_held(shmem->base.resv); ++ mutex_lock(&shmem->pages_lock); + + if (shmem->madv >= 0) + shmem->madv = madv; + + madv = shmem->madv; + ++ mutex_unlock(&shmem->pages_lock); ++ + return (madv >= 0); + } + EXPORT_SYMBOL(drm_gem_shmem_madvise); + +-void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) ++void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + +- dma_resv_assert_held(shmem->base.resv); +- + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); +@@ -426,7 +472,7 @@ void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) + kfree(shmem->sgt); + shmem->sgt = NULL; + +- drm_gem_shmem_put_pages(shmem); ++ drm_gem_shmem_put_pages_locked(shmem); + + shmem->madv = -1; + +@@ -442,6 +488,17 @@ void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) + + invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 0, (loff_t)-1); + } ++EXPORT_SYMBOL(drm_gem_shmem_purge_locked); ++ ++bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) ++{ ++ if (!mutex_trylock(&shmem->pages_lock)) ++ return false; ++ drm_gem_shmem_purge_locked(shmem); ++ mutex_unlock(&shmem->pages_lock); ++ ++ return true; ++} + EXPORT_SYMBOL(drm_gem_shmem_purge); + + /** +@@ -494,7 +551,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + /* We don't use vmf->pgoff since that has the fake offset */ + page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + +- dma_resv_lock(shmem->base.resv, NULL); ++ mutex_lock(&shmem->pages_lock); + + if (page_offset >= num_pages || + drm_WARN_ON_ONCE(obj->dev, !shmem->pages) || +@@ -506,7 +563,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page)); + } + +- dma_resv_unlock(shmem->base.resv); ++ mutex_unlock(&shmem->pages_lock); + + return ret; + } +@@ -518,7 +575,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + + drm_WARN_ON(obj->dev, obj->import_attach); + +- dma_resv_lock(shmem->base.resv, NULL); ++ mutex_lock(&shmem->pages_lock); + + /* + * We should have already pinned the pages when the buffer was first +@@ -528,7 +585,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + shmem->pages_use_count++; + +- dma_resv_unlock(shmem->base.resv); ++ mutex_unlock(&shmem->pages_lock); + + drm_gem_vm_open(vma); + } +@@ -538,10 +595,7 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + +- dma_resv_lock(shmem->base.resv, NULL); + drm_gem_shmem_put_pages(shmem); +- dma_resv_unlock(shmem->base.resv); +- + drm_gem_vm_close(vma); + } + +@@ -585,10 +639,7 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct + return ret; + } + +- dma_resv_lock(shmem->base.resv, NULL); + ret = drm_gem_shmem_get_pages(shmem); +- dma_resv_unlock(shmem->base.resv); +- + if (ret) + return ret; + +@@ -654,7 +705,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + + drm_WARN_ON(obj->dev, obj->import_attach); + +- ret = drm_gem_shmem_get_pages(shmem); ++ ret = drm_gem_shmem_get_pages_locked(shmem); + if (ret) + return ERR_PTR(ret); + +@@ -676,7 +727,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + sg_free_table(sgt); + kfree(sgt); + err_put_pages: +- drm_gem_shmem_put_pages(shmem); ++ drm_gem_shmem_put_pages_locked(shmem); + return ERR_PTR(ret); + } + +@@ -701,11 +752,11 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) + int ret; + struct sg_table *sgt; + +- ret = dma_resv_lock_interruptible(shmem->base.resv, NULL); ++ ret = mutex_lock_interruptible(&shmem->pages_lock); + if (ret) + return ERR_PTR(ret); + sgt = drm_gem_shmem_get_pages_sgt_locked(shmem); +- dma_resv_unlock(shmem->base.resv); ++ mutex_unlock(&shmem->pages_lock); + + return sgt; + } +diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c +index 5008f0c2428f93..0f1ca0b0db495c 100644 +--- a/drivers/gpu/drm/lima/lima_gem.c ++++ b/drivers/gpu/drm/lima/lima_gem.c +@@ -34,7 +34,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + + new_size = min(new_size, bo->base.base.size); + +- dma_resv_lock(bo->base.base.resv, NULL); ++ mutex_lock(&bo->base.pages_lock); + + if (bo->base.pages) { + pages = bo->base.pages; +@@ -42,7 +42,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, + sizeof(*pages), GFP_KERNEL | __GFP_ZERO); + if (!pages) { +- dma_resv_unlock(bo->base.base.resv); ++ mutex_unlock(&bo->base.pages_lock); + return -ENOMEM; + } + +@@ -56,13 +56,13 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + struct page *page = shmem_read_mapping_page(mapping, i); + + if (IS_ERR(page)) { +- dma_resv_unlock(bo->base.base.resv); ++ mutex_unlock(&bo->base.pages_lock); + return PTR_ERR(page); + } + pages[i] = page; + } + +- dma_resv_unlock(bo->base.base.resv); ++ mutex_unlock(&bo->base.pages_lock); + + ret = sg_alloc_table_from_pages(&sgt, pages, i, 0, + new_size, GFP_KERNEL); +diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c +index b93589f535e08f..919e6cc0498281 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_drv.c ++++ b/drivers/gpu/drm/panfrost/panfrost_drv.c +@@ -414,10 +414,6 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + + bo = to_panfrost_bo(gem_obj); + +- ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); +- if (ret) +- goto out_put_object; +- + mutex_lock(&pfdev->shrinker_lock); + mutex_lock(&bo->mappings.lock); + if (args->madv == PANFROST_MADV_DONTNEED) { +@@ -455,8 +451,7 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + out_unlock_mappings: + mutex_unlock(&bo->mappings.lock); + mutex_unlock(&pfdev->shrinker_lock); +- dma_resv_unlock(bo->base.base.resv); +-out_put_object: ++ + drm_gem_object_put(gem_obj); + return ret; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +index 6a71a2555f85ca..bf0170782f2585 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c ++++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +@@ -48,14 +48,14 @@ static bool panfrost_gem_purge(struct drm_gem_object *obj) + if (!mutex_trylock(&bo->mappings.lock)) + return false; + +- if (!dma_resv_trylock(shmem->base.resv)) ++ if (!mutex_trylock(&shmem->pages_lock)) + goto unlock_mappings; + + panfrost_gem_teardown_mappings_locked(bo); +- drm_gem_shmem_purge(&bo->base); ++ drm_gem_shmem_purge_locked(&bo->base); + ret = true; + +- dma_resv_unlock(shmem->base.resv); ++ mutex_unlock(&shmem->pages_lock); + + unlock_mappings: + mutex_unlock(&bo->mappings.lock); +diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c +index c0123d09f699c7..e961fa27702ce6 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_mmu.c ++++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c +@@ -443,7 +443,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + struct panfrost_gem_mapping *bomapping; + struct panfrost_gem_object *bo; + struct address_space *mapping; +- struct drm_gem_object *obj; + pgoff_t page_offset; + struct sg_table *sgt; + struct page **pages; +@@ -466,16 +465,15 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + page_offset = addr >> PAGE_SHIFT; + page_offset -= bomapping->mmnode.start; + +- obj = &bo->base.base; +- +- dma_resv_lock(obj->resv, NULL); ++ mutex_lock(&bo->base.pages_lock); + + if (!bo->base.pages) { + bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, + sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); + if (!bo->sgts) { ++ mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_unlock; ++ goto err_bo; + } + + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, +@@ -483,8 +481,9 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + if (!pages) { + kvfree(bo->sgts); + bo->sgts = NULL; ++ mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_unlock; ++ goto err_bo; + } + bo->base.pages = pages; + bo->base.pages_use_count = 1; +@@ -492,6 +491,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + pages = bo->base.pages; + if (pages[page_offset]) { + /* Pages are already mapped, bail out. */ ++ mutex_unlock(&bo->base.pages_lock); + goto out; + } + } +@@ -502,12 +502,15 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { + pages[i] = shmem_read_mapping_page(mapping, i); + if (IS_ERR(pages[i])) { ++ mutex_unlock(&bo->base.pages_lock); + ret = PTR_ERR(pages[i]); + pages[i] = NULL; + goto err_pages; + } + } + ++ mutex_unlock(&bo->base.pages_lock); ++ + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + ret = sg_alloc_table_from_pages(sgt, pages + page_offset, + NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); +@@ -526,8 +529,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); + + out: +- dma_resv_unlock(obj->resv); +- + panfrost_gem_mapping_put(bomapping); + + return 0; +@@ -536,8 +537,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + sg_free_table(sgt); + err_pages: + drm_gem_shmem_put_pages(&bo->base); +-err_unlock: +- dma_resv_unlock(obj->resv); + err_bo: + panfrost_gem_mapping_put(bomapping); + return ret; +diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h +index 3b055d238584cb..a2201b2488c56c 100644 +--- a/include/drm/drm_gem_shmem_helper.h ++++ b/include/drm/drm_gem_shmem_helper.h +@@ -26,6 +26,11 @@ struct drm_gem_shmem_object { + */ + struct drm_gem_object base; + ++ /** ++ * @pages_lock: Protects the page table and use count ++ */ ++ struct mutex pages_lock; ++ + /** + * @pages: Page table + */ +@@ -74,6 +79,11 @@ struct drm_gem_shmem_object { + */ + struct sg_table *sgt; + ++ /** ++ * @vmap_lock: Protects the vmap address and use count ++ */ ++ struct mutex vmap_lock; ++ + /** + * @vaddr: Kernel virtual address of the backing memory + */ +@@ -99,6 +109,7 @@ struct drm_gem_shmem_object { + struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size); + void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem); + ++int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem); + int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem); +@@ -117,7 +128,8 @@ static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem + !shmem->base.dma_buf && !shmem->base.import_attach; + } + +-void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); ++void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); ++bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); + + struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); + struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem); + +From bd9555386d2b107037202b501e6f753379ee4e60 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Tue, 30 May 2023 01:39:35 +0300 +Subject: [PATCH 028/150] drm/shmem-helper: Switch to reservation lock + +Replace all drm-shmem locks with a GEM reservation lock. This makes locks +consistent with dma-buf locking convention where importers are responsible +for holding reservation lock for all operations performed over dma-bufs, +preventing deadlock between dma-buf importers and exporters. + +Suggested-by: Daniel Vetter +Acked-by: Thomas Zimmermann +Reviewed-by: Emil Velikov +Signed-off-by: Dmitry Osipenko +Link: https://patchwork.freedesktop.org/patch/msgid/20230529223935.2672495-7-dmitry.osipenko@collabora.com +--- + drivers/gpu/drm/drm_gem_shmem_helper.c | 210 ++++++++---------- + drivers/gpu/drm/lima/lima_gem.c | 8 +- + drivers/gpu/drm/panfrost/panfrost_drv.c | 7 +- + .../gpu/drm/panfrost/panfrost_gem_shrinker.c | 6 +- + drivers/gpu/drm/panfrost/panfrost_mmu.c | 19 +- + include/drm/drm_gem_shmem_helper.h | 14 +- + 6 files changed, 116 insertions(+), 148 deletions(-) + +diff --git a/drivers/gpu/drm/drm_gem_shmem_helper.c b/drivers/gpu/drm/drm_gem_shmem_helper.c +index 1308c534a7ad38..926e893f652764 100644 +--- a/drivers/gpu/drm/drm_gem_shmem_helper.c ++++ b/drivers/gpu/drm/drm_gem_shmem_helper.c +@@ -88,8 +88,6 @@ __drm_gem_shmem_create(struct drm_device *dev, size_t size, bool private) + if (ret) + goto err_release; + +- mutex_init(&shmem->pages_lock); +- mutex_init(&shmem->vmap_lock); + INIT_LIST_HEAD(&shmem->madv_list); + + if (!private) { +@@ -141,11 +139,13 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + +- drm_WARN_ON(obj->dev, shmem->vmap_use_count); +- + if (obj->import_attach) { + drm_prime_gem_destroy(obj, shmem->sgt); + } else { ++ dma_resv_lock(shmem->base.resv, NULL); ++ ++ drm_WARN_ON(obj->dev, shmem->vmap_use_count); ++ + if (shmem->sgt) { + dma_unmap_sgtable(obj->dev->dev, shmem->sgt, + DMA_BIDIRECTIONAL, 0); +@@ -154,22 +154,24 @@ void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem) + } + if (shmem->pages) + drm_gem_shmem_put_pages(shmem); +- } + +- drm_WARN_ON(obj->dev, shmem->pages_use_count); ++ drm_WARN_ON(obj->dev, shmem->pages_use_count); ++ ++ dma_resv_unlock(shmem->base.resv); ++ } + + drm_gem_object_release(obj); +- mutex_destroy(&shmem->pages_lock); +- mutex_destroy(&shmem->vmap_lock); + kfree(shmem); + } + EXPORT_SYMBOL_GPL(drm_gem_shmem_free); + +-static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) ++static int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct page **pages; + ++ dma_resv_assert_held(shmem->base.resv); ++ + if (shmem->pages_use_count++ > 0) + return 0; + +@@ -197,35 +199,16 @@ static int drm_gem_shmem_get_pages_locked(struct drm_gem_shmem_object *shmem) + } + + /* +- * drm_gem_shmem_get_pages - Allocate backing pages for a shmem GEM object ++ * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object + * @shmem: shmem GEM object + * +- * This function makes sure that backing pages exists for the shmem GEM object +- * and increases the use count. +- * +- * Returns: +- * 0 on success or a negative error code on failure. ++ * This function decreases the use count and puts the backing pages when use drops to zero. + */ +-int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem) ++void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; +- int ret; + +- drm_WARN_ON(obj->dev, obj->import_attach); +- +- ret = mutex_lock_interruptible(&shmem->pages_lock); +- if (ret) +- return ret; +- ret = drm_gem_shmem_get_pages_locked(shmem); +- mutex_unlock(&shmem->pages_lock); +- +- return ret; +-} +-EXPORT_SYMBOL(drm_gem_shmem_get_pages); +- +-static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) +-{ +- struct drm_gem_object *obj = &shmem->base; ++ dma_resv_assert_held(shmem->base.resv); + + if (drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + return; +@@ -243,20 +226,25 @@ static void drm_gem_shmem_put_pages_locked(struct drm_gem_shmem_object *shmem) + shmem->pages_mark_accessed_on_put); + shmem->pages = NULL; + } ++EXPORT_SYMBOL(drm_gem_shmem_put_pages); + +-/* +- * drm_gem_shmem_put_pages - Decrease use count on the backing pages for a shmem GEM object +- * @shmem: shmem GEM object +- * +- * This function decreases the use count and puts the backing pages when use drops to zero. +- */ +-void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem) ++static int drm_gem_shmem_pin_locked(struct drm_gem_shmem_object *shmem) + { +- mutex_lock(&shmem->pages_lock); +- drm_gem_shmem_put_pages_locked(shmem); +- mutex_unlock(&shmem->pages_lock); ++ int ret; ++ ++ dma_resv_assert_held(shmem->base.resv); ++ ++ ret = drm_gem_shmem_get_pages(shmem); ++ ++ return ret; ++} ++ ++static void drm_gem_shmem_unpin_locked(struct drm_gem_shmem_object *shmem) ++{ ++ dma_resv_assert_held(shmem->base.resv); ++ ++ drm_gem_shmem_put_pages(shmem); + } +-EXPORT_SYMBOL(drm_gem_shmem_put_pages); + + /** + * drm_gem_shmem_pin - Pin backing pages for a shmem GEM object +@@ -271,10 +259,17 @@ EXPORT_SYMBOL(drm_gem_shmem_put_pages); + int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; ++ int ret; + + drm_WARN_ON(obj->dev, obj->import_attach); + +- return drm_gem_shmem_get_pages(shmem); ++ ret = dma_resv_lock_interruptible(shmem->base.resv, NULL); ++ if (ret) ++ return ret; ++ ret = drm_gem_shmem_pin_locked(shmem); ++ dma_resv_unlock(shmem->base.resv); ++ ++ return ret; + } + EXPORT_SYMBOL(drm_gem_shmem_pin); + +@@ -291,12 +286,29 @@ void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem) + + drm_WARN_ON(obj->dev, obj->import_attach); + +- drm_gem_shmem_put_pages(shmem); ++ dma_resv_lock(shmem->base.resv, NULL); ++ drm_gem_shmem_unpin_locked(shmem); ++ dma_resv_unlock(shmem->base.resv); + } + EXPORT_SYMBOL(drm_gem_shmem_unpin); + +-static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++/* ++ * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object ++ * @shmem: shmem GEM object ++ * @map: Returns the kernel virtual address of the SHMEM GEM object's backing ++ * store. ++ * ++ * This function makes sure that a contiguous kernel virtual address mapping ++ * exists for the buffer backing the shmem GEM object. It hides the differences ++ * between dma-buf imported and natively allocated objects. ++ * ++ * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). ++ * ++ * Returns: ++ * 0 on success or a negative error code on failure. ++ */ ++int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + int ret = 0; +@@ -312,6 +324,8 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + } else { + pgprot_t prot = PAGE_KERNEL; + ++ dma_resv_assert_held(shmem->base.resv); ++ + if (shmem->vmap_use_count++ > 0) { + iosys_map_set_vaddr(map, shmem->vaddr); + return 0; +@@ -346,45 +360,30 @@ static int drm_gem_shmem_vmap_locked(struct drm_gem_shmem_object *shmem, + + return ret; + } ++EXPORT_SYMBOL(drm_gem_shmem_vmap); + + /* +- * drm_gem_shmem_vmap - Create a virtual mapping for a shmem GEM object ++ * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object + * @shmem: shmem GEM object +- * @map: Returns the kernel virtual address of the SHMEM GEM object's backing +- * store. +- * +- * This function makes sure that a contiguous kernel virtual address mapping +- * exists for the buffer backing the shmem GEM object. It hides the differences +- * between dma-buf imported and natively allocated objects. ++ * @map: Kernel virtual address where the SHMEM GEM object was mapped + * +- * Acquired mappings should be cleaned up by calling drm_gem_shmem_vunmap(). ++ * This function cleans up a kernel virtual address mapping acquired by ++ * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to ++ * zero. + * +- * Returns: +- * 0 on success or a negative error code on failure. ++ * This function hides the differences between dma-buf imported and natively ++ * allocated objects. + */ +-int drm_gem_shmem_vmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) +-{ +- int ret; +- +- ret = mutex_lock_interruptible(&shmem->vmap_lock); +- if (ret) +- return ret; +- ret = drm_gem_shmem_vmap_locked(shmem, map); +- mutex_unlock(&shmem->vmap_lock); +- +- return ret; +-} +-EXPORT_SYMBOL(drm_gem_shmem_vmap); +- +-static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) ++void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, ++ struct iosys_map *map) + { + struct drm_gem_object *obj = &shmem->base; + + if (obj->import_attach) { + dma_buf_vunmap(obj->import_attach->dmabuf, map); + } else { ++ dma_resv_assert_held(shmem->base.resv); ++ + if (drm_WARN_ON_ONCE(obj->dev, !shmem->vmap_use_count)) + return; + +@@ -397,26 +396,6 @@ static void drm_gem_shmem_vunmap_locked(struct drm_gem_shmem_object *shmem, + + shmem->vaddr = NULL; + } +- +-/* +- * drm_gem_shmem_vunmap - Unmap a virtual mapping for a shmem GEM object +- * @shmem: shmem GEM object +- * @map: Kernel virtual address where the SHMEM GEM object was mapped +- * +- * This function cleans up a kernel virtual address mapping acquired by +- * drm_gem_shmem_vmap(). The mapping is only removed when the use count drops to +- * zero. +- * +- * This function hides the differences between dma-buf imported and natively +- * allocated objects. +- */ +-void drm_gem_shmem_vunmap(struct drm_gem_shmem_object *shmem, +- struct iosys_map *map) +-{ +- mutex_lock(&shmem->vmap_lock); +- drm_gem_shmem_vunmap_locked(shmem, map); +- mutex_unlock(&shmem->vmap_lock); +-} + EXPORT_SYMBOL(drm_gem_shmem_vunmap); + + static int +@@ -447,24 +426,24 @@ drm_gem_shmem_create_with_handle(struct drm_file *file_priv, + */ + int drm_gem_shmem_madvise(struct drm_gem_shmem_object *shmem, int madv) + { +- mutex_lock(&shmem->pages_lock); ++ dma_resv_assert_held(shmem->base.resv); + + if (shmem->madv >= 0) + shmem->madv = madv; + + madv = shmem->madv; + +- mutex_unlock(&shmem->pages_lock); +- + return (madv >= 0); + } + EXPORT_SYMBOL(drm_gem_shmem_madvise); + +-void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) ++void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) + { + struct drm_gem_object *obj = &shmem->base; + struct drm_device *dev = obj->dev; + ++ dma_resv_assert_held(shmem->base.resv); ++ + drm_WARN_ON(obj->dev, !drm_gem_shmem_is_purgeable(shmem)); + + dma_unmap_sgtable(dev->dev, shmem->sgt, DMA_BIDIRECTIONAL, 0); +@@ -472,7 +451,7 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + kfree(shmem->sgt); + shmem->sgt = NULL; + +- drm_gem_shmem_put_pages_locked(shmem); ++ drm_gem_shmem_put_pages(shmem); + + shmem->madv = -1; + +@@ -488,17 +467,6 @@ void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem) + + invalidate_mapping_pages(file_inode(obj->filp)->i_mapping, 0, (loff_t)-1); + } +-EXPORT_SYMBOL(drm_gem_shmem_purge_locked); +- +-bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem) +-{ +- if (!mutex_trylock(&shmem->pages_lock)) +- return false; +- drm_gem_shmem_purge_locked(shmem); +- mutex_unlock(&shmem->pages_lock); +- +- return true; +-} + EXPORT_SYMBOL(drm_gem_shmem_purge); + + /** +@@ -551,7 +519,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + /* We don't use vmf->pgoff since that has the fake offset */ + page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT; + +- mutex_lock(&shmem->pages_lock); ++ dma_resv_lock(shmem->base.resv, NULL); + + if (page_offset >= num_pages || + drm_WARN_ON_ONCE(obj->dev, !shmem->pages) || +@@ -563,7 +531,7 @@ static vm_fault_t drm_gem_shmem_fault(struct vm_fault *vmf) + ret = vmf_insert_pfn(vma, vmf->address, page_to_pfn(page)); + } + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + return ret; + } +@@ -575,7 +543,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + + drm_WARN_ON(obj->dev, obj->import_attach); + +- mutex_lock(&shmem->pages_lock); ++ dma_resv_lock(shmem->base.resv, NULL); + + /* + * We should have already pinned the pages when the buffer was first +@@ -585,7 +553,7 @@ static void drm_gem_shmem_vm_open(struct vm_area_struct *vma) + if (!drm_WARN_ON_ONCE(obj->dev, !shmem->pages_use_count)) + shmem->pages_use_count++; + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + drm_gem_vm_open(vma); + } +@@ -595,7 +563,10 @@ static void drm_gem_shmem_vm_close(struct vm_area_struct *vma) + struct drm_gem_object *obj = vma->vm_private_data; + struct drm_gem_shmem_object *shmem = to_drm_gem_shmem_obj(obj); + ++ dma_resv_lock(shmem->base.resv, NULL); + drm_gem_shmem_put_pages(shmem); ++ dma_resv_unlock(shmem->base.resv); ++ + drm_gem_vm_close(vma); + } + +@@ -639,7 +610,10 @@ int drm_gem_shmem_mmap(struct drm_gem_shmem_object *shmem, struct vm_area_struct + return ret; + } + ++ dma_resv_lock(shmem->base.resv, NULL); + ret = drm_gem_shmem_get_pages(shmem); ++ dma_resv_unlock(shmem->base.resv); ++ + if (ret) + return ret; + +@@ -705,7 +679,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + + drm_WARN_ON(obj->dev, obj->import_attach); + +- ret = drm_gem_shmem_get_pages_locked(shmem); ++ ret = drm_gem_shmem_get_pages(shmem); + if (ret) + return ERR_PTR(ret); + +@@ -727,7 +701,7 @@ static struct sg_table *drm_gem_shmem_get_pages_sgt_locked(struct drm_gem_shmem_ + sg_free_table(sgt); + kfree(sgt); + err_put_pages: +- drm_gem_shmem_put_pages_locked(shmem); ++ drm_gem_shmem_put_pages(shmem); + return ERR_PTR(ret); + } + +@@ -752,11 +726,11 @@ struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem) + int ret; + struct sg_table *sgt; + +- ret = mutex_lock_interruptible(&shmem->pages_lock); ++ ret = dma_resv_lock_interruptible(shmem->base.resv, NULL); + if (ret) + return ERR_PTR(ret); + sgt = drm_gem_shmem_get_pages_sgt_locked(shmem); +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + return sgt; + } +diff --git a/drivers/gpu/drm/lima/lima_gem.c b/drivers/gpu/drm/lima/lima_gem.c +index 0f1ca0b0db495c..5008f0c2428f93 100644 +--- a/drivers/gpu/drm/lima/lima_gem.c ++++ b/drivers/gpu/drm/lima/lima_gem.c +@@ -34,7 +34,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + + new_size = min(new_size, bo->base.base.size); + +- mutex_lock(&bo->base.pages_lock); ++ dma_resv_lock(bo->base.base.resv, NULL); + + if (bo->base.pages) { + pages = bo->base.pages; +@@ -42,7 +42,7 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, + sizeof(*pages), GFP_KERNEL | __GFP_ZERO); + if (!pages) { +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + return -ENOMEM; + } + +@@ -56,13 +56,13 @@ int lima_heap_alloc(struct lima_bo *bo, struct lima_vm *vm) + struct page *page = shmem_read_mapping_page(mapping, i); + + if (IS_ERR(page)) { +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + return PTR_ERR(page); + } + pages[i] = page; + } + +- mutex_unlock(&bo->base.pages_lock); ++ dma_resv_unlock(bo->base.base.resv); + + ret = sg_alloc_table_from_pages(&sgt, pages, i, 0, + new_size, GFP_KERNEL); +diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c +index 919e6cc0498281..b93589f535e08f 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_drv.c ++++ b/drivers/gpu/drm/panfrost/panfrost_drv.c +@@ -414,6 +414,10 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + + bo = to_panfrost_bo(gem_obj); + ++ ret = dma_resv_lock_interruptible(bo->base.base.resv, NULL); ++ if (ret) ++ goto out_put_object; ++ + mutex_lock(&pfdev->shrinker_lock); + mutex_lock(&bo->mappings.lock); + if (args->madv == PANFROST_MADV_DONTNEED) { +@@ -451,7 +455,8 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, + out_unlock_mappings: + mutex_unlock(&bo->mappings.lock); + mutex_unlock(&pfdev->shrinker_lock); +- ++ dma_resv_unlock(bo->base.base.resv); ++out_put_object: + drm_gem_object_put(gem_obj); + return ret; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +index bf0170782f2585..6a71a2555f85ca 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c ++++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +@@ -48,14 +48,14 @@ static bool panfrost_gem_purge(struct drm_gem_object *obj) + if (!mutex_trylock(&bo->mappings.lock)) + return false; + +- if (!mutex_trylock(&shmem->pages_lock)) ++ if (!dma_resv_trylock(shmem->base.resv)) + goto unlock_mappings; + + panfrost_gem_teardown_mappings_locked(bo); +- drm_gem_shmem_purge_locked(&bo->base); ++ drm_gem_shmem_purge(&bo->base); + ret = true; + +- mutex_unlock(&shmem->pages_lock); ++ dma_resv_unlock(shmem->base.resv); + + unlock_mappings: + mutex_unlock(&bo->mappings.lock); +diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c +index e961fa27702ce6..c0123d09f699c7 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_mmu.c ++++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c +@@ -443,6 +443,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + struct panfrost_gem_mapping *bomapping; + struct panfrost_gem_object *bo; + struct address_space *mapping; ++ struct drm_gem_object *obj; + pgoff_t page_offset; + struct sg_table *sgt; + struct page **pages; +@@ -465,15 +466,16 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + page_offset = addr >> PAGE_SHIFT; + page_offset -= bomapping->mmnode.start; + +- mutex_lock(&bo->base.pages_lock); ++ obj = &bo->base.base; ++ ++ dma_resv_lock(obj->resv, NULL); + + if (!bo->base.pages) { + bo->sgts = kvmalloc_array(bo->base.base.size / SZ_2M, + sizeof(struct sg_table), GFP_KERNEL | __GFP_ZERO); + if (!bo->sgts) { +- mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_bo; ++ goto err_unlock; + } + + pages = kvmalloc_array(bo->base.base.size >> PAGE_SHIFT, +@@ -481,9 +483,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + if (!pages) { + kvfree(bo->sgts); + bo->sgts = NULL; +- mutex_unlock(&bo->base.pages_lock); + ret = -ENOMEM; +- goto err_bo; ++ goto err_unlock; + } + bo->base.pages = pages; + bo->base.pages_use_count = 1; +@@ -491,7 +492,6 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + pages = bo->base.pages; + if (pages[page_offset]) { + /* Pages are already mapped, bail out. */ +- mutex_unlock(&bo->base.pages_lock); + goto out; + } + } +@@ -502,15 +502,12 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + for (i = page_offset; i < page_offset + NUM_FAULT_PAGES; i++) { + pages[i] = shmem_read_mapping_page(mapping, i); + if (IS_ERR(pages[i])) { +- mutex_unlock(&bo->base.pages_lock); + ret = PTR_ERR(pages[i]); + pages[i] = NULL; + goto err_pages; + } + } + +- mutex_unlock(&bo->base.pages_lock); +- + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + ret = sg_alloc_table_from_pages(sgt, pages + page_offset, + NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); +@@ -529,6 +526,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); + + out: ++ dma_resv_unlock(obj->resv); ++ + panfrost_gem_mapping_put(bomapping); + + return 0; +@@ -537,6 +536,8 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, + sg_free_table(sgt); + err_pages: + drm_gem_shmem_put_pages(&bo->base); ++err_unlock: ++ dma_resv_unlock(obj->resv); + err_bo: + panfrost_gem_mapping_put(bomapping); + return ret; +diff --git a/include/drm/drm_gem_shmem_helper.h b/include/drm/drm_gem_shmem_helper.h +index a2201b2488c56c..3b055d238584cb 100644 +--- a/include/drm/drm_gem_shmem_helper.h ++++ b/include/drm/drm_gem_shmem_helper.h +@@ -26,11 +26,6 @@ struct drm_gem_shmem_object { + */ + struct drm_gem_object base; + +- /** +- * @pages_lock: Protects the page table and use count +- */ +- struct mutex pages_lock; +- + /** + * @pages: Page table + */ +@@ -79,11 +74,6 @@ struct drm_gem_shmem_object { + */ + struct sg_table *sgt; + +- /** +- * @vmap_lock: Protects the vmap address and use count +- */ +- struct mutex vmap_lock; +- + /** + * @vaddr: Kernel virtual address of the backing memory + */ +@@ -109,7 +99,6 @@ struct drm_gem_shmem_object { + struct drm_gem_shmem_object *drm_gem_shmem_create(struct drm_device *dev, size_t size); + void drm_gem_shmem_free(struct drm_gem_shmem_object *shmem); + +-int drm_gem_shmem_get_pages(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_put_pages(struct drm_gem_shmem_object *shmem); + int drm_gem_shmem_pin(struct drm_gem_shmem_object *shmem); + void drm_gem_shmem_unpin(struct drm_gem_shmem_object *shmem); +@@ -128,8 +117,7 @@ static inline bool drm_gem_shmem_is_purgeable(struct drm_gem_shmem_object *shmem + !shmem->base.dma_buf && !shmem->base.import_attach; + } + +-void drm_gem_shmem_purge_locked(struct drm_gem_shmem_object *shmem); +-bool drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); ++void drm_gem_shmem_purge(struct drm_gem_shmem_object *shmem); + + struct sg_table *drm_gem_shmem_get_sg_table(struct drm_gem_shmem_object *shmem); + struct sg_table *drm_gem_shmem_get_pages_sgt(struct drm_gem_shmem_object *shmem); + +From 2d3dfa0472a28e9bdaaf1820fc284e2c28ae51ae Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Fri, 24 Nov 2023 15:24:33 +0100 +Subject: [PATCH 029/150] iommu: Allow passing custom allocators to pgtable + drivers + +This will be useful for GPU drivers who want to keep page tables in a +pool so they can: + +- keep freed page tables in a free pool and speed-up upcoming page + table allocations +- batch page table allocation instead of allocating one page at a time +- pre-reserve pages for page tables needed for map/unmap operations, + to ensure map/unmap operations don't try to allocate memory in paths + they're allowed to block or fail + +It might also be valuable for other aspects of GPU and similar +use-cases, like fine-grained memory accounting and resource limiting. + +We will extend the Arm LPAE format to support custom allocators in a +separate commit. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Robin Murphy +Link: https://lore.kernel.org/r/20231124142434.1577550-2-boris.brezillon@collabora.com +Signed-off-by: Joerg Roedel +--- + drivers/iommu/io-pgtable.c | 23 +++++++++++++++++++++++ + include/linux/io-pgtable.h | 34 ++++++++++++++++++++++++++++++++++ + 2 files changed, 57 insertions(+) + +diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c +index b843fcd365d286..8841c1487f0048 100644 +--- a/drivers/iommu/io-pgtable.c ++++ b/drivers/iommu/io-pgtable.c +@@ -34,6 +34,26 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { + #endif + }; + ++static int check_custom_allocator(enum io_pgtable_fmt fmt, ++ struct io_pgtable_cfg *cfg) ++{ ++ /* No custom allocator, no need to check the format. */ ++ if (!cfg->alloc && !cfg->free) ++ return 0; ++ ++ /* When passing a custom allocator, both the alloc and free ++ * functions should be provided. ++ */ ++ if (!cfg->alloc || !cfg->free) ++ return -EINVAL; ++ ++ /* Make sure the format supports custom allocators. */ ++ if (io_pgtable_init_table[fmt]->caps & IO_PGTABLE_CAP_CUSTOM_ALLOCATOR) ++ return 0; ++ ++ return -EINVAL; ++} ++ + struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + struct io_pgtable_cfg *cfg, + void *cookie) +@@ -44,6 +64,9 @@ struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, + if (fmt >= IO_PGTABLE_NUM_FMTS) + return NULL; + ++ if (check_custom_allocator(fmt, cfg)) ++ return NULL; ++ + fns = io_pgtable_init_table[fmt]; + if (!fns) + return NULL; +diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h +index 1f068dfdb140c8..9e0bffe3fe5f52 100644 +--- a/include/linux/io-pgtable.h ++++ b/include/linux/io-pgtable.h +@@ -100,6 +100,30 @@ struct io_pgtable_cfg { + const struct iommu_flush_ops *tlb; + struct device *iommu_dev; + ++ /** ++ * @alloc: Custom page allocator. ++ * ++ * Optional hook used to allocate page tables. If this function is NULL, ++ * @free must be NULL too. ++ * ++ * Memory returned should be zeroed and suitable for dma_map_single() and ++ * virt_to_phys(). ++ * ++ * Not all formats support custom page allocators. Before considering ++ * passing a non-NULL value, make sure the chosen page format supports ++ * this feature. ++ */ ++ void *(*alloc)(void *cookie, size_t size, gfp_t gfp); ++ ++ /** ++ * @free: Custom page de-allocator. ++ * ++ * Optional hook used to free page tables allocated with the @alloc ++ * hook. Must be non-NULL if @alloc is not NULL, must be NULL ++ * otherwise. ++ */ ++ void (*free)(void *cookie, void *pages, size_t size); ++ + /* Low-level data specific to the table format */ + union { + struct { +@@ -243,16 +267,26 @@ io_pgtable_tlb_add_page(struct io_pgtable *iop, + iop->cfg.tlb->tlb_add_page(gather, iova, granule, iop->cookie); + } + ++/** ++ * enum io_pgtable_caps - IO page table backend capabilities. ++ */ ++enum io_pgtable_caps { ++ /** @IO_PGTABLE_CAP_CUSTOM_ALLOCATOR: Backend accepts custom page table allocators. */ ++ IO_PGTABLE_CAP_CUSTOM_ALLOCATOR = BIT(0), ++}; ++ + /** + * struct io_pgtable_init_fns - Alloc/free a set of page tables for a + * particular format. + * + * @alloc: Allocate a set of page tables described by cfg. + * @free: Free the page tables associated with iop. ++ * @caps: Combination of @io_pgtable_caps flags encoding the backend capabilities. + */ + struct io_pgtable_init_fns { + struct io_pgtable *(*alloc)(struct io_pgtable_cfg *cfg, void *cookie); + void (*free)(struct io_pgtable *iop); ++ u32 caps; + }; + + extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; + +From 59d9845ec3d1124e17b6115b3c42bffbf751b5c9 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Fri, 24 Nov 2023 15:24:34 +0100 +Subject: [PATCH 030/150] iommu: Extend LPAE page table format to support + custom allocators + +We need that in order to implement the VM_BIND ioctl in the GPU driver +targeting new Mali GPUs. + +VM_BIND is about executing MMU map/unmap requests asynchronously, +possibly after waiting for external dependencies encoded as dma_fences. +We intend to use the drm_sched framework to automate the dependency +tracking and VM job dequeuing logic, but this comes with its own set +of constraints, one of them being the fact we are not allowed to +allocate memory in the drm_gpu_scheduler_ops::run_job() to avoid this +sort of deadlocks: + +- VM_BIND map job needs to allocate a page table to map some memory + to the VM. No memory available, so kswapd is kicked +- GPU driver shrinker backend ends up waiting on the fence attached to + the VM map job or any other job fence depending on this VM operation. + +With custom allocators, we will be able to pre-reserve enough pages to +guarantee the map/unmap operations we queued will take place without +going through the system allocator. But we can also optimize +allocation/reservation by not free-ing pages immediately, so any +upcoming page table allocation requests can be serviced by some free +page table pool kept at the driver level. + +I might also be valuable for other aspects of GPU and similar +use-cases, like fine-grained memory accounting and resource limiting. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Robin Murphy +Link: https://lore.kernel.org/r/20231124142434.1577550-3-boris.brezillon@collabora.com +Signed-off-by: Joerg Roedel +--- + drivers/iommu/io-pgtable-arm.c | 55 ++++++++++++++++++++++++---------- + 1 file changed, 39 insertions(+), 16 deletions(-) + +diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c +index 0ba817e863465b..367ac917239529 100644 +--- a/drivers/iommu/io-pgtable-arm.c ++++ b/drivers/iommu/io-pgtable-arm.c +@@ -188,20 +188,28 @@ static dma_addr_t __arm_lpae_dma_addr(void *pages) + } + + static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, +- struct io_pgtable_cfg *cfg) ++ struct io_pgtable_cfg *cfg, ++ void *cookie) + { + struct device *dev = cfg->iommu_dev; + int order = get_order(size); +- struct page *p; + dma_addr_t dma; + void *pages; + + VM_BUG_ON((gfp & __GFP_HIGHMEM)); +- p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); +- if (!p) ++ ++ if (cfg->alloc) { ++ pages = cfg->alloc(cookie, size, gfp); ++ } else { ++ struct page *p; ++ ++ p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); ++ pages = p ? page_address(p) : NULL; ++ } ++ ++ if (!pages) + return NULL; + +- pages = page_address(p); + if (!cfg->coherent_walk) { + dma = dma_map_single(dev, pages, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) +@@ -220,18 +228,28 @@ static void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, + out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); ++ + out_free: +- __free_pages(p, order); ++ if (cfg->free) ++ cfg->free(cookie, pages, size); ++ else ++ free_pages((unsigned long)pages, order); ++ + return NULL; + } + + static void __arm_lpae_free_pages(void *pages, size_t size, +- struct io_pgtable_cfg *cfg) ++ struct io_pgtable_cfg *cfg, ++ void *cookie) + { + if (!cfg->coherent_walk) + dma_unmap_single(cfg->iommu_dev, __arm_lpae_dma_addr(pages), + size, DMA_TO_DEVICE); +- free_pages((unsigned long)pages, get_order(size)); ++ ++ if (cfg->free) ++ cfg->free(cookie, pages, size); ++ else ++ free_pages((unsigned long)pages, get_order(size)); + } + + static void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries, +@@ -373,13 +391,13 @@ static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova, + /* Grab a pointer to the next level */ + pte = READ_ONCE(*ptep); + if (!pte) { +- cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg); ++ cptep = __arm_lpae_alloc_pages(tblsz, gfp, cfg, data->iop.cookie); + if (!cptep) + return -ENOMEM; + + pte = arm_lpae_install_table(cptep, ptep, 0, data); + if (pte) +- __arm_lpae_free_pages(cptep, tblsz, cfg); ++ __arm_lpae_free_pages(cptep, tblsz, cfg, data->iop.cookie); + } else if (!cfg->coherent_walk && !(pte & ARM_LPAE_PTE_SW_SYNC)) { + __arm_lpae_sync_pte(ptep, 1, cfg); + } +@@ -531,7 +549,7 @@ static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl, + __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); + } + +- __arm_lpae_free_pages(start, table_size, &data->iop.cfg); ++ __arm_lpae_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie); + } + + static void arm_lpae_free_pgtable(struct io_pgtable *iop) +@@ -559,7 +577,7 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, + if (WARN_ON(lvl == ARM_LPAE_MAX_LEVELS)) + return 0; + +- tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg); ++ tablep = __arm_lpae_alloc_pages(tablesz, GFP_ATOMIC, cfg, data->iop.cookie); + if (!tablep) + return 0; /* Bytes unmapped */ + +@@ -582,7 +600,7 @@ static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, + + pte = arm_lpae_install_table(tablep, ptep, blk_pte, data); + if (pte != blk_pte) { +- __arm_lpae_free_pages(tablep, tablesz, cfg); ++ __arm_lpae_free_pages(tablep, tablesz, cfg, data->iop.cookie); + /* + * We may race against someone unmapping another part of this + * block, but anything else is invalid. We can't misinterpret +@@ -897,7 +915,7 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) + + /* Looking good; allocate a pgd */ + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), +- GFP_KERNEL, cfg); ++ GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_data; + +@@ -999,7 +1017,7 @@ arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) + + /* Allocate pgd pages */ + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), +- GFP_KERNEL, cfg); ++ GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_data; + +@@ -1074,7 +1092,7 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) + << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_DEV)); + + data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL, +- cfg); ++ cfg, cookie); + if (!data->pgd) + goto out_free_data; + +@@ -1095,26 +1113,31 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) + } + + struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns = { ++ .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, + .alloc = arm_64_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, + }; + + struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns = { ++ .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, + .alloc = arm_64_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, + }; + + struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns = { ++ .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, + .alloc = arm_32_lpae_alloc_pgtable_s1, + .free = arm_lpae_free_pgtable, + }; + + struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns = { ++ .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, + .alloc = arm_32_lpae_alloc_pgtable_s2, + .free = arm_lpae_free_pgtable, + }; + + struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns = { ++ .caps = IO_PGTABLE_CAP_CUSTOM_ALLOCATOR, + .alloc = arm_mali_lpae_alloc_pgtable, + .free = arm_lpae_free_pgtable, + }; + +From b100f1e24476c031f2eb17b7a9ba6dc8f775fb67 Mon Sep 17 00:00:00 2001 +From: Andrey Grodzovsky +Date: Fri, 30 Sep 2022 00:12:58 -0400 +Subject: [PATCH 031/150] drm/sched: Add FIFO sched policy to run queue +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When many entities are competing for the same run queue +on the same scheduler, we observe an unusually long wait +times and some jobs get starved. This has been observed on GPUVis. + +The issue is due to the Round Robin policy used by schedulers +to pick up the next entity's job queue for execution. Under stress +of many entities and long job queues within entity some +jobs could be stuck for very long time in it's entity's +queue before being popped from the queue and executed +while for other entities with smaller job queues a job +might execute earlier even though that job arrived later +then the job in the long queue. + +Fix: +Add FIFO selection policy to entities in run queue, chose next entity +on run queue in such order that if job on one entity arrived +earlier then job on another entity the first job will start +executing earlier regardless of the length of the entity's job +queue. + +v2: +Switch to rb tree structure for entities based on TS of +oldest job waiting in the job queue of an entity. Improves next +entity extraction to O(1). Entity TS update +O(log N) where N is the number of entities in the run-queue + +Drop default option in module control parameter. + +v3: +Various cosmetical fixes and minor refactoring of fifo update function. (Luben) + +v4: +Switch drm_sched_rq_select_entity_fifo to in order search (Luben) + +v5: Fix up drm_sched_rq_select_entity_fifo loop (Luben) + +v6: Add missing drm_sched_rq_remove_fifo_locked + +v7: Fix ts sampling bug and more cosmetic stuff (Luben) + +v8: Fix module parameter string (Luben) + +Cc: Luben Tuikov +Cc: Christian König +Cc: Direct Rendering Infrastructure - Development +Cc: AMD Graphics +Signed-off-by: Andrey Grodzovsky +Tested-by: Yunxiang Li (Teddy) +Signed-off-by: Luben Tuikov +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20220930041258.1050247-1-luben.tuikov@amd.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 20 +++++ + drivers/gpu/drm/scheduler/sched_main.c | 96 +++++++++++++++++++++++- + include/drm/gpu_scheduler.h | 32 ++++++++ + 3 files changed, 145 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 4b913dbb7d7b6b..c97bc1149663b4 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -73,6 +73,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + entity->priority = priority; + entity->sched_list = num_sched_list > 1 ? sched_list : NULL; + entity->last_scheduled = NULL; ++ RB_CLEAR_NODE(&entity->rb_tree_node); + + if(num_sched_list) + entity->rq = &sched_list[0]->sched_rq[entity->priority]; +@@ -448,6 +449,19 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) + smp_wmb(); + + spsc_queue_pop(&entity->job_queue); ++ ++ /* ++ * Update the entity's location in the min heap according to ++ * the timestamp of the next job, if any. ++ */ ++ if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) { ++ struct drm_sched_job *next; ++ ++ next = to_drm_sched_job(spsc_queue_peek(&entity->job_queue)); ++ if (next) ++ drm_sched_rq_update_fifo(entity, next->submit_ts); ++ } ++ + return sched_job; + } + +@@ -512,6 +526,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + atomic_inc(entity->rq->sched->score); + WRITE_ONCE(entity->last_user, current->group_leader); + first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); ++ sched_job->submit_ts = ktime_get(); + + /* first job wakes up scheduler */ + if (first) { +@@ -523,8 +538,13 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + DRM_ERROR("Trying to push to a killed entity\n"); + return; + } ++ + drm_sched_rq_add_entity(entity->rq, entity); + spin_unlock(&entity->rq_lock); ++ ++ if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) ++ drm_sched_rq_update_fifo(entity, sched_job->submit_ts); ++ + drm_sched_wakeup(entity->rq->sched); + } + } +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index f138b3be1646f6..3c97231a5d9483 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -62,6 +62,55 @@ + #define to_drm_sched_job(sched_job) \ + container_of((sched_job), struct drm_sched_job, queue_node) + ++int drm_sched_policy = DRM_SCHED_POLICY_RR; ++ ++/** ++ * DOC: sched_policy (int) ++ * Used to override default entities scheduling policy in a run queue. ++ */ ++MODULE_PARM_DESC(sched_policy, "Specify schedule policy for entities on a runqueue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin (default), " __stringify(DRM_SCHED_POLICY_FIFO) " = use FIFO."); ++module_param_named(sched_policy, drm_sched_policy, int, 0444); ++ ++static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a, ++ const struct rb_node *b) ++{ ++ struct drm_sched_entity *ent_a = rb_entry((a), struct drm_sched_entity, rb_tree_node); ++ struct drm_sched_entity *ent_b = rb_entry((b), struct drm_sched_entity, rb_tree_node); ++ ++ return ktime_before(ent_a->oldest_job_waiting, ent_b->oldest_job_waiting); ++} ++ ++static inline void drm_sched_rq_remove_fifo_locked(struct drm_sched_entity *entity) ++{ ++ struct drm_sched_rq *rq = entity->rq; ++ ++ if (!RB_EMPTY_NODE(&entity->rb_tree_node)) { ++ rb_erase_cached(&entity->rb_tree_node, &rq->rb_tree_root); ++ RB_CLEAR_NODE(&entity->rb_tree_node); ++ } ++} ++ ++void drm_sched_rq_update_fifo(struct drm_sched_entity *entity, ktime_t ts) ++{ ++ /* ++ * Both locks need to be grabbed, one to protect from entity->rq change ++ * for entity from within concurrent drm_sched_entity_select_rq and the ++ * other to update the rb tree structure. ++ */ ++ spin_lock(&entity->rq_lock); ++ spin_lock(&entity->rq->lock); ++ ++ drm_sched_rq_remove_fifo_locked(entity); ++ ++ entity->oldest_job_waiting = ts; ++ ++ rb_add_cached(&entity->rb_tree_node, &entity->rq->rb_tree_root, ++ drm_sched_entity_compare_before); ++ ++ spin_unlock(&entity->rq->lock); ++ spin_unlock(&entity->rq_lock); ++} ++ + /** + * drm_sched_rq_init - initialize a given run queue struct + * +@@ -75,6 +124,7 @@ static void drm_sched_rq_init(struct drm_gpu_scheduler *sched, + { + spin_lock_init(&rq->lock); + INIT_LIST_HEAD(&rq->entities); ++ rq->rb_tree_root = RB_ROOT_CACHED; + rq->current_entity = NULL; + rq->sched = sched; + } +@@ -92,9 +142,12 @@ void drm_sched_rq_add_entity(struct drm_sched_rq *rq, + { + if (!list_empty(&entity->list)) + return; ++ + spin_lock(&rq->lock); ++ + atomic_inc(rq->sched->score); + list_add_tail(&entity->list, &rq->entities); ++ + spin_unlock(&rq->lock); + } + +@@ -111,23 +164,30 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, + { + if (list_empty(&entity->list)) + return; ++ + spin_lock(&rq->lock); ++ + atomic_dec(rq->sched->score); + list_del_init(&entity->list); ++ + if (rq->current_entity == entity) + rq->current_entity = NULL; ++ ++ if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) ++ drm_sched_rq_remove_fifo_locked(entity); ++ + spin_unlock(&rq->lock); + } + + /** +- * drm_sched_rq_select_entity - Select an entity which could provide a job to run ++ * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run + * + * @rq: scheduler run queue to check. + * + * Try to find a ready entity, returns NULL if none found. + */ + static struct drm_sched_entity * +-drm_sched_rq_select_entity(struct drm_sched_rq *rq) ++drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) + { + struct drm_sched_entity *entity; + +@@ -163,6 +223,34 @@ drm_sched_rq_select_entity(struct drm_sched_rq *rq) + return NULL; + } + ++/** ++ * drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run ++ * ++ * @rq: scheduler run queue to check. ++ * ++ * Find oldest waiting ready entity, returns NULL if none found. ++ */ ++static struct drm_sched_entity * ++drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) ++{ ++ struct rb_node *rb; ++ ++ spin_lock(&rq->lock); ++ for (rb = rb_first_cached(&rq->rb_tree_root); rb; rb = rb_next(rb)) { ++ struct drm_sched_entity *entity; ++ ++ entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node); ++ if (drm_sched_entity_is_ready(entity)) { ++ rq->current_entity = entity; ++ reinit_completion(&entity->entity_idle); ++ break; ++ } ++ } ++ spin_unlock(&rq->lock); ++ ++ return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL; ++} ++ + /** + * drm_sched_job_done - complete a job + * @s_job: pointer to the job which is done +@@ -803,7 +891,9 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + + /* Kernel run queue has higher priority than normal run queue*/ + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { +- entity = drm_sched_rq_select_entity(&sched->sched_rq[i]); ++ entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? ++ drm_sched_rq_select_entity_fifo(&sched->sched_rq[i]) : ++ drm_sched_rq_select_entity_rr(&sched->sched_rq[i]); + if (entity) + break; + } +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 17e7e3145a0583..4d2bab37712cc6 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -58,6 +58,12 @@ enum drm_sched_priority { + DRM_SCHED_PRIORITY_COUNT + }; + ++/* Used to chose between FIFO and RR jobs scheduling */ ++extern int drm_sched_policy; ++ ++#define DRM_SCHED_POLICY_RR 0 ++#define DRM_SCHED_POLICY_FIFO 1 ++ + /** + * struct drm_sched_entity - A wrapper around a job queue (typically + * attached to the DRM file_priv). +@@ -204,6 +210,21 @@ struct drm_sched_entity { + * drm_sched_entity_fini(). + */ + struct completion entity_idle; ++ ++ /** ++ * @oldest_job_waiting: ++ * ++ * Marks earliest job waiting in SW queue ++ */ ++ ktime_t oldest_job_waiting; ++ ++ /** ++ * @rb_tree_node: ++ * ++ * The node used to insert this entity into time based priority queue ++ */ ++ struct rb_node rb_tree_node; ++ + }; + + /** +@@ -213,6 +234,7 @@ struct drm_sched_entity { + * @sched: the scheduler to which this rq belongs to. + * @entities: list of the entities to be scheduled. + * @current_entity: the entity which is to be scheduled. ++ * @rb_tree_root: root of time based priory queue of entities for FIFO scheduling + * + * Run queue is a set of entities scheduling command submissions for + * one specific ring. It implements the scheduling policy that selects +@@ -223,6 +245,7 @@ struct drm_sched_rq { + struct drm_gpu_scheduler *sched; + struct list_head entities; + struct drm_sched_entity *current_entity; ++ struct rb_root_cached rb_tree_root; + }; + + /** +@@ -322,6 +345,13 @@ struct drm_sched_job { + + /** @last_dependency: tracks @dependencies as they signal */ + unsigned long last_dependency; ++ ++ /** ++ * @submit_ts: ++ * ++ * When the job was pushed into the entity queue. ++ */ ++ ktime_t submit_ts; + }; + + static inline bool drm_sched_invalidate_job(struct drm_sched_job *s_job, +@@ -511,6 +541,8 @@ void drm_sched_rq_add_entity(struct drm_sched_rq *rq, + void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, + struct drm_sched_entity *entity); + ++void drm_sched_rq_update_fifo(struct drm_sched_entity *entity, ktime_t ts); ++ + int drm_sched_entity_init(struct drm_sched_entity *entity, + enum drm_sched_priority priority, + struct drm_gpu_scheduler **sched_list, + +From 00999a35e1ea42113626de5edcfc20f59a3306ed Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Mon, 24 Oct 2022 17:26:34 -0400 +Subject: [PATCH 032/150] drm/scheduler: Set the FIFO scheduling policy as the + default +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The currently default Round-Robin GPU scheduling can result in starvation +of entities which have a large number of jobs, over entities which have +a very small number of jobs (single digit). + +This can be illustrated in the following diagram, where jobs are +alphabetized to show their chronological order of arrival, where job A is +the oldest, B is the second oldest, and so on, to J, the most recent job to +arrive. + + ---> entities +j | H-F-----A--E--I-- +o | --G-----B-----J-- +b | --------C-------- +s\/ --------D-------- + +WLOG, assuming all jobs are "ready", then a R-R scheduling will execute them +in the following order (a slice off of the top of the entities' list), + +H, F, A, E, I, G, B, J, C, D. + +However, to mitigate job starvation, we'd rather execute C and D before E, +and so on, given, of course, that they're all ready to be executed. + +So, if all jobs are ready at this instant, the order of execution for this +and the next 9 instances of picking the next job to execute, should really +be, + +A, B, C, D, E, F, G, H, I, J, + +which is their chronological order. The only reason for this order to be +broken, is if an older job is not yet ready, but a younger job is ready, at +an instant of picking a new job to execute. For instance if job C wasn't +ready at time 2, but job D was ready, then we'd pick job D, like this: + +0 +1 +2 ... +A, B, D, ... + +And from then on, C would be preferred before all other jobs, if it is ready +at the time when a new job for execution is picked. So, if C became ready +two steps later, the execution order would look like this: + +......0 +1 +2 ... +A, B, D, E, C, F, G, H, I, J + +This is what the FIFO GPU scheduling algorithm achieves. It uses a +Red-Black tree to keep jobs sorted in chronological order, where picking +the oldest job is O(1) (we use the "cached" structure), and balancing the +tree is O(log n). IOW, it picks the *oldest ready* job to execute now. + +The implementation is already in the kernel, and this commit only changes +the default GPU scheduling algorithm to use. + +This was tested and achieves about 1% faster performance over the Round +Robin algorithm. + +Cc: Christian König +Cc: Alex Deucher +Cc: Direct Rendering Infrastructure - Development +Signed-off-by: Luben Tuikov +Reviewed-by: Christian König +Link: https://patchwork.freedesktop.org/patch/msgid/20221024212634.27230-1-luben.tuikov@amd.com +Signed-off-by: Christian König +--- + drivers/gpu/drm/scheduler/sched_main.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 3c97231a5d9483..feb4cf75fea94d 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -62,13 +62,13 @@ + #define to_drm_sched_job(sched_job) \ + container_of((sched_job), struct drm_sched_job, queue_node) + +-int drm_sched_policy = DRM_SCHED_POLICY_RR; ++int drm_sched_policy = DRM_SCHED_POLICY_FIFO; + + /** + * DOC: sched_policy (int) + * Used to override default entities scheduling policy in a run queue. + */ +-MODULE_PARM_DESC(sched_policy, "Specify schedule policy for entities on a runqueue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin (default), " __stringify(DRM_SCHED_POLICY_FIFO) " = use FIFO."); ++MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default)."); + module_param_named(sched_policy, drm_sched_policy, int, 0444); + + static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a, + +From 39949223782f8944230e86a991fb0b68e0d6de23 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Wed, 28 Sep 2022 10:17:40 +0200 +Subject: [PATCH 033/150] drm/scheduler: add + drm_sched_job_add_resv_dependencies +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Add a new function to update job dependencies from a resv obj. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20221014084641.128280-3-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 49 ++++++++++++++++++-------- + include/drm/gpu_scheduler.h | 5 +++ + 2 files changed, 39 insertions(+), 15 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index feb4cf75fea94d..aea90136a511e1 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -773,32 +773,28 @@ int drm_sched_job_add_dependency(struct drm_sched_job *job, + EXPORT_SYMBOL(drm_sched_job_add_dependency); + + /** +- * drm_sched_job_add_implicit_dependencies - adds implicit dependencies as job +- * dependencies ++ * drm_sched_job_add_resv_dependencies - add all fences from the resv to the job + * @job: scheduler job to add the dependencies to +- * @obj: the gem object to add new dependencies from. +- * @write: whether the job might write the object (so we need to depend on +- * shared fences in the reservation object). ++ * @resv: the dma_resv object to get the fences from ++ * @usage: the dma_resv_usage to use to filter the fences + * +- * This should be called after drm_gem_lock_reservations() on your array of +- * GEM objects used in the job but before updating the reservations with your +- * own fences. ++ * This adds all fences matching the given usage from @resv to @job. ++ * Must be called with the @resv lock held. + * + * Returns: + * 0 on success, or an error on failing to expand the array. + */ +-int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job, +- struct drm_gem_object *obj, +- bool write) ++int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job, ++ struct dma_resv *resv, ++ enum dma_resv_usage usage) + { + struct dma_resv_iter cursor; + struct dma_fence *fence; + int ret; + +- dma_resv_assert_held(obj->resv); ++ dma_resv_assert_held(resv); + +- dma_resv_for_each_fence(&cursor, obj->resv, dma_resv_usage_rw(write), +- fence) { ++ dma_resv_for_each_fence(&cursor, resv, usage, fence) { + /* Make sure to grab an additional ref on the added fence */ + dma_fence_get(fence); + ret = drm_sched_job_add_dependency(job, fence); +@@ -809,8 +805,31 @@ int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job, + } + return 0; + } +-EXPORT_SYMBOL(drm_sched_job_add_implicit_dependencies); ++EXPORT_SYMBOL(drm_sched_job_add_resv_dependencies); + ++/** ++ * drm_sched_job_add_implicit_dependencies - adds implicit dependencies as job ++ * dependencies ++ * @job: scheduler job to add the dependencies to ++ * @obj: the gem object to add new dependencies from. ++ * @write: whether the job might write the object (so we need to depend on ++ * shared fences in the reservation object). ++ * ++ * This should be called after drm_gem_lock_reservations() on your array of ++ * GEM objects used in the job but before updating the reservations with your ++ * own fences. ++ * ++ * Returns: ++ * 0 on success, or an error on failing to expand the array. ++ */ ++int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job, ++ struct drm_gem_object *obj, ++ bool write) ++{ ++ return drm_sched_job_add_resv_dependencies(job, obj->resv, ++ dma_resv_usage_rw(write)); ++} ++EXPORT_SYMBOL(drm_sched_job_add_implicit_dependencies); + + /** + * drm_sched_job_cleanup - clean up scheduler job resources +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 4d2bab37712cc6..22066e339acd2e 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -41,6 +41,8 @@ + */ + #define DRM_SCHED_FENCE_DONT_PIPELINE DMA_FENCE_FLAG_USER_BITS + ++enum dma_resv_usage; ++struct dma_resv; + struct drm_gem_object; + + struct drm_gpu_scheduler; +@@ -513,6 +515,9 @@ int drm_sched_job_init(struct drm_sched_job *job, + void drm_sched_job_arm(struct drm_sched_job *job); + int drm_sched_job_add_dependency(struct drm_sched_job *job, + struct dma_fence *fence); ++int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job, ++ struct dma_resv *resv, ++ enum dma_resv_usage usage); + int drm_sched_job_add_implicit_dependencies(struct drm_sched_job *job, + struct drm_gem_object *obj, + bool write); + +From 1ebf730c97cb32996463bb50210b38becb137048 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Thu, 29 Sep 2022 14:08:13 +0200 +Subject: [PATCH 034/150] drm/scheduler: remove drm_sched_dependency_optimized +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Not used any more. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20221014084641.128280-12-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 26 -------------------------- + include/drm/gpu_scheduler.h | 2 -- + 2 files changed, 28 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index aea90136a511e1..0367f200762cb6 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -285,32 +285,6 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb) + drm_sched_job_done(s_job); + } + +-/** +- * drm_sched_dependency_optimized - test if the dependency can be optimized +- * +- * @fence: the dependency fence +- * @entity: the entity which depends on the above fence +- * +- * Returns true if the dependency can be optimized and false otherwise +- */ +-bool drm_sched_dependency_optimized(struct dma_fence* fence, +- struct drm_sched_entity *entity) +-{ +- struct drm_gpu_scheduler *sched = entity->rq->sched; +- struct drm_sched_fence *s_fence; +- +- if (!fence || dma_fence_is_signaled(fence)) +- return false; +- if (fence->context == entity->fence_context) +- return true; +- s_fence = to_drm_sched_fence(fence); +- if (s_fence && s_fence->sched == sched) +- return true; +- +- return false; +-} +-EXPORT_SYMBOL(drm_sched_dependency_optimized); +- + /** + * drm_sched_start_timeout - start timeout for reset worker + * +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 22066e339acd2e..18a733a715a590 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -536,8 +536,6 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max); + void drm_sched_increase_karma(struct drm_sched_job *bad); + void drm_sched_reset_karma(struct drm_sched_job *bad); + void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type); +-bool drm_sched_dependency_optimized(struct dma_fence* fence, +- struct drm_sched_entity *entity); + void drm_sched_fault(struct drm_gpu_scheduler *sched); + void drm_sched_job_kickout(struct drm_sched_job *s_job); + + +From 244dd3d6cb8a6517e381b91090a2f66b3a422075 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Thu, 29 Sep 2022 14:50:56 +0200 +Subject: [PATCH 035/150] drm/scheduler: rework entity flush, kill and fini +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This was buggy because when we had to wait for entities which were +killed as well we would just deadlock. + +Instead move all the dependency handling into the callbacks so that +will all happen asynchronously. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20221014084641.128280-13-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 196 +++++++++++------------ + 1 file changed, 91 insertions(+), 105 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index c97bc1149663b4..111d68180a0d60 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -140,6 +140,73 @@ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) + return true; + } + ++static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) ++{ ++ struct drm_sched_job *job = container_of(wrk, typeof(*job), work); ++ ++ drm_sched_fence_finished(job->s_fence); ++ WARN_ON(job->s_fence->parent); ++ job->sched->ops->free_job(job); ++} ++ ++/* Signal the scheduler finished fence when the entity in question is killed. */ ++static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, ++ struct dma_fence_cb *cb) ++{ ++ struct drm_sched_job *job = container_of(cb, struct drm_sched_job, ++ finish_cb); ++ int r; ++ ++ dma_fence_put(f); ++ ++ /* Wait for all dependencies to avoid data corruptions */ ++ while (!xa_empty(&job->dependencies)) { ++ f = xa_erase(&job->dependencies, job->last_dependency++); ++ r = dma_fence_add_callback(f, &job->finish_cb, ++ drm_sched_entity_kill_jobs_cb); ++ if (!r) ++ return; ++ ++ dma_fence_put(f); ++ } ++ ++ INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work); ++ schedule_work(&job->work); ++} ++ ++/* Remove the entity from the scheduler and kill all pending jobs */ ++static void drm_sched_entity_kill(struct drm_sched_entity *entity) ++{ ++ struct drm_sched_job *job; ++ struct dma_fence *prev; ++ ++ if (!entity->rq) ++ return; ++ ++ spin_lock(&entity->rq_lock); ++ entity->stopped = true; ++ drm_sched_rq_remove_entity(entity->rq, entity); ++ spin_unlock(&entity->rq_lock); ++ ++ /* Make sure this entity is not used by the scheduler at the moment */ ++ wait_for_completion(&entity->entity_idle); ++ ++ prev = dma_fence_get(entity->last_scheduled); ++ while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { ++ struct drm_sched_fence *s_fence = job->s_fence; ++ ++ dma_fence_set_error(&s_fence->finished, -ESRCH); ++ ++ dma_fence_get(&s_fence->finished); ++ if (!prev || dma_fence_add_callback(prev, &job->finish_cb, ++ drm_sched_entity_kill_jobs_cb)) ++ drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); ++ ++ prev = &s_fence->finished; ++ } ++ dma_fence_put(prev); ++} ++ + /** + * drm_sched_entity_flush - Flush a context entity + * +@@ -180,91 +247,13 @@ long drm_sched_entity_flush(struct drm_sched_entity *entity, long timeout) + /* For killed process disable any more IBs enqueue right now */ + last_user = cmpxchg(&entity->last_user, current->group_leader, NULL); + if ((!last_user || last_user == current->group_leader) && +- (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) { +- spin_lock(&entity->rq_lock); +- entity->stopped = true; +- drm_sched_rq_remove_entity(entity->rq, entity); +- spin_unlock(&entity->rq_lock); +- } ++ (current->flags & PF_EXITING) && (current->exit_code == SIGKILL)) ++ drm_sched_entity_kill(entity); + + return ret; + } + EXPORT_SYMBOL(drm_sched_entity_flush); + +-static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) +-{ +- struct drm_sched_job *job = container_of(wrk, typeof(*job), work); +- +- drm_sched_fence_finished(job->s_fence); +- WARN_ON(job->s_fence->parent); +- job->sched->ops->free_job(job); +-} +- +- +-/* Signal the scheduler finished fence when the entity in question is killed. */ +-static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, +- struct dma_fence_cb *cb) +-{ +- struct drm_sched_job *job = container_of(cb, struct drm_sched_job, +- finish_cb); +- +- dma_fence_put(f); +- INIT_WORK(&job->work, drm_sched_entity_kill_jobs_work); +- schedule_work(&job->work); +-} +- +-static struct dma_fence * +-drm_sched_job_dependency(struct drm_sched_job *job, +- struct drm_sched_entity *entity) +-{ +- if (!xa_empty(&job->dependencies)) +- return xa_erase(&job->dependencies, job->last_dependency++); +- +- if (job->sched->ops->dependency) +- return job->sched->ops->dependency(job, entity); +- +- return NULL; +-} +- +-static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) +-{ +- struct drm_sched_job *job; +- struct dma_fence *f; +- int r; +- +- while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { +- struct drm_sched_fence *s_fence = job->s_fence; +- +- /* Wait for all dependencies to avoid data corruptions */ +- while ((f = drm_sched_job_dependency(job, entity))) { +- dma_fence_wait(f, false); +- dma_fence_put(f); +- } +- +- drm_sched_fence_scheduled(s_fence); +- dma_fence_set_error(&s_fence->finished, -ESRCH); +- +- /* +- * When pipe is hanged by older entity, new entity might +- * not even have chance to submit it's first job to HW +- * and so entity->last_scheduled will remain NULL +- */ +- if (!entity->last_scheduled) { +- drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); +- continue; +- } +- +- dma_fence_get(entity->last_scheduled); +- r = dma_fence_add_callback(entity->last_scheduled, +- &job->finish_cb, +- drm_sched_entity_kill_jobs_cb); +- if (r == -ENOENT) +- drm_sched_entity_kill_jobs_cb(NULL, &job->finish_cb); +- else if (r) +- DRM_ERROR("fence add callback failed (%d)\n", r); +- } +-} +- + /** + * drm_sched_entity_fini - Destroy a context entity + * +@@ -278,33 +267,17 @@ static void drm_sched_entity_kill_jobs(struct drm_sched_entity *entity) + */ + void drm_sched_entity_fini(struct drm_sched_entity *entity) + { +- struct drm_gpu_scheduler *sched = NULL; +- +- if (entity->rq) { +- sched = entity->rq->sched; +- drm_sched_rq_remove_entity(entity->rq, entity); +- } +- +- /* Consumption of existing IBs wasn't completed. Forcefully +- * remove them here. ++ /* ++ * If consumption of existing IBs wasn't completed. Forcefully remove ++ * them here. Also makes sure that the scheduler won't touch this entity ++ * any more. + */ +- if (spsc_queue_count(&entity->job_queue)) { +- if (sched) { +- /* +- * Wait for thread to idle to make sure it isn't processing +- * this entity. +- */ +- wait_for_completion(&entity->entity_idle); +- +- } +- if (entity->dependency) { +- dma_fence_remove_callback(entity->dependency, +- &entity->cb); +- dma_fence_put(entity->dependency); +- entity->dependency = NULL; +- } ++ drm_sched_entity_kill(entity); + +- drm_sched_entity_kill_jobs(entity); ++ if (entity->dependency) { ++ dma_fence_remove_callback(entity->dependency, &entity->cb); ++ dma_fence_put(entity->dependency); ++ entity->dependency = NULL; + } + + dma_fence_put(entity->last_scheduled); +@@ -417,6 +390,19 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) + return false; + } + ++static struct dma_fence * ++drm_sched_job_dependency(struct drm_sched_job *job, ++ struct drm_sched_entity *entity) ++{ ++ if (!xa_empty(&job->dependencies)) ++ return xa_erase(&job->dependencies, job->last_dependency++); ++ ++ if (job->sched->ops->dependency) ++ return job->sched->ops->dependency(job, entity); ++ ++ return NULL; ++} ++ + struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) + { + struct drm_sched_job *sched_job; + +From 7e0f0d23685f051b69ca7c2693c2e9055a0c3788 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Thu, 29 Sep 2022 15:01:57 +0200 +Subject: [PATCH 036/150] drm/scheduler: rename dependency callback into + prepare_job +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This now matches much better what this is doing. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20221014084641.128280-14-christian.koenig@amd.com +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 5 +++-- + drivers/gpu/drm/scheduler/sched_entity.c | 4 ++-- + include/drm/gpu_scheduler.h | 13 ++++++------- + 3 files changed, 11 insertions(+), 11 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +index 3bf0e893c07dfb..2a4daf64a58534 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +@@ -248,7 +248,8 @@ int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, + return 0; + } + +-static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job, ++static struct dma_fence * ++amdgpu_job_prepare_job(struct drm_sched_job *sched_job, + struct drm_sched_entity *s_entity) + { + struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->rq->sched); +@@ -351,7 +352,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) + } + + const struct drm_sched_backend_ops amdgpu_sched_ops = { +- .dependency = amdgpu_job_dependency, ++ .prepare_job = amdgpu_job_prepare_job, + .run_job = amdgpu_job_run, + .timedout_job = amdgpu_job_timedout, + .free_job = amdgpu_job_free_cb +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 111d68180a0d60..fe09e5be79bddc 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -397,8 +397,8 @@ drm_sched_job_dependency(struct drm_sched_job *job, + if (!xa_empty(&job->dependencies)) + return xa_erase(&job->dependencies, job->last_dependency++); + +- if (job->sched->ops->dependency) +- return job->sched->ops->dependency(job, entity); ++ if (job->sched->ops->prepare_job) ++ return job->sched->ops->prepare_job(job, entity); + + return NULL; + } +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 18a733a715a590..8ca6cd55618198 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -328,7 +328,7 @@ struct drm_sched_job { + */ + union { + struct dma_fence_cb finish_cb; +- struct work_struct work; ++ struct work_struct work; + }; + + uint64_t id; +@@ -376,18 +376,17 @@ enum drm_gpu_sched_stat { + */ + struct drm_sched_backend_ops { + /** +- * @dependency: ++ * @prepare_job: + * + * Called when the scheduler is considering scheduling this job next, to + * get another struct dma_fence for this job to block on. Once it + * returns NULL, run_job() may be called. + * +- * If a driver exclusively uses drm_sched_job_add_dependency() and +- * drm_sched_job_add_implicit_dependencies() this can be ommitted and +- * left as NULL. ++ * Can be NULL if no additional preparation to the dependencies are ++ * necessary. Skipped when jobs are killed instead of run. + */ +- struct dma_fence *(*dependency)(struct drm_sched_job *sched_job, +- struct drm_sched_entity *s_entity); ++ struct dma_fence *(*prepare_job)(struct drm_sched_job *sched_job, ++ struct drm_sched_entity *s_entity); + + /** + * @run_job: Called to execute the job once all of the dependencies + +From fb79e758c6065618d8d8d33cde83c430485fce7a Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Wed, 26 Oct 2022 12:26:37 +0200 +Subject: [PATCH 037/150] drm/amdgpu: revert "implement tdr advanced mode" +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This reverts commit e6c6338f393b74ac0b303d567bb918b44ae7ad75. + +This feature basically re-submits one job after another to +figure out which one was the one causing a hang. + +This is obviously incompatible with gang-submit which requires +that multiple jobs run at the same time. It's also absolutely +not helpful to crash the hardware multiple times if a clean +recovery is desired. + +For testing and debugging environments we should rather disable +recovery alltogether to be able to inspect the state with a hw +debugger. + +Additional to that the sw implementation is clearly buggy and causes +reference count issues for the hardware fence. + +Signed-off-by: Christian König +Reviewed-by: Alex Deucher +Signed-off-by: Alex Deucher +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 103 --------------------- + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 58 ++---------- + include/drm/gpu_scheduler.h | 3 - + 4 files changed, 10 insertions(+), 156 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +index a5352e5e2bd476..e357e1632dbf7f 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +@@ -5176,94 +5176,6 @@ static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev) + return 0; + } + +-static void amdgpu_device_recheck_guilty_jobs( +- struct amdgpu_device *adev, struct list_head *device_list_handle, +- struct amdgpu_reset_context *reset_context) +-{ +- int i, r = 0; +- +- for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { +- struct amdgpu_ring *ring = adev->rings[i]; +- int ret = 0; +- struct drm_sched_job *s_job; +- +- if (!ring || !ring->sched.thread) +- continue; +- +- s_job = list_first_entry_or_null(&ring->sched.pending_list, +- struct drm_sched_job, list); +- if (s_job == NULL) +- continue; +- +- /* clear job's guilty and depend the folowing step to decide the real one */ +- drm_sched_reset_karma(s_job); +- drm_sched_resubmit_jobs_ext(&ring->sched, 1); +- +- if (!s_job->s_fence->parent) { +- DRM_WARN("Failed to get a HW fence for job!"); +- continue; +- } +- +- ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout); +- if (ret == 0) { /* timeout */ +- DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n", +- ring->sched.name, s_job->id); +- +- +- amdgpu_fence_driver_isr_toggle(adev, true); +- +- /* Clear this failed job from fence array */ +- amdgpu_fence_driver_clear_job_fences(ring); +- +- amdgpu_fence_driver_isr_toggle(adev, false); +- +- /* Since the job won't signal and we go for +- * another resubmit drop this parent pointer +- */ +- dma_fence_put(s_job->s_fence->parent); +- s_job->s_fence->parent = NULL; +- +- /* set guilty */ +- drm_sched_increase_karma(s_job); +- amdgpu_reset_prepare_hwcontext(adev, reset_context); +-retry: +- /* do hw reset */ +- if (amdgpu_sriov_vf(adev)) { +- amdgpu_virt_fini_data_exchange(adev); +- r = amdgpu_device_reset_sriov(adev, false); +- if (r) +- adev->asic_reset_res = r; +- } else { +- clear_bit(AMDGPU_SKIP_HW_RESET, +- &reset_context->flags); +- r = amdgpu_do_asic_reset(device_list_handle, +- reset_context); +- if (r && r == -EAGAIN) +- goto retry; +- } +- +- /* +- * add reset counter so that the following +- * resubmitted job could flush vmid +- */ +- atomic_inc(&adev->gpu_reset_counter); +- continue; +- } +- +- /* got the hw fence, signal finished fence */ +- atomic_dec(ring->sched.score); +- dma_fence_get(&s_job->s_fence->finished); +- dma_fence_signal(&s_job->s_fence->finished); +- dma_fence_put(&s_job->s_fence->finished); +- +- /* remove node from list and free the job */ +- spin_lock(&ring->sched.job_list_lock); +- list_del_init(&s_job->list); +- spin_unlock(&ring->sched.job_list_lock); +- ring->sched.ops->free_job(s_job); +- } +-} +- + static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) + { + struct amdgpu_ras *con = amdgpu_ras_get_context(adev); +@@ -5284,7 +5196,6 @@ static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev) + + } + +- + /** + * amdgpu_device_gpu_recover - reset the asic and recover scheduler + * +@@ -5307,7 +5218,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + int i, r = 0; + bool need_emergency_restart = false; + bool audio_suspended = false; +- int tmp_vram_lost_counter; + bool gpu_reset_for_dev_remove = false; + + gpu_reset_for_dev_remove = +@@ -5454,7 +5364,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + amdgpu_device_stop_pending_resets(tmp_adev); + } + +- tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter)); + /* Actual ASIC resets if needed.*/ + /* Host driver will handle XGMI hive reset for SRIOV */ + if (amdgpu_sriov_vf(adev)) { +@@ -5479,18 +5388,6 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + /* Post ASIC reset for all devs .*/ + list_for_each_entry(tmp_adev, device_list_handle, reset_list) { + +- /* +- * Sometimes a later bad compute job can block a good gfx job as gfx +- * and compute ring share internal GC HW mutually. We add an additional +- * guilty jobs recheck step to find the real guilty job, it synchronously +- * submits and pends for the first job being signaled. If it gets timeout, +- * we identify it as a real guilty job. +- */ +- if (amdgpu_gpu_recovery == 2 && +- !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter))) +- amdgpu_device_recheck_guilty_jobs( +- tmp_adev, device_list_handle, reset_context); +- + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = tmp_adev->rings[i]; + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +index a8e1f2cfe12dcd..be4d1aeef47b44 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +@@ -534,7 +534,7 @@ module_param_named(compute_multipipe, amdgpu_compute_multipipe, int, 0444); + * DOC: gpu_recovery (int) + * Set to enable GPU recovery mechanism (1 = enable, 0 = disable). The default is -1 (auto, disabled except SRIOV). + */ +-MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (2 = advanced tdr mode, 1 = enable, 0 = disable, -1 = auto)"); ++MODULE_PARM_DESC(gpu_recovery, "Enable GPU recovery mechanism, (1 = enable, 0 = disable, -1 = auto)"); + module_param_named(gpu_recovery, amdgpu_gpu_recovery, int, 0444); + + /** +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 0367f200762cb6..c79571406d98bb 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -417,27 +417,6 @@ static void drm_sched_job_timedout(struct work_struct *work) + } + } + +- /** +- * drm_sched_increase_karma - Update sched_entity guilty flag +- * +- * @bad: The job guilty of time out +- * +- * Increment on every hang caused by the 'bad' job. If this exceeds the hang +- * limit of the scheduler then the respective sched entity is marked guilty and +- * jobs from it will not be scheduled further +- */ +-void drm_sched_increase_karma(struct drm_sched_job *bad) +-{ +- drm_sched_increase_karma_ext(bad, 1); +-} +-EXPORT_SYMBOL(drm_sched_increase_karma); +- +-void drm_sched_reset_karma(struct drm_sched_job *bad) +-{ +- drm_sched_increase_karma_ext(bad, 0); +-} +-EXPORT_SYMBOL(drm_sched_reset_karma); +- + /** + * drm_sched_stop - stop the scheduler + * +@@ -578,32 +557,15 @@ EXPORT_SYMBOL(drm_sched_start); + * + */ + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) +-{ +- drm_sched_resubmit_jobs_ext(sched, INT_MAX); +-} +-EXPORT_SYMBOL(drm_sched_resubmit_jobs); +- +-/** +- * drm_sched_resubmit_jobs_ext - helper to relunch certain number of jobs from mirror ring list +- * +- * @sched: scheduler instance +- * @max: job numbers to relaunch +- * +- */ +-void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) + { + struct drm_sched_job *s_job, *tmp; + uint64_t guilty_context; + bool found_guilty = false; + struct dma_fence *fence; +- int i = 0; + + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { + struct drm_sched_fence *s_fence = s_job->s_fence; + +- if (i >= max) +- break; +- + if (!found_guilty && atomic_read(&s_job->karma) > sched->hang_limit) { + found_guilty = true; + guilty_context = s_job->s_fence->scheduled.context; +@@ -613,7 +575,6 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) + dma_fence_set_error(&s_fence->finished, -ECANCELED); + + fence = sched->ops->run_job(s_job); +- i++; + + if (IS_ERR_OR_NULL(fence)) { + if (IS_ERR(fence)) +@@ -629,7 +590,7 @@ void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max) + } + } + } +-EXPORT_SYMBOL(drm_sched_resubmit_jobs_ext); ++EXPORT_SYMBOL(drm_sched_resubmit_jobs); + + /** + * drm_sched_job_init - init a scheduler job +@@ -1165,13 +1126,15 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + EXPORT_SYMBOL(drm_sched_fini); + + /** +- * drm_sched_increase_karma_ext - Update sched_entity guilty flag ++ * drm_sched_increase_karma - Update sched_entity guilty flag + * + * @bad: The job guilty of time out +- * @type: type for increase/reset karma + * ++ * Increment on every hang caused by the 'bad' job. If this exceeds the hang ++ * limit of the scheduler then the respective sched entity is marked guilty and ++ * jobs from it will not be scheduled further + */ +-void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) ++void drm_sched_increase_karma(struct drm_sched_job *bad) + { + int i; + struct drm_sched_entity *tmp; +@@ -1183,10 +1146,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) + * corrupt but keep in mind that kernel jobs always considered good. + */ + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { +- if (type == 0) +- atomic_set(&bad->karma, 0); +- else if (type == 1) +- atomic_inc(&bad->karma); ++ atomic_inc(&bad->karma); + + for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; + i++) { +@@ -1197,7 +1157,7 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) + if (bad->s_fence->scheduled.context == + entity->fence_context) { + if (entity->guilty) +- atomic_set(entity->guilty, type); ++ atomic_set(entity->guilty, 1); + break; + } + } +@@ -1207,4 +1167,4 @@ void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type) + } + } + } +-EXPORT_SYMBOL(drm_sched_increase_karma_ext); ++EXPORT_SYMBOL(drm_sched_increase_karma); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 8ca6cd55618198..15edfe3ca45864 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -531,10 +531,7 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched); + void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad); + void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); +-void drm_sched_resubmit_jobs_ext(struct drm_gpu_scheduler *sched, int max); + void drm_sched_increase_karma(struct drm_sched_job *bad); +-void drm_sched_reset_karma(struct drm_sched_job *bad); +-void drm_sched_increase_karma_ext(struct drm_sched_job *bad, int type); + void drm_sched_fault(struct drm_gpu_scheduler *sched); + void drm_sched_job_kickout(struct drm_sched_job *s_job); + + +From 09849e3e087dee597ebfba29fa3aac44847704fd Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Wed, 23 Nov 2022 03:13:03 +0300 +Subject: [PATCH 038/150] drm/scheduler: Fix lockup in drm_sched_entity_kill() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The drm_sched_entity_kill() is invoked twice by drm_sched_entity_destroy() +while userspace process is exiting or being killed. First time it's invoked +when sched entity is flushed and second time when entity is released. This +causes a lockup within wait_for_completion(entity_idle) due to how completion +API works. + +Calling wait_for_completion() more times than complete() was invoked is a +error condition that causes lockup because completion internally uses +counter for complete/wait calls. The complete_all() must be used instead +in such cases. + +This patch fixes lockup of Panfrost driver that is reproducible by killing +any application in a middle of 3d drawing operation. + +Fixes: 2fdb8a8f07c2 ("drm/scheduler: rework entity flush, kill and fini") +Signed-off-by: Dmitry Osipenko +Reviewed-by: Christian König +Link: https://patchwork.freedesktop.org/patch/msgid/20221123001303.533968-1-dmitry.osipenko@collabora.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index fe09e5be79bddc..15d04a0ec62346 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -81,7 +81,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + init_completion(&entity->entity_idle); + + /* We start in an idle state. */ +- complete(&entity->entity_idle); ++ complete_all(&entity->entity_idle); + + spin_lock_init(&entity->rq_lock); + spsc_queue_init(&entity->job_queue); +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index c79571406d98bb..e3bee9b9efab60 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -987,7 +987,7 @@ static int drm_sched_main(void *param) + sched_job = drm_sched_entity_pop_job(entity); + + if (!sched_job) { +- complete(&entity->entity_idle); ++ complete_all(&entity->entity_idle); + continue; + } + +@@ -998,7 +998,7 @@ static int drm_sched_main(void *param) + + trace_drm_run_job(sched_job, entity); + fence = sched->ops->run_job(sched_job); +- complete(&entity->entity_idle); ++ complete_all(&entity->entity_idle); + drm_sched_fence_scheduled(s_fence); + + if (!IS_ERR_OR_NULL(fence)) { + +From eb5ea8f54f518d90cc5d3cec00d84393ba0d1ecf Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Wed, 26 Oct 2022 17:09:34 +0200 +Subject: [PATCH 039/150] drm/scheduler: deprecate drm_sched_resubmit_jobs +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This interface is not working as it should. + +Signed-off-by: Christian König +Reviewed-by: Alex Deucher +Link: https://patchwork.freedesktop.org/patch/msgid/20221109095010.141189-5-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 13 ++++++++++++- + 1 file changed, 12 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index e3bee9b9efab60..24de860fa0ca3e 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -551,10 +551,21 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + EXPORT_SYMBOL(drm_sched_start); + + /** +- * drm_sched_resubmit_jobs - helper to relaunch jobs from the pending list ++ * drm_sched_resubmit_jobs - Deprecated, don't use in new code! + * + * @sched: scheduler instance + * ++ * Re-submitting jobs was a concept AMD came up as cheap way to implement ++ * recovery after a job timeout. ++ * ++ * This turned out to be not working very well. First of all there are many ++ * problem with the dma_fence implementation and requirements. Either the ++ * implementation is risking deadlocks with core memory management or violating ++ * documented implementation details of the dma_fence object. ++ * ++ * Drivers can still save and restore their state for recovery operations, but ++ * we shouldn't make this a general scheduler feature around the dma_fence ++ * interface. + */ + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched) + { + +From a0048134fc716da660b3f5cbe3a3b5aacee8ea02 Mon Sep 17 00:00:00 2001 +From: Lucas Stach +Date: Wed, 1 Feb 2023 16:26:07 +0100 +Subject: [PATCH 040/150] drm/scheduler: track GPU active time per entity + +Track the accumulated time that jobs from this entity were active +on the GPU. This allows drivers using the scheduler to trivially +implement the DRM fdinfo when the hardware doesn't provide more +specific information than signalling job completion anyways. + +[Bagas: Append missing colon to @elapsed_ns] +Signed-off-by: Bagas Sanjaya +Signed-off-by: Lucas Stach +Reviewed-by: Andrey Grodzovsky +--- + drivers/gpu/drm/scheduler/sched_main.c | 6 ++++++ + include/drm/gpu_scheduler.h | 7 +++++++ + 2 files changed, 13 insertions(+) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 24de860fa0ca3e..774f51c1f8a3b4 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -906,6 +906,12 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) + + spin_unlock(&sched->job_list_lock); + ++ if (job) { ++ job->entity->elapsed_ns += ktime_to_ns( ++ ktime_sub(job->s_fence->finished.timestamp, ++ job->s_fence->scheduled.timestamp)); ++ } ++ + return job; + } + +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 15edfe3ca45864..0ae1c6f40a5ad9 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -227,6 +227,13 @@ struct drm_sched_entity { + */ + struct rb_node rb_tree_node; + ++ /** ++ * @elapsed_ns: ++ * ++ * Records the amount of time where jobs from this entity were active ++ * on the GPU. ++ */ ++ uint64_t elapsed_ns; + }; + + /** + +From 5f987234255fb75bb06cd1a0d2c68a4bcee43ba1 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Ma=C3=ADra=20Canal?= +Date: Thu, 9 Feb 2023 09:44:44 -0300 +Subject: [PATCH 041/150] drm/sched: Create wrapper to add a syncobj dependency + to job +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In order to add a syncobj's fence as a dependency to a job, it is +necessary to call drm_syncobj_find_fence() to find the fence and then +add the dependency with drm_sched_job_add_dependency(). So, wrap these +steps in one single function, drm_sched_job_add_syncobj_dependency(). + +Reviewed-by: Christian König +Reviewed-by: Luben Tuikov +Signed-off-by: Maíra Canal +Signed-off-by: Maíra Canal +Link: https://patchwork.freedesktop.org/patch/msgid/20230209124447.467867-2-mcanal@igalia.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 29 ++++++++++++++++++++++++++ + include/drm/gpu_scheduler.h | 6 ++++++ + 2 files changed, 35 insertions(+) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 774f51c1f8a3b4..1dcd9679f32877 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -53,6 +53,7 @@ + + #include + #include ++#include + #include + #include + +@@ -718,6 +719,34 @@ int drm_sched_job_add_dependency(struct drm_sched_job *job, + } + EXPORT_SYMBOL(drm_sched_job_add_dependency); + ++/** ++ * drm_sched_job_add_syncobj_dependency - adds a syncobj's fence as a job dependency ++ * @job: scheduler job to add the dependencies to ++ * @file_private: drm file private pointer ++ * @handle: syncobj handle to lookup ++ * @point: timeline point ++ * ++ * This adds the fence matching the given syncobj to @job. ++ * ++ * Returns: ++ * 0 on success, or an error on failing to expand the array. ++ */ ++int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job, ++ struct drm_file *file, ++ u32 handle, ++ u32 point) ++{ ++ struct dma_fence *fence; ++ int ret; ++ ++ ret = drm_syncobj_find_fence(file, handle, point, 0, &fence); ++ if (ret) ++ return ret; ++ ++ return drm_sched_job_add_dependency(job, fence); ++} ++EXPORT_SYMBOL(drm_sched_job_add_syncobj_dependency); ++ + /** + * drm_sched_job_add_resv_dependencies - add all fences from the resv to the job + * @job: scheduler job to add the dependencies to +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 0ae1c6f40a5ad9..61bf69d6451fbf 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -48,6 +48,8 @@ struct drm_gem_object; + struct drm_gpu_scheduler; + struct drm_sched_rq; + ++struct drm_file; ++ + /* These are often used as an (initial) index + * to an array, and as such should start at 0. + */ +@@ -521,6 +523,10 @@ int drm_sched_job_init(struct drm_sched_job *job, + void drm_sched_job_arm(struct drm_sched_job *job); + int drm_sched_job_add_dependency(struct drm_sched_job *job, + struct dma_fence *fence); ++int drm_sched_job_add_syncobj_dependency(struct drm_sched_job *job, ++ struct drm_file *file, ++ u32 handle, ++ u32 point); + int drm_sched_job_add_resv_dependencies(struct drm_sched_job *job, + struct dma_resv *resv, + enum dma_resv_usage usage); + +From d5684a096ab2d900e85c3823de0011fddfd11f36 Mon Sep 17 00:00:00 2001 +From: Caio Novais +Date: Sat, 25 Mar 2023 10:15:32 -0300 +Subject: [PATCH 042/150] drm/scheduler: Fix variable name in function + description + +Compiling AMD GPU drivers displays two warnings: + +drivers/gpu/drm/scheduler/sched_main.c:738: warning: Function parameter or member 'file' not described in 'drm_sched_job_add_syncobj_dependency' +drivers/gpu/drm/scheduler/sched_main.c:738: warning: Excess function +parameter 'file_private' description in +'drm_sched_job_add_syncobj_dependency' + +Get rid of them by renaming the variable name on the function description + +Signed-off-by: Caio Novais +Link: https://lore.kernel.org/r/20230325131532.6356-1-caionovais@usp.br +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 1dcd9679f32877..c2b01fa8514aa6 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -722,7 +722,7 @@ EXPORT_SYMBOL(drm_sched_job_add_dependency); + /** + * drm_sched_job_add_syncobj_dependency - adds a syncobj's fence as a job dependency + * @job: scheduler job to add the dependencies to +- * @file_private: drm file private pointer ++ * @file: drm file private pointer + * @handle: syncobj handle to lookup + * @point: timeline point + * + +From b1cd5cd9b93c77291eca3018cfad9d02cca16b4e Mon Sep 17 00:00:00 2001 +From: Rob Clark +Date: Tue, 21 Sep 2021 09:35:50 -0700 +Subject: [PATCH 043/150] drm/scheduler: Add fence deadline support + +As the finished fence is the one that is exposed to userspace, and +therefore the one that other operations, like atomic update, would +block on, we need to propagate the deadline from from the finished +fence to the actual hw fence. + +v2: Split into drm_sched_fence_set_parent() (ckoenig) +v3: Ensure a thread calling drm_sched_fence_set_deadline_finished() sees + fence->parent set before drm_sched_fence_set_parent() does this + test_bit(DMA_FENCE_FLAG_HAS_DEADLINE_BIT). + +Signed-off-by: Rob Clark +Acked-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_fence.c | 46 +++++++++++++++++++++++++ + drivers/gpu/drm/scheduler/sched_main.c | 2 +- + include/drm/gpu_scheduler.h | 17 +++++++++ + 3 files changed, 64 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c +index 7fd869520ef2cd..fe9c6468e44019 100644 +--- a/drivers/gpu/drm/scheduler/sched_fence.c ++++ b/drivers/gpu/drm/scheduler/sched_fence.c +@@ -123,6 +123,37 @@ static void drm_sched_fence_release_finished(struct dma_fence *f) + dma_fence_put(&fence->scheduled); + } + ++static void drm_sched_fence_set_deadline_finished(struct dma_fence *f, ++ ktime_t deadline) ++{ ++ struct drm_sched_fence *fence = to_drm_sched_fence(f); ++ struct dma_fence *parent; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&fence->lock, flags); ++ ++ /* If we already have an earlier deadline, keep it: */ ++ if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags) && ++ ktime_before(fence->deadline, deadline)) { ++ spin_unlock_irqrestore(&fence->lock, flags); ++ return; ++ } ++ ++ fence->deadline = deadline; ++ set_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, &f->flags); ++ ++ spin_unlock_irqrestore(&fence->lock, flags); ++ ++ /* ++ * smp_load_aquire() to ensure that if we are racing another ++ * thread calling drm_sched_fence_set_parent(), that we see ++ * the parent set before it calls test_bit(HAS_DEADLINE_BIT) ++ */ ++ parent = smp_load_acquire(&fence->parent); ++ if (parent) ++ dma_fence_set_deadline(parent, deadline); ++} ++ + static const struct dma_fence_ops drm_sched_fence_ops_scheduled = { + .get_driver_name = drm_sched_fence_get_driver_name, + .get_timeline_name = drm_sched_fence_get_timeline_name, +@@ -133,6 +164,7 @@ static const struct dma_fence_ops drm_sched_fence_ops_finished = { + .get_driver_name = drm_sched_fence_get_driver_name, + .get_timeline_name = drm_sched_fence_get_timeline_name, + .release = drm_sched_fence_release_finished, ++ .set_deadline = drm_sched_fence_set_deadline_finished, + }; + + struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f) +@@ -147,6 +179,20 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f) + } + EXPORT_SYMBOL(to_drm_sched_fence); + ++void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, ++ struct dma_fence *fence) ++{ ++ /* ++ * smp_store_release() to ensure another thread racing us ++ * in drm_sched_fence_set_deadline_finished() sees the ++ * fence's parent set before test_bit() ++ */ ++ smp_store_release(&s_fence->parent, dma_fence_get(fence)); ++ if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, ++ &s_fence->finished.flags)) ++ dma_fence_set_deadline(fence, s_fence->deadline); ++} ++ + struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *entity, + void *owner) + { +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index c2b01fa8514aa6..99350aa97c6fa6 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1048,7 +1048,7 @@ static int drm_sched_main(void *param) + drm_sched_fence_scheduled(s_fence); + + if (!IS_ERR_OR_NULL(fence)) { +- s_fence->parent = dma_fence_get(fence); ++ drm_sched_fence_set_parent(s_fence, fence); + /* Drop for original kref_init of the fence */ + dma_fence_put(fence); + +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 61bf69d6451fbf..baf630a19e42d6 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -41,6 +41,15 @@ + */ + #define DRM_SCHED_FENCE_DONT_PIPELINE DMA_FENCE_FLAG_USER_BITS + ++/** ++ * DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT - A fence deadline hint has been set ++ * ++ * Because we could have a deadline hint can be set before the backing hw ++ * fence is created, we need to keep track of whether a deadline has already ++ * been set. ++ */ ++#define DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT (DMA_FENCE_FLAG_USER_BITS + 1) ++ + enum dma_resv_usage; + struct dma_resv; + struct drm_gem_object; +@@ -281,6 +290,12 @@ struct drm_sched_fence { + */ + struct dma_fence finished; + ++ /** ++ * @deadline: deadline set on &drm_sched_fence.finished which ++ * potentially needs to be propagated to &drm_sched_fence.parent ++ */ ++ ktime_t deadline; ++ + /** + * @parent: the fence returned by &drm_sched_backend_ops.run_job + * when scheduling the job on hardware. We signal the +@@ -570,6 +585,8 @@ void drm_sched_entity_set_priority(struct drm_sched_entity *entity, + enum drm_sched_priority priority); + bool drm_sched_entity_is_ready(struct drm_sched_entity *entity); + ++void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, ++ struct dma_fence *fence); + struct drm_sched_fence *drm_sched_fence_alloc( + struct drm_sched_entity *s_entity, void *owner); + void drm_sched_fence_init(struct drm_sched_fence *fence, + +From 7a7e2e5d6a66cccf25d98437995f4bf6e9b43f24 Mon Sep 17 00:00:00 2001 +From: Lucas Stach +Date: Thu, 30 Mar 2023 17:35:13 +0200 +Subject: [PATCH 044/150] Revert "drm/scheduler: track GPU active time per + entity" + +This reverts commit df622729ddbf as it introduces a use-after-free, +which isn't easy to fix without going back to the design drawing board. + +Reported-by: Danilo Krummrich +Signed-off-by: Lucas Stach +--- + drivers/gpu/drm/scheduler/sched_main.c | 6 ------ + include/drm/gpu_scheduler.h | 7 ------- + 2 files changed, 13 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 99350aa97c6fa6..c534df9293c8bc 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -935,12 +935,6 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) + + spin_unlock(&sched->job_list_lock); + +- if (job) { +- job->entity->elapsed_ns += ktime_to_ns( +- ktime_sub(job->s_fence->finished.timestamp, +- job->s_fence->scheduled.timestamp)); +- } +- + return job; + } + +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index baf630a19e42d6..65facabdab9502 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -238,13 +238,6 @@ struct drm_sched_entity { + */ + struct rb_node rb_tree_node; + +- /** +- * @elapsed_ns: +- * +- * Records the amount of time where jobs from this entity were active +- * on the GPU. +- */ +- uint64_t elapsed_ns; + }; + + /** + +From cd0c34be1e7337dbe52721d00bbd1e6466efc73f Mon Sep 17 00:00:00 2001 +From: Asahi Lina +Date: Thu, 6 Apr 2023 01:37:39 +0900 +Subject: [PATCH 045/150] drm/scheduler: Fix UAF race in + drm_sched_entity_push_job() + +After a job is pushed into the queue, it is owned by the scheduler core +and may be freed at any time, so we can't write nor read the submit +timestamp after that point. + +Fixes oopses observed with the drm/asahi driver, found with kASAN. + +Signed-off-by: Asahi Lina +Link: https://lore.kernel.org/r/20230406-scheduler-uaf-2-v1-1-972531cf0a81@asahilina.net +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_entity.c | 11 +++++++++-- + 1 file changed, 9 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 15d04a0ec62346..e0a8890a62e23a 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -507,12 +507,19 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + { + struct drm_sched_entity *entity = sched_job->entity; + bool first; ++ ktime_t submit_ts; + + trace_drm_sched_job(sched_job, entity); + atomic_inc(entity->rq->sched->score); + WRITE_ONCE(entity->last_user, current->group_leader); ++ ++ /* ++ * After the sched_job is pushed into the entity queue, it may be ++ * completed and freed up at any time. We can no longer access it. ++ * Make sure to set the submit_ts first, to avoid a race. ++ */ ++ sched_job->submit_ts = submit_ts = ktime_get(); + first = spsc_queue_push(&entity->job_queue, &sched_job->queue_node); +- sched_job->submit_ts = ktime_get(); + + /* first job wakes up scheduler */ + if (first) { +@@ -529,7 +536,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + spin_unlock(&entity->rq_lock); + + if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) +- drm_sched_rq_update_fifo(entity, sched_job->submit_ts); ++ drm_sched_rq_update_fifo(entity, submit_ts); + + drm_sched_wakeup(entity->rq->sched); + } + +From eadcbb20a3f7347f7e061f15b7848bee0bb5a0aa Mon Sep 17 00:00:00 2001 +From: Vitaly Prosyak +Date: Thu, 6 Apr 2023 16:00:54 -0400 +Subject: [PATCH 046/150] drm/sched: Check scheduler ready before calling + timeout handling +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +During an IGT GPU reset test we see the following oops, + +[ +0.000003] ------------[ cut here ]------------ +[ +0.000000] WARNING: CPU: 9 PID: 0 at kernel/workqueue.c:1656 __queue_delayed_work+0x6d/0xa0 +[ +0.000004] Modules linked in: iptable_filter bpfilter amdgpu(OE) nls_iso8859_1 snd_hda_codec_realtek snd_hda_codec_generic intel_rapl_msr ledtrig_audio snd_hda_codec_hdmi intel_rapl_common snd_hda_intel edac_mce_amd snd_intel_dspcfg snd_intel_sdw_acpi snd_hda_codec snd_hda_core iommu_v2 gpu_sched(OE) kvm_amd drm_buddy snd_hwdep kvm video drm_ttm_helper snd_pcm ttm snd_seq_midi drm_display_helper snd_seq_midi_event snd_rawmidi cec crct10dif_pclmul ghash_clmulni_intel sha512_ssse3 snd_seq aesni_intel rc_core crypto_simd cryptd binfmt_misc drm_kms_helper rapl snd_seq_device input_leds joydev snd_timer i2c_algo_bit syscopyarea snd ccp sysfillrect sysimgblt wmi_bmof k10temp soundcore mac_hid sch_fq_codel msr parport_pc ppdev drm lp parport ramoops reed_solomon pstore_blk pstore_zone efi_pstore ip_tables x_tables autofs4 hid_generic usbhid hid r8169 ahci xhci_pci gpio_amdpt realtek i2c_piix4 wmi crc32_pclmul xhci_pci_renesas libahci gpio_generic +[ +0.000070] CPU: 9 PID: 0 Comm: swapper/9 Tainted: G W OE 6.1.11+ #2 +[ +0.000003] Hardware name: Gigabyte Technology Co., Ltd. AB350-Gaming 3/AB350-Gaming 3-CF, BIOS F7 06/16/2017 +[ +0.000001] RIP: 0010:__queue_delayed_work+0x6d/0xa0 +[ +0.000003] Code: 7a 50 48 01 c1 48 89 4a 30 81 ff 00 20 00 00 75 38 4c 89 cf e8 64 3e 0a 00 5d e9 1e c5 11 01 e8 99 f7 ff ff 5d e9 13 c5 11 01 <0f> 0b eb c1 0f 0b 48 81 7a 38 70 5c 0e 81 74 9f 0f 0b 48 8b 42 28 +[ +0.000002] RSP: 0018:ffffc90000398d60 EFLAGS: 00010007 +[ +0.000002] RAX: ffff88810d589c60 RBX: 0000000000000000 RCX: 0000000000000000 +[ +0.000002] RDX: ffff88810d589c58 RSI: 0000000000000000 RDI: 0000000000002000 +[ +0.000001] RBP: ffffc90000398d60 R08: 0000000000000000 R09: ffff88810d589c78 +[ +0.000002] R10: 72705f305f39765f R11: 7866673a6d72645b R12: ffff88810d589c58 +[ +0.000001] R13: 0000000000002000 R14: 0000000000000000 R15: 0000000000000000 +[ +0.000002] FS: 0000000000000000(0000) GS:ffff8887fee40000(0000) knlGS:0000000000000000 +[ +0.000001] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ +0.000002] CR2: 00005562c4797fa0 CR3: 0000000110da0000 CR4: 00000000003506e0 +[ +0.000002] Call Trace: +[ +0.000001] +[ +0.000001] mod_delayed_work_on+0x5e/0xa0 +[ +0.000004] drm_sched_fault+0x23/0x30 [gpu_sched] +[ +0.000007] gfx_v9_0_fault.isra.0+0xa6/0xd0 [amdgpu] +[ +0.000258] gfx_v9_0_priv_reg_irq+0x29/0x40 [amdgpu] +[ +0.000254] amdgpu_irq_dispatch+0x1ac/0x2b0 [amdgpu] +[ +0.000243] amdgpu_ih_process+0x89/0x130 [amdgpu] +[ +0.000245] amdgpu_irq_handler+0x24/0x60 [amdgpu] +[ +0.000165] __handle_irq_event_percpu+0x4f/0x1a0 +[ +0.000003] handle_irq_event_percpu+0x15/0x50 +[ +0.000001] handle_irq_event+0x39/0x60 +[ +0.000002] handle_edge_irq+0xa8/0x250 +[ +0.000003] __common_interrupt+0x7b/0x150 +[ +0.000002] common_interrupt+0xc1/0xe0 +[ +0.000003] +[ +0.000000] +[ +0.000001] asm_common_interrupt+0x27/0x40 +[ +0.000002] RIP: 0010:native_safe_halt+0xb/0x10 +[ +0.000003] Code: 46 ff ff ff cc cc cc cc cc cc cc cc cc cc cc eb 07 0f 00 2d 69 f2 5e 00 f4 e9 f1 3b 3e 00 90 eb 07 0f 00 2d 59 f2 5e 00 fb f4 e0 3b 3e 00 0f 1f 44 00 00 55 48 89 e5 53 e8 b1 d4 fe ff 66 90 +[ +0.000002] RSP: 0018:ffffc9000018fdc8 EFLAGS: 00000246 +[ +0.000002] RAX: 0000000000004000 RBX: 000000000002e5a8 RCX: 000000000000001f +[ +0.000001] RDX: 0000000000000001 RSI: ffff888101298800 RDI: ffff888101298864 +[ +0.000001] RBP: ffffc9000018fdd0 R08: 000000527f64bd8b R09: 000000000001dc90 +[ +0.000001] R10: 000000000001dc90 R11: 0000000000000003 R12: 0000000000000001 +[ +0.000001] R13: ffff888101298864 R14: ffffffff832d9e20 R15: ffff888193aa8c00 +[ +0.000003] ? acpi_idle_do_entry+0x5e/0x70 +[ +0.000002] acpi_idle_enter+0xd1/0x160 +[ +0.000003] cpuidle_enter_state+0x9a/0x6e0 +[ +0.000003] cpuidle_enter+0x2e/0x50 +[ +0.000003] call_cpuidle+0x23/0x50 +[ +0.000002] do_idle+0x1de/0x260 +[ +0.000002] cpu_startup_entry+0x20/0x30 +[ +0.000002] start_secondary+0x120/0x150 +[ +0.000003] secondary_startup_64_no_verify+0xe5/0xeb +[ +0.000004] +[ +0.000000] ---[ end trace 0000000000000000 ]--- +[ +0.000003] BUG: kernel NULL pointer dereference, address: 0000000000000102 +[ +0.006233] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* ring gfx_low timeout, signaled seq=3, emitted seq=4 +[ +0.000734] #PF: supervisor read access in kernel mode +[ +0.009670] [drm:amdgpu_job_timedout [amdgpu]] *ERROR* Process information: process amd_deadlock pid 2002 thread amd_deadlock pid 2002 +[ +0.005135] #PF: error_code(0x0000) - not-present page +[ +0.000002] PGD 0 P4D 0 +[ +0.000002] Oops: 0000 [#1] PREEMPT SMP NOPTI +[ +0.000002] CPU: 9 PID: 0 Comm: swapper/9 Tainted: G W OE 6.1.11+ #2 +[ +0.000002] Hardware name: Gigabyte Technology Co., Ltd. AB350-Gaming 3/AB350-Gaming 3-CF, BIOS F7 06/16/2017 +[ +0.012101] amdgpu 0000:0c:00.0: amdgpu: GPU reset begin! +[ +0.005136] RIP: 0010:__queue_work+0x1f/0x4e0 +[ +0.000004] Code: 87 cd 11 01 0f 1f 80 00 00 00 00 0f 1f 44 00 00 55 48 89 e5 41 57 41 56 41 55 49 89 d5 41 54 49 89 f4 53 48 83 ec 10 89 7d d4 86 02 01 00 00 01 0f 85 6c 03 00 00 e8 7f 36 08 00 8b 45 d4 48 + +For gfx_rings the schedulers may not be initialized by +amdgpu_device_init_schedulers() due to ring->no_scheduler flag being set to +true and thus the timeout_wq is NULL. As a result, since all ASICs call +drm_sched_fault() unconditionally even for schedulers which have not been +initialized, it is simpler to use the ready condition which indicates whether +the given scheduler worker thread runs and whether the timeout_wq of the reset +domain has been initialized. + +Signed-off-by: Vitaly Prosyak +Cc: Christian König +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +Link: https://lore.kernel.org/r/20230406200054.633379-1-luben.tuikov@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index c534df9293c8bc..989fced0a97844 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -309,7 +309,8 @@ static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched) + */ + void drm_sched_fault(struct drm_gpu_scheduler *sched) + { +- mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0); ++ if (sched->ready) ++ mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0); + } + EXPORT_SYMBOL(drm_sched_fault); + + +From c1855668c09acdfaf8b996eeb92761ce5704260c Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Tue, 18 Apr 2023 12:04:53 +0200 +Subject: [PATCH 047/150] drm/scheduler: set entity to NULL in + drm_sched_entity_pop_job() + +It already happend a few times that patches slipped through which +implemented access to an entity through a job that was already removed +from the entities queue. Since jobs and entities might have different +lifecycles, this can potentially cause UAF bugs. + +In order to make it obvious that a jobs entity pointer shouldn't be +accessed after drm_sched_entity_pop_job() was called successfully, set +the jobs entity pointer to NULL once the job is removed from the entity +queue. + +Moreover, debugging a potential NULL pointer dereference is way easier +than potentially corrupted memory through a UAF. + +Signed-off-by: Danilo Krummrich +Link: https://lore.kernel.org/r/20230418100453.4433-1-dakr@redhat.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_entity.c | 6 ++++++ + drivers/gpu/drm/scheduler/sched_main.c | 4 ++++ + 2 files changed, 10 insertions(+) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index e0a8890a62e23a..3e2a31d8190eba 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -448,6 +448,12 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) + drm_sched_rq_update_fifo(entity, next->submit_ts); + } + ++ /* Jobs and entities might have different lifecycles. Since we're ++ * removing the job from the entities queue, set the jobs entity pointer ++ * to NULL to prevent any future access of the entity through this job. ++ */ ++ sched_job->entity = NULL; ++ + return sched_job; + } + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 989fced0a97844..1f33cd2b556226 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -42,6 +42,10 @@ + * the hardware. + * + * The jobs in a entity are always scheduled in the order that they were pushed. ++ * ++ * Note that once a job was taken from the entities queue and pushed to the ++ * hardware, i.e. the pending queue, the entity must not be referenced anymore ++ * through the jobs entity pointer. + */ + + #include + +From dc88bbd33cd09b1f3c8b5fc810fbe0aeb6bee287 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Mon, 17 Apr 2023 13:36:02 +0200 +Subject: [PATCH 048/150] drm/scheduler: properly forward fence errors +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When a hw fence is signaled with an error properly forward that to the +finished fence. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20230420115752.31470-1-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 4 +--- + drivers/gpu/drm/scheduler/sched_fence.c | 4 +++- + drivers/gpu/drm/scheduler/sched_main.c | 18 ++++++++---------- + include/drm/gpu_scheduler.h | 2 +- + 4 files changed, 13 insertions(+), 15 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 3e2a31d8190eba..1795cd7e42ed73 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -144,7 +144,7 @@ static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) + { + struct drm_sched_job *job = container_of(wrk, typeof(*job), work); + +- drm_sched_fence_finished(job->s_fence); ++ drm_sched_fence_finished(job->s_fence, -ESRCH); + WARN_ON(job->s_fence->parent); + job->sched->ops->free_job(job); + } +@@ -195,8 +195,6 @@ static void drm_sched_entity_kill(struct drm_sched_entity *entity) + while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { + struct drm_sched_fence *s_fence = job->s_fence; + +- dma_fence_set_error(&s_fence->finished, -ESRCH); +- + dma_fence_get(&s_fence->finished); + if (!prev || dma_fence_add_callback(prev, &job->finish_cb, + drm_sched_entity_kill_jobs_cb)) +diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c +index fe9c6468e44019..ef120475e7c618 100644 +--- a/drivers/gpu/drm/scheduler/sched_fence.c ++++ b/drivers/gpu/drm/scheduler/sched_fence.c +@@ -53,8 +53,10 @@ void drm_sched_fence_scheduled(struct drm_sched_fence *fence) + dma_fence_signal(&fence->scheduled); + } + +-void drm_sched_fence_finished(struct drm_sched_fence *fence) ++void drm_sched_fence_finished(struct drm_sched_fence *fence, int result) + { ++ if (result) ++ dma_fence_set_error(&fence->finished, result); + dma_fence_signal(&fence->finished); + } + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 1f33cd2b556226..16018d4c7e3911 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -262,7 +262,7 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) + * + * Finish the job's fence and wake up the worker thread. + */ +-static void drm_sched_job_done(struct drm_sched_job *s_job) ++static void drm_sched_job_done(struct drm_sched_job *s_job, int result) + { + struct drm_sched_fence *s_fence = s_job->s_fence; + struct drm_gpu_scheduler *sched = s_fence->sched; +@@ -273,7 +273,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job) + trace_drm_sched_process_job(s_fence); + + dma_fence_get(&s_fence->finished); +- drm_sched_fence_finished(s_fence); ++ drm_sched_fence_finished(s_fence, result); + dma_fence_put(&s_fence->finished); + wake_up_interruptible(&sched->wake_up_worker); + } +@@ -287,7 +287,7 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb) + { + struct drm_sched_job *s_job = container_of(cb, struct drm_sched_job, cb); + +- drm_sched_job_done(s_job); ++ drm_sched_job_done(s_job, f->error); + } + + /** +@@ -538,12 +538,12 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + r = dma_fence_add_callback(fence, &s_job->cb, + drm_sched_job_done_cb); + if (r == -ENOENT) +- drm_sched_job_done(s_job); ++ drm_sched_job_done(s_job, fence->error); + else if (r) + DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", + r); + } else +- drm_sched_job_done(s_job); ++ drm_sched_job_done(s_job, 0); + } + + if (full_recovery) { +@@ -1054,15 +1054,13 @@ static int drm_sched_main(void *param) + r = dma_fence_add_callback(fence, &sched_job->cb, + drm_sched_job_done_cb); + if (r == -ENOENT) +- drm_sched_job_done(sched_job); ++ drm_sched_job_done(sched_job, fence->error); + else if (r) + DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", + r); + } else { +- if (IS_ERR(fence)) +- dma_fence_set_error(&s_fence->finished, PTR_ERR(fence)); +- +- drm_sched_job_done(sched_job); ++ drm_sched_job_done(sched_job, IS_ERR(fence) ? ++ PTR_ERR(fence) : 0); + } + + wake_up(&sched->job_scheduled); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 65facabdab9502..43eabb29295b87 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -587,7 +587,7 @@ void drm_sched_fence_init(struct drm_sched_fence *fence, + void drm_sched_fence_free(struct drm_sched_fence *fence); + + void drm_sched_fence_scheduled(struct drm_sched_fence *fence); +-void drm_sched_fence_finished(struct drm_sched_fence *fence); ++void drm_sched_fence_finished(struct drm_sched_fence *fence, int result); + + unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched); + void drm_sched_resume_timeout(struct drm_gpu_scheduler *sched, + +From afed5f41b3206f9b7c52e5a1f16cb892b1d2ca58 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Mon, 17 Apr 2023 17:32:11 +0200 +Subject: [PATCH 049/150] drm/scheduler: add drm_sched_entity_error and use rcu + for last_scheduled +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Switch to using RCU handling for the last scheduled job and add a +function to return the error code of it. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20230420115752.31470-2-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 39 +++++++++++++++++++----- + include/drm/gpu_scheduler.h | 3 +- + 2 files changed, 33 insertions(+), 9 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 1795cd7e42ed73..cfb433e9200586 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -72,7 +72,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + entity->num_sched_list = num_sched_list; + entity->priority = priority; + entity->sched_list = num_sched_list > 1 ? sched_list : NULL; +- entity->last_scheduled = NULL; ++ RCU_INIT_POINTER(entity->last_scheduled, NULL); + RB_CLEAR_NODE(&entity->rb_tree_node); + + if(num_sched_list) +@@ -140,6 +140,27 @@ bool drm_sched_entity_is_ready(struct drm_sched_entity *entity) + return true; + } + ++/** ++ * drm_sched_entity_error - return error of last scheduled job ++ * @entity: scheduler entity to check ++ * ++ * Opportunistically return the error of the last scheduled job. Result can ++ * change any time when new jobs are pushed to the hw. ++ */ ++int drm_sched_entity_error(struct drm_sched_entity *entity) ++{ ++ struct dma_fence *fence; ++ int r; ++ ++ rcu_read_lock(); ++ fence = rcu_dereference(entity->last_scheduled); ++ r = fence ? fence->error : 0; ++ rcu_read_unlock(); ++ ++ return r; ++} ++EXPORT_SYMBOL(drm_sched_entity_error); ++ + static void drm_sched_entity_kill_jobs_work(struct work_struct *wrk) + { + struct drm_sched_job *job = container_of(wrk, typeof(*job), work); +@@ -191,7 +212,9 @@ static void drm_sched_entity_kill(struct drm_sched_entity *entity) + /* Make sure this entity is not used by the scheduler at the moment */ + wait_for_completion(&entity->entity_idle); + +- prev = dma_fence_get(entity->last_scheduled); ++ /* The entity is guaranteed to not be used by the scheduler */ ++ prev = rcu_dereference_check(entity->last_scheduled, true); ++ dma_fence_get(prev); + while ((job = to_drm_sched_job(spsc_queue_pop(&entity->job_queue)))) { + struct drm_sched_fence *s_fence = job->s_fence; + +@@ -278,8 +301,8 @@ void drm_sched_entity_fini(struct drm_sched_entity *entity) + entity->dependency = NULL; + } + +- dma_fence_put(entity->last_scheduled); +- entity->last_scheduled = NULL; ++ dma_fence_put(rcu_dereference_check(entity->last_scheduled, true)); ++ RCU_INIT_POINTER(entity->last_scheduled, NULL); + } + EXPORT_SYMBOL(drm_sched_entity_fini); + +@@ -421,9 +444,9 @@ struct drm_sched_job *drm_sched_entity_pop_job(struct drm_sched_entity *entity) + if (entity->guilty && atomic_read(entity->guilty)) + dma_fence_set_error(&sched_job->s_fence->finished, -ECANCELED); + +- dma_fence_put(entity->last_scheduled); +- +- entity->last_scheduled = dma_fence_get(&sched_job->s_fence->finished); ++ dma_fence_put(rcu_dereference_check(entity->last_scheduled, true)); ++ rcu_assign_pointer(entity->last_scheduled, ++ dma_fence_get(&sched_job->s_fence->finished)); + + /* + * If the queue is empty we allow drm_sched_entity_select_rq() to +@@ -477,7 +500,7 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) + */ + smp_rmb(); + +- fence = entity->last_scheduled; ++ fence = rcu_dereference_check(entity->last_scheduled, true); + + /* stay on the same engine if the previous job hasn't finished */ + if (fence && !dma_fence_is_signaled(fence)) +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 43eabb29295b87..098321741b311e 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -200,7 +200,7 @@ struct drm_sched_entity { + * by the scheduler thread, can be accessed locklessly from + * drm_sched_job_arm() iff the queue is empty. + */ +- struct dma_fence *last_scheduled; ++ struct dma_fence __rcu *last_scheduled; + + /** + * @last_user: last group leader pushing a job into the entity. +@@ -577,6 +577,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job); + void drm_sched_entity_set_priority(struct drm_sched_entity *entity, + enum drm_sched_priority priority); + bool drm_sched_entity_is_ready(struct drm_sched_entity *entity); ++int drm_sched_entity_error(struct drm_sched_entity *entity); + + void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, + struct dma_fence *fence); + +From 8c885797796afea6deb9deeca58305e6ef927204 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Thu, 27 Apr 2023 14:05:43 +0200 +Subject: [PATCH 050/150] drm/scheduler: mark jobs without fence as canceled +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When no hw fence is provided for a job that means that the job didn't executed. + +Signed-off-by: Christian König +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20230427122726.1290170-1-christian.koenig@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 16018d4c7e3911..11fc1e6da89026 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -543,7 +543,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", + r); + } else +- drm_sched_job_done(s_job, 0); ++ drm_sched_job_done(s_job, -ECANCELED); + } + + if (full_recovery) { + +From fd17736ad539e09056c6829fc07b97995f2b51eb Mon Sep 17 00:00:00 2001 +From: Vitaly Prosyak +Date: Wed, 10 May 2023 09:51:11 -0400 +Subject: [PATCH 051/150] drm/sched: Check scheduler work queue before calling + timeout handling + +During an IGT GPU reset test we see again oops despite of +commit 0c8c901aaaebc9 (drm/sched: Check scheduler ready before calling +timeout handling). + +It uses ready condition whether to call drm_sched_fault which unwind +the TDR leads to GPU reset. +However it looks the ready condition is overloaded with other meanings, +for example, for the following stack is related GPU reset : + +0 gfx_v9_0_cp_gfx_start +1 gfx_v9_0_cp_gfx_resume +2 gfx_v9_0_cp_resume +3 gfx_v9_0_hw_init +4 gfx_v9_0_resume +5 amdgpu_device_ip_resume_phase2 + +does the following: + /* start the ring */ + gfx_v9_0_cp_gfx_start(adev); + ring->sched.ready = true; + +The same approach is for other ASICs as well : +gfx_v8_0_cp_gfx_resume +gfx_v10_0_kiq_resume, etc... + +As a result, our GPU reset test causes GPU fault which calls unconditionally gfx_v9_0_fault +and then drm_sched_fault. However now it depends on whether the interrupt service routine +drm_sched_fault is executed after gfx_v9_0_cp_gfx_start is completed which sets the ready +field of the scheduler to true even for uninitialized schedulers and causes oops vs +no fault or when ISR drm_sched_fault is completed prior gfx_v9_0_cp_gfx_start and +NULL pointer dereference does not occur. + +Use the field timeout_wq to prevent oops for uninitialized schedulers. +The field could be initialized by the work queue of resetting the domain. + +v1: Corrections to commit message (Luben) + +Fixes: 11b3b9f461c5c4 ("drm/sched: Check scheduler ready before calling timeout handling") +Signed-off-by: Vitaly Prosyak +Link: https://lore.kernel.org/r/20230510135111.58631-1-vitaly.prosyak@amd.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 11fc1e6da89026..ecc76e9512c908 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -313,7 +313,7 @@ static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched) + */ + void drm_sched_fault(struct drm_gpu_scheduler *sched) + { +- if (sched->ready) ++ if (sched->timeout_wq) + mod_delayed_work(sched->timeout_wq, &sched->work_tdr, 0); + } + EXPORT_SYMBOL(drm_sched_fault); + +From b593d93f3d37f2fd1109d76a378e56100d489196 Mon Sep 17 00:00:00 2001 +From: Vladislav Efanov +Date: Wed, 17 May 2023 15:52:47 +0300 +Subject: [PATCH 052/150] drm/sched: Remove redundant check + +The rq pointer points inside the drm_gpu_scheduler structure. Thus +it can't be NULL. + +Found by Linux Verification Center (linuxtesting.org) with SVACE. + +Fixes: c61cdbdbffc1 ("drm/scheduler: Fix hang when sched_entity released") +Signed-off-by: Vladislav Efanov +Link: https://lore.kernel.org/r/20230517125247.434103-1-VEfanov@ispras.ru +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 3 --- + 1 file changed, 3 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index ecc76e9512c908..43f0ff9ea3d405 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1143,9 +1143,6 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + struct drm_sched_rq *rq = &sched->sched_rq[i]; + +- if (!rq) +- continue; +- + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) + /* + +From d8824c10130154548081a9d1469441b0fd00fc7c Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Wed, 17 May 2023 19:35:49 -0400 +Subject: [PATCH 053/150] drm/sched: Rename to drm_sched_can_queue() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Rename drm_sched_ready() to drm_sched_can_queue(). "ready" can mean many +things and is thus meaningless in this context. Instead, rename to a name +which precisely conveys what is being checked. + +Cc: Christian König +Cc: Alex Deucher +Signed-off-by: Luben Tuikov +Reviewed-by: Alex Deucher +Link: https://lore.kernel.org/r/20230517233550.377847-1-luben.tuikov@amd.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 43f0ff9ea3d405..6aec61b1b3e304 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -848,13 +848,12 @@ void drm_sched_job_cleanup(struct drm_sched_job *job) + EXPORT_SYMBOL(drm_sched_job_cleanup); + + /** +- * drm_sched_ready - is the scheduler ready +- * ++ * drm_sched_can_queue -- Can we queue more to the hardware? + * @sched: scheduler instance + * + * Return true if we can push more jobs to the hw, otherwise false. + */ +-static bool drm_sched_ready(struct drm_gpu_scheduler *sched) ++static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + { + return atomic_read(&sched->hw_rq_count) < + sched->hw_submission_limit; +@@ -868,7 +867,7 @@ static bool drm_sched_ready(struct drm_gpu_scheduler *sched) + */ + void drm_sched_wakeup(struct drm_gpu_scheduler *sched) + { +- if (drm_sched_ready(sched)) ++ if (drm_sched_can_queue(sched)) + wake_up_interruptible(&sched->wake_up_worker); + } + +@@ -885,7 +884,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + struct drm_sched_entity *entity; + int i; + +- if (!drm_sched_ready(sched)) ++ if (!drm_sched_can_queue(sched)) + return NULL; + + /* Kernel run queue has higher priority than normal run queue*/ + +From a2190f971eeec0a544d4bd720967c1a42ac5425a Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Wed, 17 May 2023 19:35:50 -0400 +Subject: [PATCH 054/150] drm/sched: Rename to drm_sched_wakeup_if_can_queue() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Rename drm_sched_wakeup() to drm_sched_wakeup_if_canqueue() since the former +is misleading, as it wakes up the GPU scheduler _only if_ more jobs can be +queued to the underlying hardware. + +This distinction is important to make, since the wake conditional in the GPU +scheduler thread wakes up when other conditions are also true, e.g. when there +are jobs to be cleaned. For instance, a user might want to wake up the +scheduler only because there are more jobs to clean, but whether we can queue +more jobs is irrelevant. + +v2: Separate "canqueue" to "can_queue". (Alex D.) + +Cc: Christian König +Cc: Alex Deucher +Signed-off-by: Luben Tuikov +Link: https://lore.kernel.org/r/20230517233550.377847-2-luben.tuikov@amd.com +Reviewed-by: Alex Deucher +--- + drivers/gpu/drm/scheduler/sched_entity.c | 4 ++-- + drivers/gpu/drm/scheduler/sched_main.c | 6 +++--- + include/drm/gpu_scheduler.h | 2 +- + 3 files changed, 6 insertions(+), 6 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index cfb433e9200586..68e807ae136ad8 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -342,7 +342,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, + container_of(cb, struct drm_sched_entity, cb); + + drm_sched_entity_clear_dep(f, cb); +- drm_sched_wakeup(entity->rq->sched); ++ drm_sched_wakeup_if_can_queue(entity->rq->sched); + } + + /** +@@ -565,7 +565,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) + drm_sched_rq_update_fifo(entity, submit_ts); + +- drm_sched_wakeup(entity->rq->sched); ++ drm_sched_wakeup_if_can_queue(entity->rq->sched); + } + } + EXPORT_SYMBOL(drm_sched_entity_push_job); +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 6aec61b1b3e304..65f39f1880a1d3 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -860,12 +860,12 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + } + + /** +- * drm_sched_wakeup - Wake up the scheduler when it is ready +- * ++ * drm_sched_wakeup_if_can_queue - Wake up the scheduler + * @sched: scheduler instance + * ++ * Wake up the scheduler if we can queue jobs. + */ +-void drm_sched_wakeup(struct drm_gpu_scheduler *sched) ++void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched) + { + if (drm_sched_can_queue(sched)) + wake_up_interruptible(&sched->wake_up_worker); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 098321741b311e..0d3c1736ca8b04 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -548,7 +548,7 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + unsigned int num_sched_list); + + void drm_sched_job_cleanup(struct drm_sched_job *job); +-void drm_sched_wakeup(struct drm_gpu_scheduler *sched); ++void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched); + void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad); + void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); + +From 078d417c11a05a210b5438f38070e52e70c4be2b Mon Sep 17 00:00:00 2001 +From: ZhenGuo Yin +Date: Tue, 9 May 2023 17:46:17 +0800 +Subject: [PATCH 055/150] drm/scheduler: avoid infinite loop if entity's + dependency is a scheduled error fence +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +[Why] +drm_sched_entity_add_dependency_cb ignores the scheduled fence and return false. +If entity's dependency is a scheduler error fence and drm_sched_stop is called +due to TDR, drm_sched_entity_pop_job will wait for the dependency infinitely. + +[How] +Do not wait or ignore the scheduled error fence, add drm_sched_entity_wakeup +callback for the dependency with scheduled error fence. + +Signed-off-by: ZhenGuo Yin +Acked-by: Alex Deucher +Reviewed-by: Christian König +Signed-off-by: Alex Deucher +--- + drivers/gpu/drm/scheduler/sched_entity.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 68e807ae136ad8..b2bbc8a68b3056 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -384,7 +384,7 @@ static bool drm_sched_entity_add_dependency_cb(struct drm_sched_entity *entity) + } + + s_fence = to_drm_sched_fence(fence); +- if (s_fence && s_fence->sched == sched && ++ if (!fence->error && s_fence && s_fence->sched == sched && + !test_bit(DRM_SCHED_FENCE_DONT_PIPELINE, &fence->flags)) { + + /* + +From 17f01640a36694280315191a9d944062ca5704cf Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Mon, 19 Jun 2023 09:19:21 +0200 +Subject: [PATCH 056/150] drm/sched: Make sure we wait for all dependencies in + kill_jobs_cb() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +drm_sched_entity_kill_jobs_cb() logic is omitting the last fence popped +from the dependency array that was waited upon before +drm_sched_entity_kill() was called (drm_sched_entity::dependency field), +so we're basically waiting for all dependencies except one. + +In theory, this wait shouldn't be needed because resources should have +their users registered to the dma_resv object, thus guaranteeing that +future jobs wanting to access these resources wait on all the previous +users (depending on the access type, of course). But we want to keep +these explicit waits in the kill entity path just in case. + +Let's make sure we keep all dependencies in the array in +drm_sched_job_dependency(), so we can iterate over the array and wait +in drm_sched_entity_kill_jobs_cb(). + +We also make sure we wait on drm_sched_fence::finished if we were +originally asked to wait on drm_sched_fence::scheduled. In that case, +we assume the intent was to delegate the wait to the firmware/GPU or +rely on the pipelining done at the entity/scheduler level, but when +killing jobs, we really want to wait for completion not just scheduling. + +v2: +- Don't evict deps in drm_sched_job_dependency() + +v3: +- Always wait for drm_sched_fence::finished fences in + drm_sched_entity_kill_jobs_cb() when we see a sched_fence + +v4: +- Fix commit message +- Fix a use-after-free bug + +v5: +- Flag deps on which we should only wait for the scheduled event + at insertion time + +v6: +- Back to v4 implementation +- Add Christian's R-b + +Cc: Frank Binns +Cc: Sarah Walker +Cc: Donald Robson +Cc: Luben Tuikov +Cc: David Airlie +Cc: Daniel Vetter +Cc: Sumit Semwal +Cc: "Christian König" +Signed-off-by: Boris Brezillon +Suggested-by: "Christian König" +Reviewed-by: "Christian König" +Acked-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20230619071921.3465992-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 41 +++++++++++++++++++----- + 1 file changed, 33 insertions(+), 8 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index b2bbc8a68b3056..a42763e1429dc1 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -176,16 +176,32 @@ static void drm_sched_entity_kill_jobs_cb(struct dma_fence *f, + { + struct drm_sched_job *job = container_of(cb, struct drm_sched_job, + finish_cb); +- int r; ++ unsigned long index; + + dma_fence_put(f); + + /* Wait for all dependencies to avoid data corruptions */ +- while (!xa_empty(&job->dependencies)) { +- f = xa_erase(&job->dependencies, job->last_dependency++); +- r = dma_fence_add_callback(f, &job->finish_cb, +- drm_sched_entity_kill_jobs_cb); +- if (!r) ++ xa_for_each(&job->dependencies, index, f) { ++ struct drm_sched_fence *s_fence = to_drm_sched_fence(f); ++ ++ if (s_fence && f == &s_fence->scheduled) { ++ /* The dependencies array had a reference on the scheduled ++ * fence, and the finished fence refcount might have ++ * dropped to zero. Use dma_fence_get_rcu() so we get ++ * a NULL fence in that case. ++ */ ++ f = dma_fence_get_rcu(&s_fence->finished); ++ ++ /* Now that we have a reference on the finished fence, ++ * we can release the reference the dependencies array ++ * had on the scheduled fence. ++ */ ++ dma_fence_put(&s_fence->scheduled); ++ } ++ ++ xa_erase(&job->dependencies, index); ++ if (f && !dma_fence_add_callback(f, &job->finish_cb, ++ drm_sched_entity_kill_jobs_cb)) + return; + + dma_fence_put(f); +@@ -415,8 +431,17 @@ static struct dma_fence * + drm_sched_job_dependency(struct drm_sched_job *job, + struct drm_sched_entity *entity) + { +- if (!xa_empty(&job->dependencies)) +- return xa_erase(&job->dependencies, job->last_dependency++); ++ struct dma_fence *f; ++ ++ /* We keep the fence around, so we can iterate over all dependencies ++ * in drm_sched_entity_kill_jobs_cb() to ensure all deps are signaled ++ * before killing the job. ++ */ ++ f = xa_load(&job->dependencies, job->last_dependency); ++ if (f) { ++ job->last_dependency++; ++ return dma_fence_get(f); ++ } + + if (job->sched->ops->prepare_job) + return job->sched->ops->prepare_job(job, entity); + +From 1b0b5fccb8acd1fe9ce10d0173e0ca7d3ad2e1e7 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Fri, 23 Jun 2023 09:52:04 +0200 +Subject: [PATCH 057/150] drm/sched: Call drm_sched_fence_set_parent() from + drm_sched_fence_scheduled() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Drivers that can delegate waits to the firmware/GPU pass the scheduled +fence to drm_sched_job_add_dependency(), and issue wait commands to +the firmware/GPU at job submission time. For this to be possible, they +need all their 'native' dependencies to have a valid parent since this +is where the actual HW fence information are encoded. + +In drm_sched_main(), we currently call drm_sched_fence_set_parent() +after drm_sched_fence_scheduled(), leaving a short period of time +during which the job depending on this fence can be submitted. + +Since setting parent and signaling the fence are two things that are +kinda related (you can't have a parent if the job hasn't been +scheduled), +it probably makes sense to pass the parent fence to +drm_sched_fence_scheduled() and let it call drm_sched_fence_set_parent() +before it signals the scheduled fence. + +Here is a detailed description of the race we are fixing here: + +Thread A Thread B + +- calls drm_sched_fence_scheduled() +- signals s_fence->scheduled which + wakes up thread B + + - entity dep signaled, checking + the next dep + - no more deps waiting + - entity is picked for job + submission by drm_gpu_scheduler + - run_job() is called + - run_job() tries to + collect native fence info from + s_fence->parent, but it's + NULL => + BOOM, we can't do our native + wait + +- calls drm_sched_fence_set_parent() + +v2: +* Fix commit message + +v3: +* Add a detailed description of the race to the commit message +* Add Luben's R-b + +Signed-off-by: Boris Brezillon +Cc: Frank Binns +Cc: Sarah Walker +Cc: Donald Robson +Cc: Luben Tuikov +Cc: David Airlie +Cc: Daniel Vetter +Cc: Sumit Semwal +Cc: "Christian König" +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20230623075204.382350-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/scheduler/sched_fence.c | 40 +++++++++++++++---------- + drivers/gpu/drm/scheduler/sched_main.c | 3 +- + include/drm/gpu_scheduler.h | 5 ++-- + 3 files changed, 28 insertions(+), 20 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_fence.c b/drivers/gpu/drm/scheduler/sched_fence.c +index ef120475e7c618..06cedfe4b4867a 100644 +--- a/drivers/gpu/drm/scheduler/sched_fence.c ++++ b/drivers/gpu/drm/scheduler/sched_fence.c +@@ -48,8 +48,32 @@ static void __exit drm_sched_fence_slab_fini(void) + kmem_cache_destroy(sched_fence_slab); + } + +-void drm_sched_fence_scheduled(struct drm_sched_fence *fence) ++static void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, ++ struct dma_fence *fence) + { ++ /* ++ * smp_store_release() to ensure another thread racing us ++ * in drm_sched_fence_set_deadline_finished() sees the ++ * fence's parent set before test_bit() ++ */ ++ smp_store_release(&s_fence->parent, dma_fence_get(fence)); ++ if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, ++ &s_fence->finished.flags)) ++ dma_fence_set_deadline(fence, s_fence->deadline); ++} ++ ++void drm_sched_fence_scheduled(struct drm_sched_fence *fence, ++ struct dma_fence *parent) ++{ ++ /* Set the parent before signaling the scheduled fence, such that, ++ * any waiter expecting the parent to be filled after the job has ++ * been scheduled (which is the case for drivers delegating waits ++ * to some firmware) doesn't have to busy wait for parent to show ++ * up. ++ */ ++ if (!IS_ERR_OR_NULL(parent)) ++ drm_sched_fence_set_parent(fence, parent); ++ + dma_fence_signal(&fence->scheduled); + } + +@@ -181,20 +205,6 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f) + } + EXPORT_SYMBOL(to_drm_sched_fence); + +-void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, +- struct dma_fence *fence) +-{ +- /* +- * smp_store_release() to ensure another thread racing us +- * in drm_sched_fence_set_deadline_finished() sees the +- * fence's parent set before test_bit() +- */ +- smp_store_release(&s_fence->parent, dma_fence_get(fence)); +- if (test_bit(DRM_SCHED_FENCE_FLAG_HAS_DEADLINE_BIT, +- &s_fence->finished.flags)) +- dma_fence_set_deadline(fence, s_fence->deadline); +-} +- + struct drm_sched_fence *drm_sched_fence_alloc(struct drm_sched_entity *entity, + void *owner) + { +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 65f39f1880a1d3..5a3a622fc672f3 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1043,10 +1043,9 @@ static int drm_sched_main(void *param) + trace_drm_run_job(sched_job, entity); + fence = sched->ops->run_job(sched_job); + complete_all(&entity->entity_idle); +- drm_sched_fence_scheduled(s_fence); ++ drm_sched_fence_scheduled(s_fence, fence); + + if (!IS_ERR_OR_NULL(fence)) { +- drm_sched_fence_set_parent(s_fence, fence); + /* Drop for original kref_init of the fence */ + dma_fence_put(fence); + +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 0d3c1736ca8b04..88b9eca04447db 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -579,15 +579,14 @@ void drm_sched_entity_set_priority(struct drm_sched_entity *entity, + bool drm_sched_entity_is_ready(struct drm_sched_entity *entity); + int drm_sched_entity_error(struct drm_sched_entity *entity); + +-void drm_sched_fence_set_parent(struct drm_sched_fence *s_fence, +- struct dma_fence *fence); + struct drm_sched_fence *drm_sched_fence_alloc( + struct drm_sched_entity *s_entity, void *owner); + void drm_sched_fence_init(struct drm_sched_fence *fence, + struct drm_sched_entity *entity); + void drm_sched_fence_free(struct drm_sched_fence *fence); + +-void drm_sched_fence_scheduled(struct drm_sched_fence *fence); ++void drm_sched_fence_scheduled(struct drm_sched_fence *fence, ++ struct dma_fence *parent); + void drm_sched_fence_finished(struct drm_sched_fence *fence, int result); + + unsigned long drm_sched_suspend_timeout(struct drm_gpu_scheduler *sched); + +From 857b029ed6951f73d9a27fe3d58f8886eeb7f602 Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Sat, 14 Oct 2023 21:15:35 -0400 +Subject: [PATCH 058/150] drm/sched: Convert the GPU scheduler to variable + number of run-queues +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The GPU scheduler has now a variable number of run-queues, which are set +up at +drm_sched_init() time. This way, each driver announces how many +run-queues it +requires (supports) per each GPU scheduler it creates. Note, that +run-queues +correspond to scheduler "priorities", thus if the number of run-queues +is set +to 1 at drm_sched_init(), then that scheduler supports a single +run-queue, +i.e. single "priority". If a driver further sets a single entity per +run-queue, then this creates a 1-to-1 correspondence between a scheduler +and +a scheduled entity. + +Cc: Lucas Stach +Cc: Russell King +Cc: Qiang Yu +Cc: Rob Clark +Cc: Abhinav Kumar +Cc: Dmitry Baryshkov +Cc: Danilo Krummrich +Cc: Matthew Brost +Cc: Boris Brezillon +Cc: Alex Deucher +Cc: Christian König +Cc: Emma Anholt +Cc: etnaviv@lists.freedesktop.org +Cc: lima@lists.freedesktop.org +Cc: linux-arm-msm@vger.kernel.org +Cc: freedreno@lists.freedesktop.org +Cc: nouveau@lists.freedesktop.org +Cc: dri-devel@lists.freedesktop.org +Signed-off-by: Luben Tuikov +Acked-by: Christian König +Link: https://lore.kernel.org/r/20231023032251.164775-1-luben.tuikov@amd.com +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 1 + + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 4 +- + drivers/gpu/drm/etnaviv/etnaviv_sched.c | 1 + + drivers/gpu/drm/lima/lima_sched.c | 4 +- + drivers/gpu/drm/msm/msm_ringbuffer.c | 5 +- + drivers/gpu/drm/panfrost/panfrost_job.c | 1 + + drivers/gpu/drm/scheduler/sched_entity.c | 18 +++++- + drivers/gpu/drm/scheduler/sched_main.c | 74 ++++++++++++++++++---- + drivers/gpu/drm/v3d/v3d_sched.c | 5 ++ + include/drm/gpu_scheduler.h | 9 ++- + 10 files changed, 97 insertions(+), 25 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +index e357e1632dbf7f..1b9bcc7f3b0077 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +@@ -2354,6 +2354,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) + } + + r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + ring->num_hw_submission, amdgpu_job_hang_limit, + timeout, adev->reset_domain->wq, + ring->sched_score, ring->name, +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +index 2a4daf64a58534..580c55131eb59a 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +@@ -327,8 +327,8 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) + int i; + + /* Signal all jobs not yet scheduled */ +- for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { +- struct drm_sched_rq *rq = &sched->sched_rq[i]; ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ struct drm_sched_rq *rq = sched->sched_rq[i]; + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) { + while ((s_job = to_drm_sched_job(spsc_queue_pop(&s_entity->job_queue)))) { +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +index 72e2553fbc9849..8dbcf973d518ba 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +@@ -134,6 +134,7 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu) + int ret; + + ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + etnaviv_hw_jobs_limit, etnaviv_job_hang_limit, + msecs_to_jiffies(500), NULL, NULL, + dev_name(gpu->dev), gpu->dev); +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index e82931712d8a2a..4000f56a4c606b 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -488,7 +488,9 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name) + + INIT_WORK(&pipe->recover_work, lima_sched_recover_work); + +- return drm_sched_init(&pipe->base, &lima_sched_ops, 1, ++ return drm_sched_init(&pipe->base, &lima_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, ++ 1, + lima_job_hang_limit, + msecs_to_jiffies(timeout), NULL, + NULL, name, pipe->ldev->dev); +diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c +index 57a8e9564540ee..9278b957c701b1 100644 +--- a/drivers/gpu/drm/msm/msm_ringbuffer.c ++++ b/drivers/gpu/drm/msm/msm_ringbuffer.c +@@ -96,8 +96,9 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, + sched_timeout = MAX_SCHEDULE_TIMEOUT; + + ret = drm_sched_init(&ring->sched, &msm_sched_ops, +- num_hw_submissions, 0, sched_timeout, +- NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev); ++ DRM_SCHED_PRIORITY_COUNT, ++ num_hw_submissions, 0, sched_timeout, ++ NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev); + if (ret) { + goto fail; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c +index dbc597ab46fb9f..3154b8051c91fb 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_job.c ++++ b/drivers/gpu/drm/panfrost/panfrost_job.c +@@ -816,6 +816,7 @@ int panfrost_job_init(struct panfrost_device *pfdev) + + ret = drm_sched_init(&js->queue[j].sched, + &panfrost_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + nentries, 0, + msecs_to_jiffies(JOB_TIMEOUT_MS), + pfdev->reset.wq, +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index a42763e1429dc1..409e4256f6e7d6 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -75,8 +75,20 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + RCU_INIT_POINTER(entity->last_scheduled, NULL); + RB_CLEAR_NODE(&entity->rb_tree_node); + +- if(num_sched_list) +- entity->rq = &sched_list[0]->sched_rq[entity->priority]; ++ if (!sched_list[0]->sched_rq) { ++ /* Warn drivers not to do this and to fix their DRM ++ * calling order. ++ */ ++ pr_warn("%s: called with uninitialized scheduler\n", __func__); ++ } else if (num_sched_list) { ++ /* The "priority" of an entity cannot exceed the number ++ * of run-queues of a scheduler. ++ */ ++ if (entity->priority >= sched_list[0]->num_rqs) ++ entity->priority = max_t(u32, sched_list[0]->num_rqs, ++ DRM_SCHED_PRIORITY_MIN); ++ entity->rq = sched_list[0]->sched_rq[entity->priority]; ++ } + + init_completion(&entity->entity_idle); + +@@ -533,7 +545,7 @@ void drm_sched_entity_select_rq(struct drm_sched_entity *entity) + + spin_lock(&entity->rq_lock); + sched = drm_sched_pick_best(entity->sched_list, entity->num_sched_list); +- rq = sched ? &sched->sched_rq[entity->priority] : NULL; ++ rq = sched ? sched->sched_rq[entity->priority] : NULL; + if (rq != entity->rq) { + drm_sched_rq_remove_entity(entity->rq, entity); + entity->rq = rq; +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 5a3a622fc672f3..99797a8c836ac7 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -632,8 +632,14 @@ int drm_sched_job_init(struct drm_sched_job *job, + struct drm_sched_entity *entity, + void *owner) + { +- if (!entity->rq) ++ if (!entity->rq) { ++ /* This will most likely be followed by missing frames ++ * or worse--a blank screen--leave a trail in the ++ * logs, so this can be debugged easier. ++ */ ++ drm_err(job->sched, "%s: entity has no rq!\n", __func__); + return -ENOENT; ++ } + + job->entity = entity; + job->s_fence = drm_sched_fence_alloc(entity, owner); +@@ -671,7 +677,7 @@ void drm_sched_job_arm(struct drm_sched_job *job) + sched = entity->rq->sched; + + job->sched = sched; +- job->s_priority = entity->rq - sched->sched_rq; ++ job->s_priority = entity->priority; + job->id = atomic64_inc_return(&sched->job_id_count); + + drm_sched_fence_init(job->s_fence, job->entity); +@@ -888,10 +894,10 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + return NULL; + + /* Kernel run queue has higher priority than normal run queue*/ +- for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? +- drm_sched_rq_select_entity_fifo(&sched->sched_rq[i]) : +- drm_sched_rq_select_entity_rr(&sched->sched_rq[i]); ++ drm_sched_rq_select_entity_fifo(sched->sched_rq[i]) : ++ drm_sched_rq_select_entity_rr(sched->sched_rq[i]); + if (entity) + break; + } +@@ -1071,6 +1077,7 @@ static int drm_sched_main(void *param) + * + * @sched: scheduler instance + * @ops: backend operations for this scheduler ++ * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT + * @hw_submission: number of hw submissions that can be in flight + * @hang_limit: number of times to allow a job to hang before dropping it + * @timeout: timeout value in jiffies for the scheduler +@@ -1084,11 +1091,12 @@ static int drm_sched_main(void *param) + */ + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, +- unsigned hw_submission, unsigned hang_limit, ++ u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev) + { + int i, ret; ++ + sched->ops = ops; + sched->hw_submission_limit = hw_submission; + sched->name = name; +@@ -1097,8 +1105,36 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + sched->hang_limit = hang_limit; + sched->score = score ? score : &sched->_score; + sched->dev = dev; +- for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_COUNT; i++) +- drm_sched_rq_init(sched, &sched->sched_rq[i]); ++ ++ if (num_rqs > DRM_SCHED_PRIORITY_COUNT) { ++ /* This is a gross violation--tell drivers what the problem is. ++ */ ++ drm_err(sched, "%s: num_rqs cannot be greater than DRM_SCHED_PRIORITY_COUNT\n", ++ __func__); ++ return -EINVAL; ++ } else if (sched->sched_rq) { ++ /* Not an error, but warn anyway so drivers can ++ * fine-tune their DRM calling order, and return all ++ * is good. ++ */ ++ drm_warn(sched, "%s: scheduler already initialized!\n", __func__); ++ return 0; ++ } ++ ++ sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq), ++ GFP_KERNEL | __GFP_ZERO); ++ if (!sched->sched_rq) { ++ drm_err(sched, "%s: out of memory for sched_rq\n", __func__); ++ return -ENOMEM; ++ } ++ sched->num_rqs = num_rqs; ++ ret = -ENOMEM; ++ for (i = DRM_SCHED_PRIORITY_MIN; i < sched->num_rqs; i++) { ++ sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); ++ if (!sched->sched_rq[i]) ++ goto Out_unroll; ++ drm_sched_rq_init(sched, sched->sched_rq[i]); ++ } + + init_waitqueue_head(&sched->wake_up_worker); + init_waitqueue_head(&sched->job_scheduled); +@@ -1115,11 +1151,18 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + ret = PTR_ERR(sched->thread); + sched->thread = NULL; + DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name); +- return ret; ++ goto Out_unroll; + } + + sched->ready = true; + return 0; ++Out_unroll: ++ for (--i ; i >= DRM_SCHED_PRIORITY_MIN; i--) ++ kfree(sched->sched_rq[i]); ++ kfree(sched->sched_rq); ++ sched->sched_rq = NULL; ++ drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); ++ return ret; + } + EXPORT_SYMBOL(drm_sched_init); + +@@ -1138,8 +1181,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + if (sched->thread) + kthread_stop(sched->thread); + +- for (i = DRM_SCHED_PRIORITY_COUNT - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { +- struct drm_sched_rq *rq = &sched->sched_rq[i]; ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) +@@ -1150,7 +1193,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + */ + s_entity->stopped = true; + spin_unlock(&rq->lock); +- ++ kfree(sched->sched_rq[i]); + } + + /* Wakeup everyone stuck in drm_sched_entity_flush for this scheduler */ +@@ -1160,6 +1203,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + cancel_delayed_work_sync(&sched->work_tdr); + + sched->ready = false; ++ kfree(sched->sched_rq); ++ sched->sched_rq = NULL; + } + EXPORT_SYMBOL(drm_sched_fini); + +@@ -1186,9 +1231,10 @@ void drm_sched_increase_karma(struct drm_sched_job *bad) + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + +- for (i = DRM_SCHED_PRIORITY_MIN; i < DRM_SCHED_PRIORITY_KERNEL; ++ for (i = DRM_SCHED_PRIORITY_MIN; ++ i < min_t(typeof(sched->num_rqs), sched->num_rqs, DRM_SCHED_PRIORITY_KERNEL); + i++) { +- struct drm_sched_rq *rq = &sched->sched_rq[i]; ++ struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); + list_for_each_entry_safe(entity, tmp, &rq->entities, list) { +diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c +index 06238e6d7f5cda..038e1ae589c718 100644 +--- a/drivers/gpu/drm/v3d/v3d_sched.c ++++ b/drivers/gpu/drm/v3d/v3d_sched.c +@@ -389,6 +389,7 @@ v3d_sched_init(struct v3d_dev *v3d) + + ret = drm_sched_init(&v3d->queue[V3D_BIN].sched, + &v3d_bin_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_bin", v3d->drm.dev); +@@ -397,6 +398,7 @@ v3d_sched_init(struct v3d_dev *v3d) + + ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched, + &v3d_render_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_render", v3d->drm.dev); +@@ -405,6 +407,7 @@ v3d_sched_init(struct v3d_dev *v3d) + + ret = drm_sched_init(&v3d->queue[V3D_TFU].sched, + &v3d_tfu_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_tfu", v3d->drm.dev); +@@ -414,6 +417,7 @@ v3d_sched_init(struct v3d_dev *v3d) + if (v3d_has_csd(v3d)) { + ret = drm_sched_init(&v3d->queue[V3D_CSD].sched, + &v3d_csd_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_csd", v3d->drm.dev); +@@ -422,6 +426,7 @@ v3d_sched_init(struct v3d_dev *v3d) + + ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched, + &v3d_cache_clean_sched_ops, ++ DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_cache_clean", v3d->drm.dev); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 88b9eca04447db..59b456d069a78e 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -471,7 +471,9 @@ struct drm_sched_backend_ops { + * @hw_submission_limit: the max size of the hardware queue. + * @timeout: the time after which a job is removed from the scheduler. + * @name: name of the ring for which this scheduler is being used. +- * @sched_rq: priority wise array of run queues. ++ * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT, ++ * as there's usually one run-queue per priority, but could be less. ++ * @sched_rq: An allocated array of run-queues of size @num_rqs; + * @wake_up_worker: the wait queue on which the scheduler sleeps until a job + * is ready to be scheduled. + * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler +@@ -500,7 +502,8 @@ struct drm_gpu_scheduler { + uint32_t hw_submission_limit; + long timeout; + const char *name; +- struct drm_sched_rq sched_rq[DRM_SCHED_PRIORITY_COUNT]; ++ u32 num_rqs; ++ struct drm_sched_rq **sched_rq; + wait_queue_head_t wake_up_worker; + wait_queue_head_t job_scheduled; + atomic_t hw_rq_count; +@@ -520,7 +523,7 @@ struct drm_gpu_scheduler { + + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, +- uint32_t hw_submission, unsigned hang_limit, ++ u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev); + + +From e60880fd0c6ecbf7ee7e2e37a5a2bf1c0736f9a6 Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Mon, 30 Oct 2023 20:24:35 -0700 +Subject: [PATCH 059/150] [MOD] drm/sched: Add drm_sched_wqueue_* helpers + +Add scheduler wqueue ready, stop, and start helpers to hide the +implementation details of the scheduler from the drivers. + +v2: + - s/sched_wqueue/sched_wqueue (Luben) + - Remove the extra white line after the return-statement (Luben) + - update drm_sched_wqueue_ready comment (Luben) + +Cc: Luben Tuikov +Signed-off-by: Matthew Brost +Reviewed-by: Luben Tuikov +Link: https://lore.kernel.org/r/20231031032439.1558703-2-matthew.brost@intel.com +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c | 15 ++++---- + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 12 +++---- + drivers/gpu/drm/msm/adreno/adreno_device.c | 6 ++-- + drivers/gpu/drm/scheduler/sched_main.c | 39 +++++++++++++++++++-- + include/drm/gpu_scheduler.h | 3 ++ + 5 files changed, 58 insertions(+), 17 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +index 06ab6066da61a8..d68f6f2277223e 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c +@@ -1501,9 +1501,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; +- kthread_park(ring->sched.thread); ++ drm_sched_wqueue_stop(&ring->sched); + } + + seq_printf(m, "run ib test:\n"); +@@ -1517,9 +1517,9 @@ static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused) + for (i = 0; i < AMDGPU_MAX_RINGS; i++) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; +- kthread_unpark(ring->sched.thread); ++ drm_sched_wqueue_start(&ring->sched); + } + + up_write(&adev->reset_domain->sem); +@@ -1739,7 +1739,8 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) + + ring = adev->rings[val]; + +- if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread) ++ if (!ring || !ring->funcs->preempt_ib || ++ !drm_sched_wqueue_ready(&ring->sched)) + return -EINVAL; + + /* the last preemption failed */ +@@ -1757,7 +1758,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) + goto pro_end; + + /* stop the scheduler */ +- kthread_park(ring->sched.thread); ++ drm_sched_wqueue_stop(&ring->sched); + + resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev); + +@@ -1793,7 +1794,7 @@ static int amdgpu_debugfs_ib_preempt(void *data, u64 val) + + failure: + /* restart the scheduler */ +- kthread_unpark(ring->sched.thread); ++ drm_sched_wqueue_start(&ring->sched); + + up_read(&adev->reset_domain->sem); + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +index 1b9bcc7f3b0077..98d314707b513b 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +@@ -4651,7 +4651,7 @@ bool amdgpu_device_has_job_running(struct amdgpu_device *adev) + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + spin_lock(&ring->sched.job_list_lock); +@@ -4778,7 +4778,7 @@ int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev, + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + /*clear job fence from fence drv to avoid force_completion +@@ -5318,7 +5318,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = tmp_adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + drm_sched_stop(&ring->sched, job ? &job->base : NULL); +@@ -5392,7 +5392,7 @@ int amdgpu_device_gpu_recover(struct amdgpu_device *adev, + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = tmp_adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + /* No point to resubmit jobs if we didn't HW reset*/ +@@ -5721,7 +5721,7 @@ pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_sta + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + drm_sched_stop(&ring->sched, NULL); +@@ -5849,7 +5849,7 @@ void amdgpu_pci_resume(struct pci_dev *pdev) + for (i = 0; i < AMDGPU_MAX_RINGS; ++i) { + struct amdgpu_ring *ring = adev->rings[i]; + +- if (!ring || !ring->sched.thread) ++ if (!ring || !drm_sched_wqueue_ready(&ring->sched)) + continue; + + +diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c +index ed1e0c650bb1a8..41e901cdf07629 100644 +--- a/drivers/gpu/drm/msm/adreno/adreno_device.c ++++ b/drivers/gpu/drm/msm/adreno/adreno_device.c +@@ -667,7 +667,8 @@ static void suspend_scheduler(struct msm_gpu *gpu) + */ + for (i = 0; i < gpu->nr_rings; i++) { + struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched; +- kthread_park(sched->thread); ++ ++ drm_sched_wqueue_stop(sched); + } + } + +@@ -677,7 +678,8 @@ static void resume_scheduler(struct msm_gpu *gpu) + + for (i = 0; i < gpu->nr_rings; i++) { + struct drm_gpu_scheduler *sched = &gpu->rb[i]->sched; +- kthread_unpark(sched->thread); ++ ++ drm_sched_wqueue_start(sched); + } + } + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 99797a8c836ac7..54c1c5fe01ba9f 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -439,7 +439,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) + { + struct drm_sched_job *s_job, *tmp; + +- kthread_park(sched->thread); ++ drm_sched_wqueue_stop(sched); + + /* + * Reinsert back the bad job here - now it's safe as +@@ -552,7 +552,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + spin_unlock(&sched->job_list_lock); + } + +- kthread_unpark(sched->thread); ++ drm_sched_wqueue_start(sched); + } + EXPORT_SYMBOL(drm_sched_start); + +@@ -1252,3 +1252,38 @@ void drm_sched_increase_karma(struct drm_sched_job *bad) + } + } + EXPORT_SYMBOL(drm_sched_increase_karma); ++ ++/** ++ * drm_sched_wqueue_ready - Is the scheduler ready for submission ++ * ++ * @sched: scheduler instance ++ * ++ * Returns true if submission is ready ++ */ ++bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched) ++{ ++ return !!sched->thread; ++} ++EXPORT_SYMBOL(drm_sched_wqueue_ready); ++ ++/** ++ * drm_sched_wqueue_stop - stop scheduler submission ++ * ++ * @sched: scheduler instance ++ */ ++void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched) ++{ ++ kthread_park(sched->thread); ++} ++EXPORT_SYMBOL(drm_sched_wqueue_stop); ++ ++/** ++ * drm_sched_wqueue_start - start scheduler submission ++ * ++ * @sched: scheduler instance ++ */ ++void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched) ++{ ++ kthread_unpark(sched->thread); ++} ++EXPORT_SYMBOL(drm_sched_wqueue_start); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 59b456d069a78e..27ae19682b46c9 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -552,6 +552,9 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + + void drm_sched_job_cleanup(struct drm_sched_job *job); + void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched); ++bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); ++void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched); ++void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched); + void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad); + void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); + +From a29cf1848caafa9058aaea8191b68fe12d73a44f Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Mon, 30 Oct 2023 20:24:36 -0700 +Subject: [PATCH 060/150] [MOD] drm/sched: Convert drm scheduler to use a work + queue rather than kthread + +In Xe, the new Intel GPU driver, a choice has made to have a 1 to 1 +mapping between a drm_gpu_scheduler and drm_sched_entity. At first this +seems a bit odd but let us explain the reasoning below. + +1. In Xe the submission order from multiple drm_sched_entity is not +guaranteed to be the same completion even if targeting the same hardware +engine. This is because in Xe we have a firmware scheduler, the GuC, +which allowed to reorder, timeslice, and preempt submissions. If a using +shared drm_gpu_scheduler across multiple drm_sched_entity, the TDR falls +apart as the TDR expects submission order == completion order. Using a +dedicated drm_gpu_scheduler per drm_sched_entity solve this problem. + +2. In Xe submissions are done via programming a ring buffer (circular +buffer), a drm_gpu_scheduler provides a limit on number of jobs, if the +limit of number jobs is set to RING_SIZE / MAX_SIZE_PER_JOB we get flow +control on the ring for free. + +A problem with this design is currently a drm_gpu_scheduler uses a +kthread for submission / job cleanup. This doesn't scale if a large +number of drm_gpu_scheduler are used. To work around the scaling issue, +use a worker rather than kthread for submission / job cleanup. + +v2: + - (Rob Clark) Fix msm build + - Pass in run work queue +v3: + - (Boris) don't have loop in worker +v4: + - (Tvrtko) break out submit ready, stop, start helpers into own patch +v5: + - (Boris) default to ordered work queue +v6: + - (Luben / checkpatch) fix alignment in msm_ringbuffer.c + - (Luben) s/drm_sched_submit_queue/drm_sched_wqueue_enqueue + - (Luben) Update comment for drm_sched_wqueue_enqueue + - (Luben) Positive check for submit_wq in drm_sched_init + - (Luben) s/alloc_submit_wq/own_submit_wq +v7: + - (Luben) s/drm_sched_wqueue_enqueue/drm_sched_run_job_queue +v8: + - (Luben) Adjust var names / comments + +Signed-off-by: Matthew Brost +Reviewed-by: Luben Tuikov +Link: https://lore.kernel.org/r/20231031032439.1558703-3-matthew.brost@intel.com +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_device.c | 2 +- + drivers/gpu/drm/etnaviv/etnaviv_sched.c | 2 +- + drivers/gpu/drm/lima/lima_sched.c | 2 +- + drivers/gpu/drm/msm/msm_ringbuffer.c | 2 +- + drivers/gpu/drm/panfrost/panfrost_job.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 131 +++++++++++---------- + drivers/gpu/drm/v3d/v3d_sched.c | 10 +- + include/drm/gpu_scheduler.h | 14 ++- + 8 files changed, 85 insertions(+), 80 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +index 98d314707b513b..eb0958dba33f6f 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +@@ -2353,7 +2353,7 @@ static int amdgpu_device_init_schedulers(struct amdgpu_device *adev) + break; + } + +- r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, ++ r = drm_sched_init(&ring->sched, &amdgpu_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + ring->num_hw_submission, amdgpu_job_hang_limit, + timeout, adev->reset_domain->wq, +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +index 8dbcf973d518ba..8d8ef85cbad57a 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +@@ -133,7 +133,7 @@ int etnaviv_sched_init(struct etnaviv_gpu *gpu) + { + int ret; + +- ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, ++ ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + etnaviv_hw_jobs_limit, etnaviv_job_hang_limit, + msecs_to_jiffies(500), NULL, NULL, +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index 4000f56a4c606b..3e74516af9c7b5 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -488,7 +488,7 @@ int lima_sched_pipe_init(struct lima_sched_pipe *pipe, const char *name) + + INIT_WORK(&pipe->recover_work, lima_sched_recover_work); + +- return drm_sched_init(&pipe->base, &lima_sched_ops, ++ return drm_sched_init(&pipe->base, &lima_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + 1, + lima_job_hang_limit, +diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c +index 9278b957c701b1..546bd6388b2eb5 100644 +--- a/drivers/gpu/drm/msm/msm_ringbuffer.c ++++ b/drivers/gpu/drm/msm/msm_ringbuffer.c +@@ -95,7 +95,7 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int id, + /* currently managing hangcheck ourselves: */ + sched_timeout = MAX_SCHEDULE_TIMEOUT; + +- ret = drm_sched_init(&ring->sched, &msm_sched_ops, ++ ret = drm_sched_init(&ring->sched, &msm_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + num_hw_submissions, 0, sched_timeout, + NULL, NULL, to_msm_bo(ring->bo)->name, gpu->dev->dev); +diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c +index 3154b8051c91fb..72629cec1187f7 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_job.c ++++ b/drivers/gpu/drm/panfrost/panfrost_job.c +@@ -815,7 +815,7 @@ int panfrost_job_init(struct panfrost_device *pfdev) + js->queue[j].fence_context = dma_fence_context_alloc(1); + + ret = drm_sched_init(&js->queue[j].sched, +- &panfrost_sched_ops, ++ &panfrost_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + nentries, 0, + msecs_to_jiffies(JOB_TIMEOUT_MS), +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 54c1c5fe01ba9f..d1ae05bded15c9 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -48,7 +48,6 @@ + * through the jobs entity pointer. + */ + +-#include + #include + #include + #include +@@ -256,6 +255,16 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) + return rb ? rb_entry(rb, struct drm_sched_entity, rb_tree_node) : NULL; + } + ++/** ++ * drm_sched_run_job_queue - enqueue run-job work ++ * @sched: scheduler instance ++ */ ++static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) ++{ ++ if (!READ_ONCE(sched->pause_submit)) ++ queue_work(sched->submit_wq, &sched->work_run_job); ++} ++ + /** + * drm_sched_job_done - complete a job + * @s_job: pointer to the job which is done +@@ -275,7 +284,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result) + dma_fence_get(&s_fence->finished); + drm_sched_fence_finished(s_fence, result); + dma_fence_put(&s_fence->finished); +- wake_up_interruptible(&sched->wake_up_worker); ++ drm_sched_run_job_queue(sched); + } + + /** +@@ -874,7 +883,7 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched) + { + if (drm_sched_can_queue(sched)) +- wake_up_interruptible(&sched->wake_up_worker); ++ drm_sched_run_job_queue(sched); + } + + /** +@@ -985,60 +994,41 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list, + EXPORT_SYMBOL(drm_sched_pick_best); + + /** +- * drm_sched_blocked - check if the scheduler is blocked ++ * drm_sched_run_job_work - main scheduler thread + * +- * @sched: scheduler instance +- * +- * Returns true if blocked, otherwise false. ++ * @w: run job work + */ +-static bool drm_sched_blocked(struct drm_gpu_scheduler *sched) ++static void drm_sched_run_job_work(struct work_struct *w) + { +- if (kthread_should_park()) { +- kthread_parkme(); +- return true; +- } +- +- return false; +-} +- +-/** +- * drm_sched_main - main scheduler thread +- * +- * @param: scheduler instance +- * +- * Returns 0. +- */ +-static int drm_sched_main(void *param) +-{ +- struct drm_gpu_scheduler *sched = (struct drm_gpu_scheduler *)param; ++ struct drm_gpu_scheduler *sched = ++ container_of(w, struct drm_gpu_scheduler, work_run_job); ++ struct drm_sched_entity *entity; ++ struct drm_sched_job *cleanup_job; + int r; + +- sched_set_fifo_low(current); ++ if (READ_ONCE(sched->pause_submit)) ++ return; + +- while (!kthread_should_stop()) { +- struct drm_sched_entity *entity = NULL; +- struct drm_sched_fence *s_fence; +- struct drm_sched_job *sched_job; +- struct dma_fence *fence; +- struct drm_sched_job *cleanup_job = NULL; ++ cleanup_job = drm_sched_get_cleanup_job(sched); ++ entity = drm_sched_select_entity(sched); + +- wait_event_interruptible(sched->wake_up_worker, +- (cleanup_job = drm_sched_get_cleanup_job(sched)) || +- (!drm_sched_blocked(sched) && +- (entity = drm_sched_select_entity(sched))) || +- kthread_should_stop()); ++ if (!entity && !cleanup_job) ++ return; /* No more work */ + +- if (cleanup_job) +- sched->ops->free_job(cleanup_job); ++ if (cleanup_job) ++ sched->ops->free_job(cleanup_job); + +- if (!entity) +- continue; ++ if (entity) { ++ struct dma_fence *fence; ++ struct drm_sched_fence *s_fence; ++ struct drm_sched_job *sched_job; + + sched_job = drm_sched_entity_pop_job(entity); +- + if (!sched_job) { + complete_all(&entity->entity_idle); +- continue; ++ if (!cleanup_job) ++ return; /* No more work */ ++ goto again; + } + + s_fence = sched_job->s_fence; +@@ -1069,7 +1059,9 @@ static int drm_sched_main(void *param) + + wake_up(&sched->job_scheduled); + } +- return 0; ++ ++again: ++ drm_sched_run_job_queue(sched); + } + + /** +@@ -1077,6 +1069,8 @@ static int drm_sched_main(void *param) + * + * @sched: scheduler instance + * @ops: backend operations for this scheduler ++ * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is ++ * allocated and used + * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT + * @hw_submission: number of hw submissions that can be in flight + * @hang_limit: number of times to allow a job to hang before dropping it +@@ -1091,6 +1085,7 @@ static int drm_sched_main(void *param) + */ + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, ++ struct workqueue_struct *submit_wq, + u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev) +@@ -1121,14 +1116,22 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + return 0; + } + ++ if (submit_wq) { ++ sched->submit_wq = submit_wq; ++ sched->own_submit_wq = false; ++ } else { ++ sched->submit_wq = alloc_ordered_workqueue(name, 0); ++ if (!sched->submit_wq) ++ return -ENOMEM; ++ ++ sched->own_submit_wq = true; ++ } ++ ret = -ENOMEM; + sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq), + GFP_KERNEL | __GFP_ZERO); +- if (!sched->sched_rq) { +- drm_err(sched, "%s: out of memory for sched_rq\n", __func__); +- return -ENOMEM; +- } ++ if (!sched->sched_rq) ++ goto Out_free; + sched->num_rqs = num_rqs; +- ret = -ENOMEM; + for (i = DRM_SCHED_PRIORITY_MIN; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); + if (!sched->sched_rq[i]) +@@ -1136,31 +1139,26 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + drm_sched_rq_init(sched, sched->sched_rq[i]); + } + +- init_waitqueue_head(&sched->wake_up_worker); + init_waitqueue_head(&sched->job_scheduled); + INIT_LIST_HEAD(&sched->pending_list); + spin_lock_init(&sched->job_list_lock); + atomic_set(&sched->hw_rq_count, 0); + INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout); ++ INIT_WORK(&sched->work_run_job, drm_sched_run_job_work); + atomic_set(&sched->_score, 0); + atomic64_set(&sched->job_id_count, 0); +- +- /* Each scheduler will run on a seperate kernel thread */ +- sched->thread = kthread_run(drm_sched_main, sched, sched->name); +- if (IS_ERR(sched->thread)) { +- ret = PTR_ERR(sched->thread); +- sched->thread = NULL; +- DRM_DEV_ERROR(sched->dev, "Failed to create scheduler for %s.\n", name); +- goto Out_unroll; +- } ++ sched->pause_submit = false; + + sched->ready = true; + return 0; + Out_unroll: + for (--i ; i >= DRM_SCHED_PRIORITY_MIN; i--) + kfree(sched->sched_rq[i]); ++Out_free: + kfree(sched->sched_rq); + sched->sched_rq = NULL; ++ if (sched->own_submit_wq) ++ destroy_workqueue(sched->submit_wq); + drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); + return ret; + } +@@ -1178,8 +1176,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + struct drm_sched_entity *s_entity; + int i; + +- if (sched->thread) +- kthread_stop(sched->thread); ++ drm_sched_wqueue_stop(sched); + + for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + struct drm_sched_rq *rq = sched->sched_rq[i]; +@@ -1202,6 +1199,8 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + /* Confirm no work left behind accessing device structures */ + cancel_delayed_work_sync(&sched->work_tdr); + ++ if (sched->own_submit_wq) ++ destroy_workqueue(sched->submit_wq); + sched->ready = false; + kfree(sched->sched_rq); + sched->sched_rq = NULL; +@@ -1262,7 +1261,7 @@ EXPORT_SYMBOL(drm_sched_increase_karma); + */ + bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched) + { +- return !!sched->thread; ++ return sched->ready; + } + EXPORT_SYMBOL(drm_sched_wqueue_ready); + +@@ -1273,7 +1272,8 @@ EXPORT_SYMBOL(drm_sched_wqueue_ready); + */ + void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched) + { +- kthread_park(sched->thread); ++ WRITE_ONCE(sched->pause_submit, true); ++ cancel_work_sync(&sched->work_run_job); + } + EXPORT_SYMBOL(drm_sched_wqueue_stop); + +@@ -1284,6 +1284,7 @@ EXPORT_SYMBOL(drm_sched_wqueue_stop); + */ + void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched) + { +- kthread_unpark(sched->thread); ++ WRITE_ONCE(sched->pause_submit, false); ++ queue_work(sched->submit_wq, &sched->work_run_job); + } + EXPORT_SYMBOL(drm_sched_wqueue_start); +diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c +index 038e1ae589c718..0b6696b0d882fa 100644 +--- a/drivers/gpu/drm/v3d/v3d_sched.c ++++ b/drivers/gpu/drm/v3d/v3d_sched.c +@@ -388,7 +388,7 @@ v3d_sched_init(struct v3d_dev *v3d) + int ret; + + ret = drm_sched_init(&v3d->queue[V3D_BIN].sched, +- &v3d_bin_sched_ops, ++ &v3d_bin_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, +@@ -397,7 +397,7 @@ v3d_sched_init(struct v3d_dev *v3d) + return ret; + + ret = drm_sched_init(&v3d->queue[V3D_RENDER].sched, +- &v3d_render_sched_ops, ++ &v3d_render_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, +@@ -406,7 +406,7 @@ v3d_sched_init(struct v3d_dev *v3d) + goto fail; + + ret = drm_sched_init(&v3d->queue[V3D_TFU].sched, +- &v3d_tfu_sched_ops, ++ &v3d_tfu_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, +@@ -416,7 +416,7 @@ v3d_sched_init(struct v3d_dev *v3d) + + if (v3d_has_csd(v3d)) { + ret = drm_sched_init(&v3d->queue[V3D_CSD].sched, +- &v3d_csd_sched_ops, ++ &v3d_csd_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, +@@ -425,7 +425,7 @@ v3d_sched_init(struct v3d_dev *v3d) + goto fail; + + ret = drm_sched_init(&v3d->queue[V3D_CACHE_CLEAN].sched, +- &v3d_cache_clean_sched_ops, ++ &v3d_cache_clean_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + hw_jobs_limit, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 27ae19682b46c9..1892a8e1c9834e 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -474,17 +474,16 @@ struct drm_sched_backend_ops { + * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT, + * as there's usually one run-queue per priority, but could be less. + * @sched_rq: An allocated array of run-queues of size @num_rqs; +- * @wake_up_worker: the wait queue on which the scheduler sleeps until a job +- * is ready to be scheduled. + * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler + * waits on this wait queue until all the scheduled jobs are + * finished. + * @hw_rq_count: the number of jobs currently in the hardware queue. + * @job_id_count: used to assign unique id to the each job. ++ * @submit_wq: workqueue used to queue @work_run_job + * @timeout_wq: workqueue used to queue @work_tdr ++ * @work_run_job: work which calls run_job op of each scheduler. + * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the + * timeout interval is over. +- * @thread: the kthread on which the scheduler which run. + * @pending_list: the list of jobs which are currently in the job queue. + * @job_list_lock: lock to protect the pending_list. + * @hang_limit: once the hangs by a job crosses this limit then it is marked +@@ -493,6 +492,8 @@ struct drm_sched_backend_ops { + * @_score: score used when the driver doesn't provide one + * @ready: marks if the underlying HW is ready to work + * @free_guilty: A hit to time out handler to free the guilty job. ++ * @pause_submit: pause queuing of @work_run_job on @submit_wq ++ * @own_submit_wq: scheduler owns allocation of @submit_wq + * @dev: system &struct device + * + * One scheduler is implemented for each hardware ring. +@@ -504,13 +505,13 @@ struct drm_gpu_scheduler { + const char *name; + u32 num_rqs; + struct drm_sched_rq **sched_rq; +- wait_queue_head_t wake_up_worker; + wait_queue_head_t job_scheduled; + atomic_t hw_rq_count; + atomic64_t job_id_count; ++ struct workqueue_struct *submit_wq; + struct workqueue_struct *timeout_wq; ++ struct work_struct work_run_job; + struct delayed_work work_tdr; +- struct task_struct *thread; + struct list_head pending_list; + spinlock_t job_list_lock; + int hang_limit; +@@ -518,11 +519,14 @@ struct drm_gpu_scheduler { + atomic_t _score; + bool ready; + bool free_guilty; ++ bool pause_submit; ++ bool own_submit_wq; + struct device *dev; + }; + + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, ++ struct workqueue_struct *submit_wq, + u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev); + +From 63811fdf69067138107e4ab193e8340868f89370 Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Mon, 30 Oct 2023 20:24:37 -0700 +Subject: [PATCH 061/150] drm/sched: Split free_job into own work item + +Rather than call free_job and run_job in same work item have a dedicated +work item for each. This aligns with the design and intended use of work +queues. + +v2: + - Test for DMA_FENCE_FLAG_TIMESTAMP_BIT before setting + timestamp in free_job() work item (Danilo) +v3: + - Drop forward dec of drm_sched_select_entity (Boris) + - Return in drm_sched_run_job_work if entity NULL (Boris) +v4: + - Replace dequeue with peek and invert logic (Luben) + - Wrap to 100 lines (Luben) + - Update comments for *_queue / *_queue_if_ready functions (Luben) +v5: + - Drop peek argument, blindly reinit idle (Luben) + - s/drm_sched_free_job_queue_if_ready/drm_sched_free_job_queue_if_done (Luben) + - Update work_run_job & work_free_job kernel doc (Luben) +v6: + - Do not move drm_sched_select_entity in file (Luben) + +Signed-off-by: Matthew Brost +Link: https://lore.kernel.org/r/20231031032439.1558703-4-matthew.brost@intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 146 +++++++++++++++++-------- + include/drm/gpu_scheduler.h | 4 +- + 2 files changed, 101 insertions(+), 49 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index d1ae05bded15c9..3b1b2f8eafe837 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -265,6 +265,32 @@ static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) + queue_work(sched->submit_wq, &sched->work_run_job); + } + ++/** ++ * drm_sched_free_job_queue - enqueue free-job work ++ * @sched: scheduler instance ++ */ ++static void drm_sched_free_job_queue(struct drm_gpu_scheduler *sched) ++{ ++ if (!READ_ONCE(sched->pause_submit)) ++ queue_work(sched->submit_wq, &sched->work_free_job); ++} ++ ++/** ++ * drm_sched_free_job_queue_if_done - enqueue free-job work if ready ++ * @sched: scheduler instance ++ */ ++static void drm_sched_free_job_queue_if_done(struct drm_gpu_scheduler *sched) ++{ ++ struct drm_sched_job *job; ++ ++ spin_lock(&sched->job_list_lock); ++ job = list_first_entry_or_null(&sched->pending_list, ++ struct drm_sched_job, list); ++ if (job && dma_fence_is_signaled(&job->s_fence->finished)) ++ drm_sched_free_job_queue(sched); ++ spin_unlock(&sched->job_list_lock); ++} ++ + /** + * drm_sched_job_done - complete a job + * @s_job: pointer to the job which is done +@@ -284,7 +310,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result) + dma_fence_get(&s_fence->finished); + drm_sched_fence_finished(s_fence, result); + dma_fence_put(&s_fence->finished); +- drm_sched_run_job_queue(sched); ++ drm_sched_free_job_queue(sched); + } + + /** +@@ -943,8 +969,10 @@ drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) + typeof(*next), list); + + if (next) { +- next->s_fence->scheduled.timestamp = +- dma_fence_timestamp(&job->s_fence->finished); ++ if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, ++ &next->s_fence->scheduled.flags)) ++ next->s_fence->scheduled.timestamp = ++ dma_fence_timestamp(&job->s_fence->finished); + /* start TO timer for next job */ + drm_sched_start_timeout(sched); + } +@@ -994,7 +1022,40 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list, + EXPORT_SYMBOL(drm_sched_pick_best); + + /** +- * drm_sched_run_job_work - main scheduler thread ++ * drm_sched_run_job_queue_if_ready - enqueue run-job work if ready ++ * @sched: scheduler instance ++ */ ++static void drm_sched_run_job_queue_if_ready(struct drm_gpu_scheduler *sched) ++{ ++ if (drm_sched_select_entity(sched)) ++ drm_sched_run_job_queue(sched); ++} ++ ++/** ++ * drm_sched_free_job_work - worker to call free_job ++ * ++ * @w: free job work ++ */ ++static void drm_sched_free_job_work(struct work_struct *w) ++{ ++ struct drm_gpu_scheduler *sched = ++ container_of(w, struct drm_gpu_scheduler, work_free_job); ++ struct drm_sched_job *cleanup_job; ++ ++ if (READ_ONCE(sched->pause_submit)) ++ return; ++ ++ cleanup_job = drm_sched_get_cleanup_job(sched); ++ if (cleanup_job) { ++ sched->ops->free_job(cleanup_job); ++ ++ drm_sched_free_job_queue_if_done(sched); ++ drm_sched_run_job_queue_if_ready(sched); ++ } ++} ++ ++/** ++ * drm_sched_run_job_work - worker to call run_job + * + * @w: run job work + */ +@@ -1003,65 +1064,51 @@ static void drm_sched_run_job_work(struct work_struct *w) + struct drm_gpu_scheduler *sched = + container_of(w, struct drm_gpu_scheduler, work_run_job); + struct drm_sched_entity *entity; +- struct drm_sched_job *cleanup_job; ++ struct dma_fence *fence; ++ struct drm_sched_fence *s_fence; ++ struct drm_sched_job *sched_job; + int r; + + if (READ_ONCE(sched->pause_submit)) + return; + +- cleanup_job = drm_sched_get_cleanup_job(sched); + entity = drm_sched_select_entity(sched); ++ if (!entity) ++ return; + +- if (!entity && !cleanup_job) ++ sched_job = drm_sched_entity_pop_job(entity); ++ if (!sched_job) { ++ complete_all(&entity->entity_idle); + return; /* No more work */ ++ } + +- if (cleanup_job) +- sched->ops->free_job(cleanup_job); +- +- if (entity) { +- struct dma_fence *fence; +- struct drm_sched_fence *s_fence; +- struct drm_sched_job *sched_job; +- +- sched_job = drm_sched_entity_pop_job(entity); +- if (!sched_job) { +- complete_all(&entity->entity_idle); +- if (!cleanup_job) +- return; /* No more work */ +- goto again; +- } +- +- s_fence = sched_job->s_fence; +- +- atomic_inc(&sched->hw_rq_count); +- drm_sched_job_begin(sched_job); ++ s_fence = sched_job->s_fence; + +- trace_drm_run_job(sched_job, entity); +- fence = sched->ops->run_job(sched_job); +- complete_all(&entity->entity_idle); +- drm_sched_fence_scheduled(s_fence, fence); ++ atomic_inc(&sched->hw_rq_count); ++ drm_sched_job_begin(sched_job); + +- if (!IS_ERR_OR_NULL(fence)) { +- /* Drop for original kref_init of the fence */ +- dma_fence_put(fence); ++ trace_drm_run_job(sched_job, entity); ++ fence = sched->ops->run_job(sched_job); ++ complete_all(&entity->entity_idle); ++ drm_sched_fence_scheduled(s_fence, fence); + +- r = dma_fence_add_callback(fence, &sched_job->cb, +- drm_sched_job_done_cb); +- if (r == -ENOENT) +- drm_sched_job_done(sched_job, fence->error); +- else if (r) +- DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", +- r); +- } else { +- drm_sched_job_done(sched_job, IS_ERR(fence) ? +- PTR_ERR(fence) : 0); +- } ++ if (!IS_ERR_OR_NULL(fence)) { ++ /* Drop for original kref_init of the fence */ ++ dma_fence_put(fence); + +- wake_up(&sched->job_scheduled); ++ r = dma_fence_add_callback(fence, &sched_job->cb, ++ drm_sched_job_done_cb); ++ if (r == -ENOENT) ++ drm_sched_job_done(sched_job, fence->error); ++ else if (r) ++ DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", r); ++ } else { ++ drm_sched_job_done(sched_job, IS_ERR(fence) ? ++ PTR_ERR(fence) : 0); + } + +-again: +- drm_sched_run_job_queue(sched); ++ wake_up(&sched->job_scheduled); ++ drm_sched_run_job_queue_if_ready(sched); + } + + /** +@@ -1145,6 +1192,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + atomic_set(&sched->hw_rq_count, 0); + INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout); + INIT_WORK(&sched->work_run_job, drm_sched_run_job_work); ++ INIT_WORK(&sched->work_free_job, drm_sched_free_job_work); + atomic_set(&sched->_score, 0); + atomic64_set(&sched->job_id_count, 0); + sched->pause_submit = false; +@@ -1274,6 +1322,7 @@ void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched) + { + WRITE_ONCE(sched->pause_submit, true); + cancel_work_sync(&sched->work_run_job); ++ cancel_work_sync(&sched->work_free_job); + } + EXPORT_SYMBOL(drm_sched_wqueue_stop); + +@@ -1286,5 +1335,6 @@ void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched) + { + WRITE_ONCE(sched->pause_submit, false); + queue_work(sched->submit_wq, &sched->work_run_job); ++ queue_work(sched->submit_wq, &sched->work_free_job); + } + EXPORT_SYMBOL(drm_sched_wqueue_start); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 1892a8e1c9834e..111e2d15965e35 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -479,9 +479,10 @@ struct drm_sched_backend_ops { + * finished. + * @hw_rq_count: the number of jobs currently in the hardware queue. + * @job_id_count: used to assign unique id to the each job. +- * @submit_wq: workqueue used to queue @work_run_job ++ * @submit_wq: workqueue used to queue @work_run_job and @work_free_job + * @timeout_wq: workqueue used to queue @work_tdr + * @work_run_job: work which calls run_job op of each scheduler. ++ * @work_free_job: work which calls free_job op of each scheduler. + * @work_tdr: schedules a delayed call to @drm_sched_job_timedout after the + * timeout interval is over. + * @pending_list: the list of jobs which are currently in the job queue. +@@ -511,6 +512,7 @@ struct drm_gpu_scheduler { + struct workqueue_struct *submit_wq; + struct workqueue_struct *timeout_wq; + struct work_struct work_run_job; ++ struct work_struct work_free_job; + struct delayed_work work_tdr; + struct list_head pending_list; + spinlock_t job_list_lock; + +From 9cb324d1ff61d996a59b27de2562cb40b792c5a0 Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Mon, 30 Oct 2023 20:24:38 -0700 +Subject: [PATCH 062/150] drm/sched: Add drm_sched_start_timeout_unlocked + helper + +Also add a lockdep assert to drm_sched_start_timeout. + +Signed-off-by: Matthew Brost +Reviewed-by: Luben Tuikov +Link: https://lore.kernel.org/r/20231031032439.1558703-5-matthew.brost@intel.com +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 23 +++++++++++++---------- + 1 file changed, 13 insertions(+), 10 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 3b1b2f8eafe837..fc387de5a0c752 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -334,11 +334,20 @@ static void drm_sched_job_done_cb(struct dma_fence *f, struct dma_fence_cb *cb) + */ + static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched) + { ++ lockdep_assert_held(&sched->job_list_lock); ++ + if (sched->timeout != MAX_SCHEDULE_TIMEOUT && + !list_empty(&sched->pending_list)) + queue_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout); + } + ++static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched) ++{ ++ spin_lock(&sched->job_list_lock); ++ drm_sched_start_timeout(sched); ++ spin_unlock(&sched->job_list_lock); ++} ++ + /** + * drm_sched_fault - immediately start timeout handler + * +@@ -451,11 +460,8 @@ static void drm_sched_job_timedout(struct work_struct *work) + spin_unlock(&sched->job_list_lock); + } + +- if (status != DRM_GPU_SCHED_STAT_ENODEV) { +- spin_lock(&sched->job_list_lock); +- drm_sched_start_timeout(sched); +- spin_unlock(&sched->job_list_lock); +- } ++ if (status != DRM_GPU_SCHED_STAT_ENODEV) ++ drm_sched_start_timeout_unlocked(sched); + } + + /** +@@ -581,11 +587,8 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + drm_sched_job_done(s_job, -ECANCELED); + } + +- if (full_recovery) { +- spin_lock(&sched->job_list_lock); +- drm_sched_start_timeout(sched); +- spin_unlock(&sched->job_list_lock); +- } ++ if (full_recovery) ++ drm_sched_start_timeout_unlocked(sched); + + drm_sched_wqueue_start(sched); + } + +From 5312fceb259e6c2fc11af7eb7f706cd2355b47f0 Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Mon, 30 Oct 2023 20:24:39 -0700 +Subject: [PATCH 063/150] drm/sched: Add a helper to queue TDR immediately + +Add a helper whereby a driver can invoke TDR immediately. + +v2: + - Drop timeout args, rename function, use mod delayed work (Luben) +v3: + - s/XE/Xe (Luben) + - present tense in commit message (Luben) + - Adjust comment for drm_sched_tdr_queue_imm (Luben) +v4: + - Adjust commit message (Luben) + +Cc: Luben Tuikov +Signed-off-by: Matthew Brost +Reviewed-by: Luben Tuikov +Link: https://lore.kernel.org/r/20231031032439.1558703-6-matthew.brost@intel.com +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 18 +++++++++++++++++- + include/drm/gpu_scheduler.h | 1 + + 2 files changed, 18 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index fc387de5a0c752..98b2ad54fc7071 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -338,7 +338,7 @@ static void drm_sched_start_timeout(struct drm_gpu_scheduler *sched) + + if (sched->timeout != MAX_SCHEDULE_TIMEOUT && + !list_empty(&sched->pending_list)) +- queue_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout); ++ mod_delayed_work(sched->timeout_wq, &sched->work_tdr, sched->timeout); + } + + static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched) +@@ -348,6 +348,22 @@ static void drm_sched_start_timeout_unlocked(struct drm_gpu_scheduler *sched) + spin_unlock(&sched->job_list_lock); + } + ++/** ++ * drm_sched_tdr_queue_imm: - immediately start job timeout handler ++ * ++ * @sched: scheduler for which the timeout handling should be started. ++ * ++ * Start timeout handling immediately for the named scheduler. ++ */ ++void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched) ++{ ++ spin_lock(&sched->job_list_lock); ++ sched->timeout = 0; ++ drm_sched_start_timeout(sched); ++ spin_unlock(&sched->job_list_lock); ++} ++EXPORT_SYMBOL(drm_sched_tdr_queue_imm); ++ + /** + * drm_sched_fault - immediately start timeout handler + * +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 111e2d15965e35..3bba4a53a8e94b 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -556,6 +556,7 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + struct drm_gpu_scheduler **sched_list, + unsigned int num_sched_list); + ++void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched); + void drm_sched_job_cleanup(struct drm_sched_job *job); + void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched); + bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); + +From 721c66c4ec990f16bbc9209d63cdfc5add8ab99b Mon Sep 17 00:00:00 2001 +From: Tvrtko Ursulin +Date: Thu, 2 Nov 2023 10:55:34 +0000 +Subject: [PATCH 064/150] drm/sched: Rename drm_sched_get_cleanup_job to be + more descriptive + +"Get cleanup job" makes it sound like helper is returning a job which will +execute some cleanup, or something, while the kerneldoc itself accurately +says "fetch the next _finished_ job". So lets rename the helper to be self +documenting. + +Signed-off-by: Tvrtko Ursulin +Cc: Luben Tuikov +Cc: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20231102105538.391648-2-tvrtko.ursulin@linux.intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 98b2ad54fc7071..fb64b35451f5a9 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -448,7 +448,7 @@ static void drm_sched_job_timedout(struct work_struct *work) + + sched = container_of(work, struct drm_gpu_scheduler, work_tdr.work); + +- /* Protects against concurrent deletion in drm_sched_get_cleanup_job */ ++ /* Protects against concurrent deletion in drm_sched_get_finished_job */ + spin_lock(&sched->job_list_lock); + job = list_first_entry_or_null(&sched->pending_list, + struct drm_sched_job, list); +@@ -500,9 +500,9 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) + + /* + * Reinsert back the bad job here - now it's safe as +- * drm_sched_get_cleanup_job cannot race against us and release the ++ * drm_sched_get_finished_job cannot race against us and release the + * bad job at this point - we parked (waited for) any in progress +- * (earlier) cleanups and drm_sched_get_cleanup_job will not be called ++ * (earlier) cleanups and drm_sched_get_finished_job will not be called + * now until the scheduler thread is unparked. + */ + if (bad && bad->sched == sched) +@@ -960,7 +960,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + } + + /** +- * drm_sched_get_cleanup_job - fetch the next finished job to be destroyed ++ * drm_sched_get_finished_job - fetch the next finished job to be destroyed + * + * @sched: scheduler instance + * +@@ -968,7 +968,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + * ready for it to be destroyed. + */ + static struct drm_sched_job * +-drm_sched_get_cleanup_job(struct drm_gpu_scheduler *sched) ++drm_sched_get_finished_job(struct drm_gpu_scheduler *sched) + { + struct drm_sched_job *job, *next; + +@@ -1059,14 +1059,14 @@ static void drm_sched_free_job_work(struct work_struct *w) + { + struct drm_gpu_scheduler *sched = + container_of(w, struct drm_gpu_scheduler, work_free_job); +- struct drm_sched_job *cleanup_job; ++ struct drm_sched_job *job; + + if (READ_ONCE(sched->pause_submit)) + return; + +- cleanup_job = drm_sched_get_cleanup_job(sched); +- if (cleanup_job) { +- sched->ops->free_job(cleanup_job); ++ job = drm_sched_get_finished_job(sched); ++ if (job) { ++ sched->ops->free_job(job); + + drm_sched_free_job_queue_if_done(sched); + drm_sched_run_job_queue_if_ready(sched); + +From 1e1039c3e4a6504e46ad8c0515e8a9ce6792802b Mon Sep 17 00:00:00 2001 +From: Tvrtko Ursulin +Date: Thu, 2 Nov 2023 10:55:35 +0000 +Subject: [PATCH 065/150] drm/sched: Move free worker re-queuing out of the if + block + +Whether or not there are more jobs to clean up does not depend on the +existance of the current job, given both drm_sched_get_finished_job and +drm_sched_free_job_queue_if_done take and drop the job list lock. +Therefore it is confusing to make it read like there is a dependency. + +Signed-off-by: Tvrtko Ursulin +Cc: Luben Tuikov +Cc: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20231102105538.391648-3-tvrtko.ursulin@linux.intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 7 +++---- + 1 file changed, 3 insertions(+), 4 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index fb64b35451f5a9..e1658030613f5c 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1065,12 +1065,11 @@ static void drm_sched_free_job_work(struct work_struct *w) + return; + + job = drm_sched_get_finished_job(sched); +- if (job) { ++ if (job) + sched->ops->free_job(job); + +- drm_sched_free_job_queue_if_done(sched); +- drm_sched_run_job_queue_if_ready(sched); +- } ++ drm_sched_free_job_queue_if_done(sched); ++ drm_sched_run_job_queue_if_ready(sched); + } + + /** + +From 84b73bb6dbee4144e6f9ab6df030dbb9535a297a Mon Sep 17 00:00:00 2001 +From: Tvrtko Ursulin +Date: Thu, 2 Nov 2023 10:55:36 +0000 +Subject: [PATCH 066/150] drm/sched: Rename drm_sched_free_job_queue to be more + descriptive + +The current name makes it sound like helper will free a queue, while what +it does is it enqueues the free job worker. + +Rename it to drm_sched_run_free_queue to align with existing +drm_sched_run_job_queue. + +Despite that creating an illusion there are two queues, while in reality +there is only one, at least it creates a consistent naming for the two +enqueuing helpers. + +At the same time simplify the "if done" helper by dropping the suffix and +adding a double underscore prefix to the one which just enqueues. + +Signed-off-by: Tvrtko Ursulin +Cc: Luben Tuikov +Cc: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20231102105538.391648-4-tvrtko.ursulin@linux.intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 14 +++++++------- + 1 file changed, 7 insertions(+), 7 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index e1658030613f5c..f9baca20b43838 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -266,20 +266,20 @@ static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) + } + + /** +- * drm_sched_free_job_queue - enqueue free-job work ++ * __drm_sched_run_free_queue - enqueue free-job work + * @sched: scheduler instance + */ +-static void drm_sched_free_job_queue(struct drm_gpu_scheduler *sched) ++static void __drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) + { + if (!READ_ONCE(sched->pause_submit)) + queue_work(sched->submit_wq, &sched->work_free_job); + } + + /** +- * drm_sched_free_job_queue_if_done - enqueue free-job work if ready ++ * drm_sched_run_free_queue - enqueue free-job work if ready + * @sched: scheduler instance + */ +-static void drm_sched_free_job_queue_if_done(struct drm_gpu_scheduler *sched) ++static void drm_sched_run_free_queue(struct drm_gpu_scheduler *sched) + { + struct drm_sched_job *job; + +@@ -287,7 +287,7 @@ static void drm_sched_free_job_queue_if_done(struct drm_gpu_scheduler *sched) + job = list_first_entry_or_null(&sched->pending_list, + struct drm_sched_job, list); + if (job && dma_fence_is_signaled(&job->s_fence->finished)) +- drm_sched_free_job_queue(sched); ++ __drm_sched_run_free_queue(sched); + spin_unlock(&sched->job_list_lock); + } + +@@ -310,7 +310,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result) + dma_fence_get(&s_fence->finished); + drm_sched_fence_finished(s_fence, result); + dma_fence_put(&s_fence->finished); +- drm_sched_free_job_queue(sched); ++ __drm_sched_run_free_queue(sched); + } + + /** +@@ -1068,7 +1068,7 @@ static void drm_sched_free_job_work(struct work_struct *w) + if (job) + sched->ops->free_job(job); + +- drm_sched_free_job_queue_if_done(sched); ++ drm_sched_run_free_queue(sched); + drm_sched_run_job_queue_if_ready(sched); + } + + +From 6b4ad4dd55956a3eb2dae10b5e7cc9f3698c2f26 Mon Sep 17 00:00:00 2001 +From: Tvrtko Ursulin +Date: Thu, 2 Nov 2023 10:55:37 +0000 +Subject: [PATCH 067/150] drm/sched: Rename drm_sched_run_job_queue_if_ready + and clarify kerneldoc + +"If ready" is not immediately clear what it means - is the scheduler +ready or something else? Drop the suffix, clarify kerneldoc, and employ +the same naming scheme as in drm_sched_run_free_queue: + + - drm_sched_run_job_queue - enqueues if there is something to enqueue + *and* scheduler is ready (can queue) + - __drm_sched_run_job_queue - low-level helper to simply queue the job + +Signed-off-by: Tvrtko Ursulin +Cc: Luben Tuikov +Cc: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20231102105538.391648-5-tvrtko.ursulin@linux.intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index f9baca20b43838..d5ddbce68fb775 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -256,10 +256,10 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) + } + + /** +- * drm_sched_run_job_queue - enqueue run-job work ++ * __drm_sched_run_job_queue - enqueue run-job work + * @sched: scheduler instance + */ +-static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) ++static void __drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) + { + if (!READ_ONCE(sched->pause_submit)) + queue_work(sched->submit_wq, &sched->work_run_job); +@@ -928,7 +928,7 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched) + { + if (drm_sched_can_queue(sched)) +- drm_sched_run_job_queue(sched); ++ __drm_sched_run_job_queue(sched); + } + + /** +@@ -1041,13 +1041,13 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list, + EXPORT_SYMBOL(drm_sched_pick_best); + + /** +- * drm_sched_run_job_queue_if_ready - enqueue run-job work if ready ++ * drm_sched_run_job_queue - enqueue run-job work if there are ready entities + * @sched: scheduler instance + */ +-static void drm_sched_run_job_queue_if_ready(struct drm_gpu_scheduler *sched) ++static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) + { + if (drm_sched_select_entity(sched)) +- drm_sched_run_job_queue(sched); ++ __drm_sched_run_job_queue(sched); + } + + /** +@@ -1069,7 +1069,7 @@ static void drm_sched_free_job_work(struct work_struct *w) + sched->ops->free_job(job); + + drm_sched_run_free_queue(sched); +- drm_sched_run_job_queue_if_ready(sched); ++ drm_sched_run_job_queue(sched); + } + + /** +@@ -1126,7 +1126,7 @@ static void drm_sched_run_job_work(struct work_struct *w) + } + + wake_up(&sched->job_scheduled); +- drm_sched_run_job_queue_if_ready(sched); ++ drm_sched_run_job_queue(sched); + } + + /** + +From 4d70342365700dac3f50f6096a546c22c4eaa339 Mon Sep 17 00:00:00 2001 +From: Tvrtko Ursulin +Date: Thu, 2 Nov 2023 10:55:38 +0000 +Subject: [PATCH 068/150] drm/sched: Drop suffix from + drm_sched_wakeup_if_can_queue + +Because a) helper is exported to other parts of the scheduler and +b) there isn't a plain drm_sched_wakeup to begin with, I think we can +drop the suffix and by doing so separate the intimiate knowledge +between the scheduler components a bit better. + +Signed-off-by: Tvrtko Ursulin +Cc: Luben Tuikov +Cc: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20231102105538.391648-6-tvrtko.ursulin@linux.intel.com +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_entity.c | 4 ++-- + drivers/gpu/drm/scheduler/sched_main.c | 4 ++-- + include/drm/gpu_scheduler.h | 2 +- + 3 files changed, 5 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 409e4256f6e7d6..f1db63cc819812 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -370,7 +370,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, + container_of(cb, struct drm_sched_entity, cb); + + drm_sched_entity_clear_dep(f, cb); +- drm_sched_wakeup_if_can_queue(entity->rq->sched); ++ drm_sched_wakeup(entity->rq->sched); + } + + /** +@@ -602,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) + drm_sched_rq_update_fifo(entity, submit_ts); + +- drm_sched_wakeup_if_can_queue(entity->rq->sched); ++ drm_sched_wakeup(entity->rq->sched); + } + } + EXPORT_SYMBOL(drm_sched_entity_push_job); +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index d5ddbce68fb775..27843e37d9b769 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -920,12 +920,12 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + } + + /** +- * drm_sched_wakeup_if_can_queue - Wake up the scheduler ++ * drm_sched_wakeup - Wake up the scheduler if it is ready to queue + * @sched: scheduler instance + * + * Wake up the scheduler if we can queue jobs. + */ +-void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched) ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched) + { + if (drm_sched_can_queue(sched)) + __drm_sched_run_job_queue(sched); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 3bba4a53a8e94b..d6cd898100c432 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -558,7 +558,7 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + + void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched); + void drm_sched_job_cleanup(struct drm_sched_job *job); +-void drm_sched_wakeup_if_can_queue(struct drm_gpu_scheduler *sched); ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched); + bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched); + +From 7b3ab9a7e8668c8c991993ad14a186de856240ed Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Thu, 2 Nov 2023 18:17:20 -0400 +Subject: [PATCH 069/150] drm/sched: Don't disturb the entity when in RR-mode + scheduling + +Don't call drm_sched_select_entity() in drm_sched_run_job_queue(). In fact, +rename __drm_sched_run_job_queue() to just drm_sched_run_job_queue(), and let +it do just that, schedule the work item for execution. + +The problem is that drm_sched_run_job_queue() calls drm_sched_select_entity() +to determine if the scheduler has an entity ready in one of its run-queues, +and in the case of the Round-Robin (RR) scheduling, the function +drm_sched_rq_select_entity_rr() does just that, selects the _next_ entity +which is ready, sets up the run-queue and completion and returns that +entity. The FIFO scheduling algorithm is unaffected. + +Now, since drm_sched_run_job_work() also calls drm_sched_select_entity(), then +in the case of RR scheduling, that would result in drm_sched_select_entity() +having been called twice, which may result in skipping a ready entity if more +than one entity is ready. This commit fixes this by eliminating the call to +drm_sched_select_entity() from drm_sched_run_job_queue(), and leaves it only +in drm_sched_run_job_work(). + +v2: Rebased on top of Tvrtko's renames series of patches. (Luben) + Add fixes-tag. (Tvrtko) + +Signed-off-by: Luben Tuikov +Fixes: f7fe64ad0f22ff ("drm/sched: Split free_job into own work item") +Reviewed-by: Matthew Brost +Reviewed-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231107041020.10035-2-ltuikov89@gmail.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 16 +++------------- + 1 file changed, 3 insertions(+), 13 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 27843e37d9b769..cd0dc3f81d05f0 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -256,10 +256,10 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) + } + + /** +- * __drm_sched_run_job_queue - enqueue run-job work ++ * drm_sched_run_job_queue - enqueue run-job work + * @sched: scheduler instance + */ +-static void __drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) ++static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) + { + if (!READ_ONCE(sched->pause_submit)) + queue_work(sched->submit_wq, &sched->work_run_job); +@@ -928,7 +928,7 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + void drm_sched_wakeup(struct drm_gpu_scheduler *sched) + { + if (drm_sched_can_queue(sched)) +- __drm_sched_run_job_queue(sched); ++ drm_sched_run_job_queue(sched); + } + + /** +@@ -1040,16 +1040,6 @@ drm_sched_pick_best(struct drm_gpu_scheduler **sched_list, + } + EXPORT_SYMBOL(drm_sched_pick_best); + +-/** +- * drm_sched_run_job_queue - enqueue run-job work if there are ready entities +- * @sched: scheduler instance +- */ +-static void drm_sched_run_job_queue(struct drm_gpu_scheduler *sched) +-{ +- if (drm_sched_select_entity(sched)) +- __drm_sched_run_job_queue(sched); +-} +- + /** + * drm_sched_free_job_work - worker to call free_job + * + +From 4e30522fd5698ec8aaa44e2342ecdc1657925a80 Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Thu, 9 Nov 2023 18:53:26 -0500 +Subject: [PATCH 070/150] drm/sched: Qualify drm_sched_wakeup() by + drm_sched_entity_is_ready() + +Don't "wake up" the GPU scheduler unless the entity is ready, as well as we +can queue to the scheduler, i.e. there is no point in waking up the scheduler +for the entity unless the entity is ready. + +Signed-off-by: Luben Tuikov +Fixes: bc8d6a9df99038 ("drm/sched: Don't disturb the entity when in RR-mode scheduling") +Reviewed-by: Danilo Krummrich +Link: https://patchwork.freedesktop.org/patch/msgid/20231110000123.72565-2-ltuikov89@gmail.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 4 ++-- + drivers/gpu/drm/scheduler/sched_main.c | 8 +++++--- + include/drm/gpu_scheduler.h | 2 +- + 3 files changed, 8 insertions(+), 6 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index f1db63cc819812..4d42b1e4daa67f 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -370,7 +370,7 @@ static void drm_sched_entity_wakeup(struct dma_fence *f, + container_of(cb, struct drm_sched_entity, cb); + + drm_sched_entity_clear_dep(f, cb); +- drm_sched_wakeup(entity->rq->sched); ++ drm_sched_wakeup(entity->rq->sched, entity); + } + + /** +@@ -602,7 +602,7 @@ void drm_sched_entity_push_job(struct drm_sched_job *sched_job) + if (drm_sched_policy == DRM_SCHED_POLICY_FIFO) + drm_sched_rq_update_fifo(entity, submit_ts); + +- drm_sched_wakeup(entity->rq->sched); ++ drm_sched_wakeup(entity->rq->sched, entity); + } + } + EXPORT_SYMBOL(drm_sched_entity_push_job); +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index cd0dc3f81d05f0..8f5e466bd58239 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -925,10 +925,12 @@ static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) + * + * Wake up the scheduler if we can queue jobs. + */ +-void drm_sched_wakeup(struct drm_gpu_scheduler *sched) ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched, ++ struct drm_sched_entity *entity) + { +- if (drm_sched_can_queue(sched)) +- drm_sched_run_job_queue(sched); ++ if (drm_sched_entity_is_ready(entity)) ++ if (drm_sched_can_queue(sched)) ++ drm_sched_run_job_queue(sched); + } + + /** +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index d6cd898100c432..562b2232c81bad 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -558,7 +558,7 @@ void drm_sched_entity_modify_sched(struct drm_sched_entity *entity, + + void drm_sched_tdr_queue_imm(struct drm_gpu_scheduler *sched); + void drm_sched_job_cleanup(struct drm_sched_job *job); +-void drm_sched_wakeup(struct drm_gpu_scheduler *sched); ++void drm_sched_wakeup(struct drm_gpu_scheduler *sched, struct drm_sched_entity *entity); + bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched); + +From a48bdb43683363fa3d2f277bb2468be8a6d0c42b Mon Sep 17 00:00:00 2001 +From: Danilo Krummrich +Date: Fri, 10 Nov 2023 01:16:33 +0100 +Subject: [PATCH 071/150] drm/sched: implement dynamic job-flow control + +Currently, job flow control is implemented simply by limiting the number +of jobs in flight. Therefore, a scheduler is initialized with a credit +limit that corresponds to the number of jobs which can be sent to the +hardware. + +This implies that for each job, drivers need to account for the maximum +job size possible in order to not overflow the ring buffer. + +However, there are drivers, such as Nouveau, where the job size has a +rather large range. For such drivers it can easily happen that job +submissions not even filling the ring by 1% can block subsequent +submissions, which, in the worst case, can lead to the ring run dry. + +In order to overcome this issue, allow for tracking the actual job size +instead of the number of jobs. Therefore, add a field to track a job's +credit count, which represents the number of credits a job contributes +to the scheduler's credit limit. + +Signed-off-by: Danilo Krummrich +Reviewed-by: Luben Tuikov +Link: https://patchwork.freedesktop.org/patch/msgid/20231110001638.71750-1-dakr@redhat.com +--- + Documentation/gpu/drm-mm.rst | 6 + + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 + + drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c | 2 +- + drivers/gpu/drm/etnaviv/etnaviv_gpu.c | 2 +- + drivers/gpu/drm/lima/lima_device.c | 2 +- + drivers/gpu/drm/lima/lima_sched.c | 2 +- + drivers/gpu/drm/msm/msm_gem_submit.c | 2 +- + drivers/gpu/drm/panfrost/panfrost_drv.c | 2 +- + drivers/gpu/drm/panfrost/panfrost_job.c | 2 +- + .../gpu/drm/scheduler/gpu_scheduler_trace.h | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 170 ++++++++++++++---- + drivers/gpu/drm/v3d/v3d_gem.c | 2 +- + include/drm/gpu_scheduler.h | 28 ++- + 13 files changed, 175 insertions(+), 49 deletions(-) + +diff --git a/Documentation/gpu/drm-mm.rst b/Documentation/gpu/drm-mm.rst +index 3d5dc9dc1bfe51..709c4f149f6ec6 100644 +--- a/Documentation/gpu/drm-mm.rst ++++ b/Documentation/gpu/drm-mm.rst +@@ -550,6 +550,12 @@ Overview + .. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c + :doc: Overview + ++Flow Control ++------------ ++ ++.. kernel-doc:: drivers/gpu/drm/scheduler/sched_main.c ++ :doc: Flow Control ++ + Scheduler Function References + ----------------------------- + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +index 580c55131eb59a..d4f3fd4c6aeb85 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +@@ -111,6 +111,8 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, + (*job)->vm_pd_addr = AMDGPU_BO_INVALID_OFFSET; + + return 0; ++ ++ return drm_sched_job_init(&(*job)->base, entity, 1, owner); + } + + int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +index 1ac916b248917c..ceaebf793909b3 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +@@ -531,7 +531,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, + + ret = drm_sched_job_init(&submit->sched_job, + &ctx->sched_entity[args->pipe], +- submit->ctx); ++ 1, submit->ctx); + if (ret) + goto err_submit_put; + +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +index f667e7906d1f4e..b08709715abd18 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +@@ -1870,7 +1870,7 @@ static int etnaviv_gpu_rpm_suspend(struct device *dev) + u32 idle, mask; + + /* If there are any jobs in the HW queue, we're not idle */ +- if (atomic_read(&gpu->sched.hw_rq_count)) ++ if (atomic_read(&gpu->sched.credit_count)) + return -EBUSY; + + /* Check whether the hardware (except FE and MC) is idle */ +diff --git a/drivers/gpu/drm/lima/lima_device.c b/drivers/gpu/drm/lima/lima_device.c +index 02cef0cea6572b..0bf7105c8748b4 100644 +--- a/drivers/gpu/drm/lima/lima_device.c ++++ b/drivers/gpu/drm/lima/lima_device.c +@@ -514,7 +514,7 @@ int lima_device_suspend(struct device *dev) + + /* check any task running */ + for (i = 0; i < lima_pipe_num; i++) { +- if (atomic_read(&ldev->pipe[i].base.hw_rq_count)) ++ if (atomic_read(&ldev->pipe[i].base.credit_count)) + return -EBUSY; + } + +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index 3e74516af9c7b5..3c3c765de51169 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -123,7 +123,7 @@ int lima_sched_task_init(struct lima_sched_task *task, + for (i = 0; i < num_bos; i++) + drm_gem_object_get(&bos[i]->base.base); + +- err = drm_sched_job_init(&task->base, &context->base, vm); ++ err = drm_sched_job_init(&task->base, &context->base, 1, vm); + if (err) { + kfree(task->bos); + return err; +diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c +index c12a6ac2d38407..3a8b9098b39ae9 100644 +--- a/drivers/gpu/drm/msm/msm_gem_submit.c ++++ b/drivers/gpu/drm/msm/msm_gem_submit.c +@@ -41,7 +41,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev, + if (!submit) + return ERR_PTR(-ENOMEM); + +- ret = drm_sched_job_init(&submit->base, queue->entity, queue); ++ ret = drm_sched_job_init(&submit->base, queue->entity, 1, queue); + if (ret) { + kfree(submit); + return ERR_PTR(ret); +diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c +index b93589f535e08f..5fbdba35472cb8 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_drv.c ++++ b/drivers/gpu/drm/panfrost/panfrost_drv.c +@@ -278,7 +278,7 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data, + + ret = drm_sched_job_init(&job->base, + &file_priv->sched_entity[slot], +- NULL); ++ 1, NULL); + if (ret) + goto out_put_job; + +diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c +index 72629cec1187f7..045524a7695ba5 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_job.c ++++ b/drivers/gpu/drm/panfrost/panfrost_job.c +@@ -923,7 +923,7 @@ int panfrost_job_is_idle(struct panfrost_device *pfdev) + + for (i = 0; i < NUM_JOB_SLOTS; i++) { + /* If there are any jobs in the HW queue, we're not idle */ +- if (atomic_read(&js->queue[i].sched.hw_rq_count)) ++ if (atomic_read(&js->queue[i].sched.credit_count)) + return false; + } + +diff --git a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h +index 3143ecaaff8628..f8ed093b7356eb 100644 +--- a/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h ++++ b/drivers/gpu/drm/scheduler/gpu_scheduler_trace.h +@@ -51,7 +51,7 @@ DECLARE_EVENT_CLASS(drm_sched_job, + __assign_str(name, sched_job->sched->name); + __entry->job_count = spsc_queue_count(&entity->job_queue); + __entry->hw_job_count = atomic_read( +- &sched_job->sched->hw_rq_count); ++ &sched_job->sched->credit_count); + ), + TP_printk("entity=%p, id=%llu, fence=%p, ring=%s, job count:%u, hw job count:%d", + __entry->entity, __entry->id, +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 8f5e466bd58239..044a8c4875ba64 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -48,6 +48,30 @@ + * through the jobs entity pointer. + */ + ++/** ++ * DOC: Flow Control ++ * ++ * The DRM GPU scheduler provides a flow control mechanism to regulate the rate ++ * in which the jobs fetched from scheduler entities are executed. ++ * ++ * In this context the &drm_gpu_scheduler keeps track of a driver specified ++ * credit limit representing the capacity of this scheduler and a credit count; ++ * every &drm_sched_job carries a driver specified number of credits. ++ * ++ * Once a job is executed (but not yet finished), the job's credits contribute ++ * to the scheduler's credit count until the job is finished. If by executing ++ * one more job the scheduler's credit count would exceed the scheduler's ++ * credit limit, the job won't be executed. Instead, the scheduler will wait ++ * until the credit count has decreased enough to not overflow its credit limit. ++ * This implies waiting for previously executed jobs. ++ * ++ * Optionally, drivers may register a callback (update_job_credits) provided by ++ * struct drm_sched_backend_ops to update the job's credits dynamically. The ++ * scheduler executes this callback every time the scheduler considers a job for ++ * execution and subsequently checks whether the job fits the scheduler's credit ++ * limit. ++ */ ++ + #include + #include + #include +@@ -75,6 +99,51 @@ int drm_sched_policy = DRM_SCHED_POLICY_FIFO; + MODULE_PARM_DESC(sched_policy, "Specify the scheduling policy for entities on a run-queue, " __stringify(DRM_SCHED_POLICY_RR) " = Round Robin, " __stringify(DRM_SCHED_POLICY_FIFO) " = FIFO (default)."); + module_param_named(sched_policy, drm_sched_policy, int, 0444); + ++static u32 drm_sched_available_credits(struct drm_gpu_scheduler *sched) ++{ ++ u32 credits; ++ ++ drm_WARN_ON(sched, check_sub_overflow(sched->credit_limit, ++ atomic_read(&sched->credit_count), ++ &credits)); ++ ++ return credits; ++} ++ ++/** ++ * drm_sched_can_queue -- Can we queue more to the hardware? ++ * @sched: scheduler instance ++ * @entity: the scheduler entity ++ * ++ * Return true if we can push at least one more job from @entity, false ++ * otherwise. ++ */ ++static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched, ++ struct drm_sched_entity *entity) ++{ ++ struct drm_sched_job *s_job; ++ ++ s_job = to_drm_sched_job(spsc_queue_peek(&entity->job_queue)); ++ if (!s_job) ++ return false; ++ ++ if (sched->ops->update_job_credits) { ++ s_job->credits = sched->ops->update_job_credits(s_job); ++ ++ drm_WARN(sched, !s_job->credits, ++ "Jobs with zero credits bypass job-flow control.\n"); ++ } ++ ++ /* If a job exceeds the credit limit, truncate it to the credit limit ++ * itself to guarantee forward progress. ++ */ ++ if (drm_WARN(sched, s_job->credits > sched->credit_limit, ++ "Jobs may not exceed the credit limit, truncate.\n")) ++ s_job->credits = sched->credit_limit; ++ ++ return drm_sched_available_credits(sched) >= s_job->credits; ++} ++ + static __always_inline bool drm_sched_entity_compare_before(struct rb_node *a, + const struct rb_node *b) + { +@@ -186,12 +255,18 @@ void drm_sched_rq_remove_entity(struct drm_sched_rq *rq, + /** + * drm_sched_rq_select_entity_rr - Select an entity which could provide a job to run + * ++ * @sched: the gpu scheduler + * @rq: scheduler run queue to check. + * +- * Try to find a ready entity, returns NULL if none found. ++ * Try to find the next ready entity. ++ * ++ * Return an entity if one is found; return an error-pointer (!NULL) if an ++ * entity was ready, but the scheduler had insufficient credits to accommodate ++ * its job; return NULL, if no ready entity was found. + */ + static struct drm_sched_entity * +-drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) ++drm_sched_rq_select_entity_rr(struct drm_gpu_scheduler *sched, ++ struct drm_sched_rq *rq) + { + struct drm_sched_entity *entity; + +@@ -201,6 +276,14 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) + if (entity) { + list_for_each_entry_continue(entity, &rq->entities, list) { + if (drm_sched_entity_is_ready(entity)) { ++ /* If we can't queue yet, preserve the current ++ * entity in terms of fairness. ++ */ ++ if (!drm_sched_can_queue(sched, entity)) { ++ spin_unlock(&rq->lock); ++ return ERR_PTR(-ENOSPC); ++ } ++ + rq->current_entity = entity; + reinit_completion(&entity->entity_idle); + spin_unlock(&rq->lock); +@@ -210,8 +293,15 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) + } + + list_for_each_entry(entity, &rq->entities, list) { +- + if (drm_sched_entity_is_ready(entity)) { ++ /* If we can't queue yet, preserve the current entity in ++ * terms of fairness. ++ */ ++ if (!drm_sched_can_queue(sched, entity)) { ++ spin_unlock(&rq->lock); ++ return ERR_PTR(-ENOSPC); ++ } ++ + rq->current_entity = entity; + reinit_completion(&entity->entity_idle); + spin_unlock(&rq->lock); +@@ -230,12 +320,18 @@ drm_sched_rq_select_entity_rr(struct drm_sched_rq *rq) + /** + * drm_sched_rq_select_entity_fifo - Select an entity which provides a job to run + * ++ * @sched: the gpu scheduler + * @rq: scheduler run queue to check. + * +- * Find oldest waiting ready entity, returns NULL if none found. ++ * Find oldest waiting ready entity. ++ * ++ * Return an entity if one is found; return an error-pointer (!NULL) if an ++ * entity was ready, but the scheduler had insufficient credits to accommodate ++ * its job; return NULL, if no ready entity was found. + */ + static struct drm_sched_entity * +-drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) ++drm_sched_rq_select_entity_fifo(struct drm_gpu_scheduler *sched, ++ struct drm_sched_rq *rq) + { + struct rb_node *rb; + +@@ -245,6 +341,14 @@ drm_sched_rq_select_entity_fifo(struct drm_sched_rq *rq) + + entity = rb_entry(rb, struct drm_sched_entity, rb_tree_node); + if (drm_sched_entity_is_ready(entity)) { ++ /* If we can't queue yet, preserve the current entity in ++ * terms of fairness. ++ */ ++ if (!drm_sched_can_queue(sched, entity)) { ++ spin_unlock(&rq->lock); ++ return ERR_PTR(-ENOSPC); ++ } ++ + rq->current_entity = entity; + reinit_completion(&entity->entity_idle); + break; +@@ -302,7 +406,7 @@ static void drm_sched_job_done(struct drm_sched_job *s_job, int result) + struct drm_sched_fence *s_fence = s_job->s_fence; + struct drm_gpu_scheduler *sched = s_fence->sched; + +- atomic_dec(&sched->hw_rq_count); ++ atomic_sub(s_job->credits, &sched->credit_count); + atomic_dec(sched->score); + + trace_drm_sched_process_job(s_fence); +@@ -525,7 +629,7 @@ void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad) + &s_job->cb)) { + dma_fence_put(s_job->s_fence->parent); + s_job->s_fence->parent = NULL; +- atomic_dec(&sched->hw_rq_count); ++ atomic_sub(s_job->credits, &sched->credit_count); + } else { + /* + * remove job from pending_list. +@@ -586,7 +690,7 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) { + struct dma_fence *fence = s_job->s_fence->parent; + +- atomic_inc(&sched->hw_rq_count); ++ atomic_add(s_job->credits, &sched->credit_count); + + if (!full_recovery) + continue; +@@ -667,6 +771,8 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs); + * drm_sched_job_init - init a scheduler job + * @job: scheduler job to init + * @entity: scheduler entity to use ++ * @credits: the number of credits this job contributes to the schedulers ++ * credit limit + * @owner: job owner for debugging + * + * Refer to drm_sched_entity_push_job() documentation +@@ -684,7 +790,7 @@ EXPORT_SYMBOL(drm_sched_resubmit_jobs); + */ + int drm_sched_job_init(struct drm_sched_job *job, + struct drm_sched_entity *entity, +- void *owner) ++ u32 credits, void *owner) + { + if (!entity->rq) { + /* This will most likely be followed by missing frames +@@ -695,7 +801,13 @@ int drm_sched_job_init(struct drm_sched_job *job, + return -ENOENT; + } + ++ if (unlikely(!credits)) { ++ pr_err("*ERROR* %s: credits cannot be 0!\n", __func__); ++ return -EINVAL; ++ } ++ + job->entity = entity; ++ job->credits = credits; + job->s_fence = drm_sched_fence_alloc(entity, owner); + if (!job->s_fence) + return -ENOMEM; +@@ -907,21 +1019,10 @@ void drm_sched_job_cleanup(struct drm_sched_job *job) + } + EXPORT_SYMBOL(drm_sched_job_cleanup); + +-/** +- * drm_sched_can_queue -- Can we queue more to the hardware? +- * @sched: scheduler instance +- * +- * Return true if we can push more jobs to the hw, otherwise false. +- */ +-static bool drm_sched_can_queue(struct drm_gpu_scheduler *sched) +-{ +- return atomic_read(&sched->hw_rq_count) < +- sched->hw_submission_limit; +-} +- + /** + * drm_sched_wakeup - Wake up the scheduler if it is ready to queue + * @sched: scheduler instance ++ * @entity: the scheduler entity + * + * Wake up the scheduler if we can queue jobs. + */ +@@ -929,7 +1030,7 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched, + struct drm_sched_entity *entity) + { + if (drm_sched_entity_is_ready(entity)) +- if (drm_sched_can_queue(sched)) ++ if (drm_sched_can_queue(sched, entity)) + drm_sched_run_job_queue(sched); + } + +@@ -938,7 +1039,11 @@ void drm_sched_wakeup(struct drm_gpu_scheduler *sched, + * + * @sched: scheduler instance + * +- * Returns the entity to process or NULL if none are found. ++ * Return an entity to process or NULL if none are found. ++ * ++ * Note, that we break out of the for-loop when "entity" is non-null, which can ++ * also be an error-pointer--this assures we don't process lower priority ++ * run-queues. See comments in the respectively called functions. + */ + static struct drm_sched_entity * + drm_sched_select_entity(struct drm_gpu_scheduler *sched) +@@ -946,19 +1051,16 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + struct drm_sched_entity *entity; + int i; + +- if (!drm_sched_can_queue(sched)) +- return NULL; +- + /* Kernel run queue has higher priority than normal run queue*/ + for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { + entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? +- drm_sched_rq_select_entity_fifo(sched->sched_rq[i]) : +- drm_sched_rq_select_entity_rr(sched->sched_rq[i]); ++ drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) : ++ drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]); + if (entity) + break; + } + +- return entity; ++ return IS_ERR(entity) ? NULL : entity; + } + + /** +@@ -1094,7 +1196,7 @@ static void drm_sched_run_job_work(struct work_struct *w) + + s_fence = sched_job->s_fence; + +- atomic_inc(&sched->hw_rq_count); ++ atomic_add(sched_job->credits, &sched->credit_count); + drm_sched_job_begin(sched_job); + + trace_drm_run_job(sched_job, entity); +@@ -1129,7 +1231,7 @@ static void drm_sched_run_job_work(struct work_struct *w) + * @submit_wq: workqueue to use for submission. If NULL, an ordered wq is + * allocated and used + * @num_rqs: number of runqueues, one for each priority, up to DRM_SCHED_PRIORITY_COUNT +- * @hw_submission: number of hw submissions that can be in flight ++ * @credit_limit: the number of credits this scheduler can hold from all jobs + * @hang_limit: number of times to allow a job to hang before dropping it + * @timeout: timeout value in jiffies for the scheduler + * @timeout_wq: workqueue to use for timeout work. If NULL, the system_wq is +@@ -1143,14 +1245,14 @@ static void drm_sched_run_job_work(struct work_struct *w) + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + struct workqueue_struct *submit_wq, +- u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, ++ u32 num_rqs, u32 credit_limit, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev) + { + int i, ret; + + sched->ops = ops; +- sched->hw_submission_limit = hw_submission; ++ sched->credit_limit = credit_limit; + sched->name = name; + sched->timeout = timeout; + sched->timeout_wq = timeout_wq ? : system_wq; +@@ -1199,7 +1301,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + init_waitqueue_head(&sched->job_scheduled); + INIT_LIST_HEAD(&sched->pending_list); + spin_lock_init(&sched->job_list_lock); +- atomic_set(&sched->hw_rq_count, 0); ++ atomic_set(&sched->credit_count, 0); + INIT_DELAYED_WORK(&sched->work_tdr, drm_sched_job_timedout); + INIT_WORK(&sched->work_run_job, drm_sched_run_job_work); + INIT_WORK(&sched->work_free_job, drm_sched_free_job_work); +diff --git a/drivers/gpu/drm/v3d/v3d_gem.c b/drivers/gpu/drm/v3d/v3d_gem.c +index b8980440d137f2..1ea266f9a4d669 100644 +--- a/drivers/gpu/drm/v3d/v3d_gem.c ++++ b/drivers/gpu/drm/v3d/v3d_gem.c +@@ -473,7 +473,7 @@ v3d_job_init(struct v3d_dev *v3d, struct drm_file *file_priv, + job->free = free; + + ret = drm_sched_job_init(&job->base, &v3d_priv->sched_entity[queue], +- v3d_priv); ++ 1, v3d_priv); + if (ret) + goto fail; + +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 562b2232c81bad..0750bed01acf78 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -320,6 +320,7 @@ struct drm_sched_fence *to_drm_sched_fence(struct dma_fence *f); + * @sched: the scheduler instance on which this job is scheduled. + * @s_fence: contains the fences for the scheduling of job. + * @finish_cb: the callback for the finished fence. ++ * @credits: the number of credits this job contributes to the scheduler + * @work: Helper to reschdeule job kill to different context. + * @id: a unique id assigned to each job scheduled on the scheduler. + * @karma: increment on every hang caused by this job. If this exceeds the hang +@@ -339,6 +340,8 @@ struct drm_sched_job { + struct drm_gpu_scheduler *sched; + struct drm_sched_fence *s_fence; + ++ u32 credits; ++ + /* + * work is used only after finish_cb has been used and will not be + * accessed anymore. +@@ -462,13 +465,27 @@ struct drm_sched_backend_ops { + * and it's time to clean it up. + */ + void (*free_job)(struct drm_sched_job *sched_job); ++ ++ /** ++ * @update_job_credits: Called when the scheduler is considering this ++ * job for execution. ++ * ++ * This callback returns the number of credits the job would take if ++ * pushed to the hardware. Drivers may use this to dynamically update ++ * the job's credit count. For instance, deduct the number of credits ++ * for already signalled native fences. ++ * ++ * This callback is optional. ++ */ ++ u32 (*update_job_credits)(struct drm_sched_job *sched_job); + }; + + /** + * struct drm_gpu_scheduler - scheduler instance-specific data + * + * @ops: backend operations provided by the driver. +- * @hw_submission_limit: the max size of the hardware queue. ++ * @credit_limit: the credit limit of this scheduler ++ * @credit_count: the current credit count of this scheduler + * @timeout: the time after which a job is removed from the scheduler. + * @name: name of the ring for which this scheduler is being used. + * @num_rqs: Number of run-queues. This is at most DRM_SCHED_PRIORITY_COUNT, +@@ -477,7 +494,6 @@ struct drm_sched_backend_ops { + * @job_scheduled: once @drm_sched_entity_do_release is called the scheduler + * waits on this wait queue until all the scheduled jobs are + * finished. +- * @hw_rq_count: the number of jobs currently in the hardware queue. + * @job_id_count: used to assign unique id to the each job. + * @submit_wq: workqueue used to queue @work_run_job and @work_free_job + * @timeout_wq: workqueue used to queue @work_tdr +@@ -501,13 +517,13 @@ struct drm_sched_backend_ops { + */ + struct drm_gpu_scheduler { + const struct drm_sched_backend_ops *ops; +- uint32_t hw_submission_limit; ++ u32 credit_limit; ++ atomic_t credit_count; + long timeout; + const char *name; + u32 num_rqs; + struct drm_sched_rq **sched_rq; + wait_queue_head_t job_scheduled; +- atomic_t hw_rq_count; + atomic64_t job_id_count; + struct workqueue_struct *submit_wq; + struct workqueue_struct *timeout_wq; +@@ -529,14 +545,14 @@ struct drm_gpu_scheduler { + int drm_sched_init(struct drm_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + struct workqueue_struct *submit_wq, +- u32 num_rqs, uint32_t hw_submission, unsigned int hang_limit, ++ u32 num_rqs, u32 credit_limit, unsigned int hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev); + + void drm_sched_fini(struct drm_gpu_scheduler *sched); + int drm_sched_job_init(struct drm_sched_job *job, + struct drm_sched_entity *entity, +- void *owner); ++ u32 credits, void *owner); + void drm_sched_job_arm(struct drm_sched_job *job); + int drm_sched_job_add_dependency(struct drm_sched_job *job, + struct dma_fence *fence); + +From 9103e54424ec5d44b707ddadc7183554793fb9c5 Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Wed, 22 Nov 2023 23:58:53 -0500 +Subject: [PATCH 072/150] drm/sched: Fix bounds limiting when given a malformed + entity +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +If we're given a malformed entity in drm_sched_entity_init()--shouldn't +happen, but we verify--with out-of-bounds priority value, we set it to an +allowed value. Fix the expression which sets this limit. + +Signed-off-by: Luben Tuikov +Fixes: 56e449603f0ac5 ("drm/sched: Convert the GPU scheduler to variable number of run-queues") +Link: https://patchwork.freedesktop.org/patch/msgid/20231123122422.167832-2-ltuikov89@gmail.com +Reviewed-by: Christian König +Link: https://lore.kernel.org/r/dbb91dbe-ef77-4d79-aaf9-2adb171c1d7a@amd.com +--- + drivers/gpu/drm/scheduler/sched_entity.c | 13 ++++++++----- + 1 file changed, 8 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index 4d42b1e4daa67f..ee645d38e98d94 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -81,12 +81,15 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + */ + pr_warn("%s: called with uninitialized scheduler\n", __func__); + } else if (num_sched_list) { +- /* The "priority" of an entity cannot exceed the number +- * of run-queues of a scheduler. ++ /* The "priority" of an entity cannot exceed the number of run-queues of a ++ * scheduler. Protect against num_rqs being 0, by converting to signed. + */ +- if (entity->priority >= sched_list[0]->num_rqs) +- entity->priority = max_t(u32, sched_list[0]->num_rqs, +- DRM_SCHED_PRIORITY_MIN); ++ if (entity->priority >= sched_list[0]->num_rqs) { ++ drm_err(sched_list[0], "entity with out-of-bounds priority:%u num_rqs:%u\n", ++ entity->priority, sched_list[0]->num_rqs); ++ entity->priority = max_t(s32, (s32) sched_list[0]->num_rqs - 1, ++ (s32) DRM_SCHED_PRIORITY_MIN); ++ } + entity->rq = sched_list[0]->sched_rq[entity->priority]; + } + + +From 182b052b6539071d55f265620c89918dff5bf782 Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Tue, 14 Nov 2023 20:40:58 -0500 +Subject: [PATCH 073/150] drm/sched: Rename priority MIN to LOW +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Rename DRM_SCHED_PRIORITY_MIN to DRM_SCHED_PRIORITY_LOW. + +This mirrors DRM_SCHED_PRIORITY_HIGH, for a list of DRM scheduler priorities +in ascending order, + DRM_SCHED_PRIORITY_LOW, + DRM_SCHED_PRIORITY_NORMAL, + DRM_SCHED_PRIORITY_HIGH, + DRM_SCHED_PRIORITY_KERNEL. + +Cc: Rob Clark +Cc: Abhinav Kumar +Cc: Dmitry Baryshkov +Cc: Danilo Krummrich +Cc: Alex Deucher +Cc: Christian König +Cc: linux-arm-msm@vger.kernel.org +Cc: freedreno@lists.freedesktop.org +Cc: dri-devel@lists.freedesktop.org +Signed-off-by: Luben Tuikov +Reviewed-by: Christian König +Link: https://patchwork.freedesktop.org/patch/msgid/20231124052752.6915-5-ltuikov89@gmail.com +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c | 4 ++-- + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- + drivers/gpu/drm/msm/msm_gpu.h | 2 +- + drivers/gpu/drm/scheduler/sched_entity.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 10 +++++----- + include/drm/gpu_scheduler.h | 2 +- + 6 files changed, 11 insertions(+), 11 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +index 1ed2142a6e7bf0..76ff7b17b6c1cb 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ctx.c +@@ -68,10 +68,10 @@ amdgpu_ctx_to_drm_sched_prio(int32_t ctx_prio) + return DRM_SCHED_PRIORITY_NORMAL; + + case AMDGPU_CTX_PRIORITY_VERY_LOW: +- return DRM_SCHED_PRIORITY_MIN; ++ return DRM_SCHED_PRIORITY_LOW; + + case AMDGPU_CTX_PRIORITY_LOW: +- return DRM_SCHED_PRIORITY_MIN; ++ return DRM_SCHED_PRIORITY_LOW; + + case AMDGPU_CTX_PRIORITY_NORMAL: + return DRM_SCHED_PRIORITY_NORMAL; +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +index d4f3fd4c6aeb85..160e8450a275a1 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +@@ -329,7 +329,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) + int i; + + /* Signal all jobs not yet scheduled */ +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) { +diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h +index b39cd332751dc8..de76fd27d2264c 100644 +--- a/drivers/gpu/drm/msm/msm_gpu.h ++++ b/drivers/gpu/drm/msm/msm_gpu.h +@@ -337,7 +337,7 @@ struct msm_gpu_perfcntr { + * DRM_SCHED_PRIORITY_KERNEL priority level is treated specially in some + * cases, so we don't use it (no need for kernel generated jobs). + */ +-#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_HIGH - DRM_SCHED_PRIORITY_MIN) ++#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_HIGH - DRM_SCHED_PRIORITY_LOW) + + /** + * struct msm_file_private - per-drm_file context +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index ee645d38e98d94..dd2b8f777f5186 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -88,7 +88,7 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + drm_err(sched_list[0], "entity with out-of-bounds priority:%u num_rqs:%u\n", + entity->priority, sched_list[0]->num_rqs); + entity->priority = max_t(s32, (s32) sched_list[0]->num_rqs - 1, +- (s32) DRM_SCHED_PRIORITY_MIN); ++ (s32) DRM_SCHED_PRIORITY_LOW); + } + entity->rq = sched_list[0]->sched_rq[entity->priority]; + } +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 044a8c4875ba64..b6d7bc49ff6ef4 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1052,7 +1052,7 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + int i; + + /* Kernel run queue has higher priority than normal run queue*/ +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { + entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? + drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) : + drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]); +@@ -1291,7 +1291,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + if (!sched->sched_rq) + goto Out_free; + sched->num_rqs = num_rqs; +- for (i = DRM_SCHED_PRIORITY_MIN; i < sched->num_rqs; i++) { ++ for (i = DRM_SCHED_PRIORITY_LOW; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); + if (!sched->sched_rq[i]) + goto Out_unroll; +@@ -1312,7 +1312,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + sched->ready = true; + return 0; + Out_unroll: +- for (--i ; i >= DRM_SCHED_PRIORITY_MIN; i--) ++ for (--i ; i >= DRM_SCHED_PRIORITY_LOW; i--) + kfree(sched->sched_rq[i]); + Out_free: + kfree(sched->sched_rq); +@@ -1338,7 +1338,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + + drm_sched_wqueue_stop(sched); + +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_MIN; i--) { ++ for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); +@@ -1390,7 +1390,7 @@ void drm_sched_increase_karma(struct drm_sched_job *bad) + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + +- for (i = DRM_SCHED_PRIORITY_MIN; ++ for (i = DRM_SCHED_PRIORITY_LOW; + i < min_t(typeof(sched->num_rqs), sched->num_rqs, DRM_SCHED_PRIORITY_KERNEL); + i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 0750bed01acf78..46c615b1c3f9b6 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -63,7 +63,7 @@ struct drm_file; + * to an array, and as such should start at 0. + */ + enum drm_sched_priority { +- DRM_SCHED_PRIORITY_MIN, ++ DRM_SCHED_PRIORITY_LOW, + DRM_SCHED_PRIORITY_NORMAL, + DRM_SCHED_PRIORITY_HIGH, + DRM_SCHED_PRIORITY_KERNEL, + +From 7c0861bc3f3441d921e8f6a5c7c18b3b1df2a431 Mon Sep 17 00:00:00 2001 +From: Luben Tuikov +Date: Wed, 22 Nov 2023 22:08:48 -0500 +Subject: [PATCH 074/150] drm/sched: Reverse run-queue priority enumeration +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Reverse run-queue priority enumeration such that the higest priority is now 0, +and for each consecutive integer the prioirty diminishes. + +Run-queues correspond to priorities. To an external observer a scheduler +created with a single run-queue, and another created with +DRM_SCHED_PRIORITY_COUNT number of run-queues, should always schedule +sched->sched_rq[0] with the same "priority", as that index run-queue exists in +both schedulers, i.e. a scheduler with one run-queue or many. This patch makes +it so. + +In other words, the "priority" of sched->sched_rq[n], n >= 0, is the same for +any scheduler created with any allowable number of run-queues (priorities), 0 +to DRM_SCHED_PRIORITY_COUNT. + +Cc: Rob Clark +Cc: Abhinav Kumar +Cc: Dmitry Baryshkov +Cc: Danilo Krummrich +Cc: Alex Deucher +Cc: Christian König +Cc: linux-arm-msm@vger.kernel.org +Cc: freedreno@lists.freedesktop.org +Cc: dri-devel@lists.freedesktop.org +Signed-off-by: Luben Tuikov +Reviewed-by: Christian König +Link: https://patchwork.freedesktop.org/patch/msgid/20231124052752.6915-6-ltuikov89@gmail.com +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_job.c | 2 +- + drivers/gpu/drm/msm/msm_gpu.h | 2 +- + drivers/gpu/drm/scheduler/sched_entity.c | 5 +++-- + drivers/gpu/drm/scheduler/sched_main.c | 15 +++++++-------- + include/drm/gpu_scheduler.h | 6 +++--- + 5 files changed, 15 insertions(+), 15 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +index 160e8450a275a1..8043d9b92fac81 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +@@ -329,7 +329,7 @@ void amdgpu_job_stop_all_jobs_on_sched(struct drm_gpu_scheduler *sched) + int i; + + /* Signal all jobs not yet scheduled */ +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { ++ for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + spin_lock(&rq->lock); + list_for_each_entry(s_entity, &rq->entities, list) { +diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h +index de76fd27d2264c..175d7fa7954aaa 100644 +--- a/drivers/gpu/drm/msm/msm_gpu.h ++++ b/drivers/gpu/drm/msm/msm_gpu.h +@@ -337,7 +337,7 @@ struct msm_gpu_perfcntr { + * DRM_SCHED_PRIORITY_KERNEL priority level is treated specially in some + * cases, so we don't use it (no need for kernel generated jobs). + */ +-#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_HIGH - DRM_SCHED_PRIORITY_LOW) ++#define NR_SCHED_PRIORITIES (1 + DRM_SCHED_PRIORITY_LOW - DRM_SCHED_PRIORITY_HIGH) + + /** + * struct msm_file_private - per-drm_file context +diff --git a/drivers/gpu/drm/scheduler/sched_entity.c b/drivers/gpu/drm/scheduler/sched_entity.c +index dd2b8f777f5186..3c4f5a392b0646 100644 +--- a/drivers/gpu/drm/scheduler/sched_entity.c ++++ b/drivers/gpu/drm/scheduler/sched_entity.c +@@ -82,13 +82,14 @@ int drm_sched_entity_init(struct drm_sched_entity *entity, + pr_warn("%s: called with uninitialized scheduler\n", __func__); + } else if (num_sched_list) { + /* The "priority" of an entity cannot exceed the number of run-queues of a +- * scheduler. Protect against num_rqs being 0, by converting to signed. ++ * scheduler. Protect against num_rqs being 0, by converting to signed. Choose ++ * the lowest priority available. + */ + if (entity->priority >= sched_list[0]->num_rqs) { + drm_err(sched_list[0], "entity with out-of-bounds priority:%u num_rqs:%u\n", + entity->priority, sched_list[0]->num_rqs); + entity->priority = max_t(s32, (s32) sched_list[0]->num_rqs - 1, +- (s32) DRM_SCHED_PRIORITY_LOW); ++ (s32) DRM_SCHED_PRIORITY_KERNEL); + } + entity->rq = sched_list[0]->sched_rq[entity->priority]; + } +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index b6d7bc49ff6ef4..682aebe96db781 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1051,8 +1051,9 @@ drm_sched_select_entity(struct drm_gpu_scheduler *sched) + struct drm_sched_entity *entity; + int i; + +- /* Kernel run queue has higher priority than normal run queue*/ +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { ++ /* Start with the highest priority. ++ */ ++ for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + entity = drm_sched_policy == DRM_SCHED_POLICY_FIFO ? + drm_sched_rq_select_entity_fifo(sched, sched->sched_rq[i]) : + drm_sched_rq_select_entity_rr(sched, sched->sched_rq[i]); +@@ -1291,7 +1292,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + if (!sched->sched_rq) + goto Out_free; + sched->num_rqs = num_rqs; +- for (i = DRM_SCHED_PRIORITY_LOW; i < sched->num_rqs; i++) { ++ for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); + if (!sched->sched_rq[i]) + goto Out_unroll; +@@ -1312,7 +1313,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + sched->ready = true; + return 0; + Out_unroll: +- for (--i ; i >= DRM_SCHED_PRIORITY_LOW; i--) ++ for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--) + kfree(sched->sched_rq[i]); + Out_free: + kfree(sched->sched_rq); +@@ -1338,7 +1339,7 @@ void drm_sched_fini(struct drm_gpu_scheduler *sched) + + drm_sched_wqueue_stop(sched); + +- for (i = sched->num_rqs - 1; i >= DRM_SCHED_PRIORITY_LOW; i--) { ++ for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); +@@ -1390,9 +1391,7 @@ void drm_sched_increase_karma(struct drm_sched_job *bad) + if (bad->s_priority != DRM_SCHED_PRIORITY_KERNEL) { + atomic_inc(&bad->karma); + +- for (i = DRM_SCHED_PRIORITY_LOW; +- i < min_t(typeof(sched->num_rqs), sched->num_rqs, DRM_SCHED_PRIORITY_KERNEL); +- i++) { ++ for (i = DRM_SCHED_PRIORITY_HIGH; i < sched->num_rqs; i++) { + struct drm_sched_rq *rq = sched->sched_rq[i]; + + spin_lock(&rq->lock); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 46c615b1c3f9b6..28cd4a9f4aad8a 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -63,10 +63,10 @@ struct drm_file; + * to an array, and as such should start at 0. + */ + enum drm_sched_priority { +- DRM_SCHED_PRIORITY_LOW, +- DRM_SCHED_PRIORITY_NORMAL, +- DRM_SCHED_PRIORITY_HIGH, + DRM_SCHED_PRIORITY_KERNEL, ++ DRM_SCHED_PRIORITY_HIGH, ++ DRM_SCHED_PRIORITY_NORMAL, ++ DRM_SCHED_PRIORITY_LOW, + + DRM_SCHED_PRIORITY_COUNT + }; + +From 482e133da38e5239f185b6d5f79f46a7e4c7de5c Mon Sep 17 00:00:00 2001 +From: Bert Karwatzki +Date: Mon, 27 Nov 2023 17:09:55 +0100 +Subject: [PATCH 075/150] drm/sched: Partial revert of "Qualify + drm_sched_wakeup() by drm_sched_entity_is_ready()" + +Commit f3123c2590005c, in combination with the use of work queues by the GPU +scheduler, leads to random lock-ups of the GUI. + +This is a partial revert of of commit f3123c2590005c since drm_sched_wakeup() still +needs its entity argument to pass it to drm_sched_can_queue(). + +Link: https://gitlab.freedesktop.org/drm/amd/-/issues/2994 +Link: https://lists.freedesktop.org/archives/dri-devel/2023-November/431606.html + +Signed-off-by: Bert Karwatzki +Link: https://patchwork.freedesktop.org/patch/msgid/20231127160955.87879-1-spasswolf@web.de +Link: https://lore.kernel.org/r/36bece178ff5dc705065e53d1e5e41f6db6d87e4.camel@web.de +Fixes: f3123c2590005c ("drm/sched: Qualify drm_sched_wakeup() by drm_sched_entity_is_ready()") +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 5 ++--- + 1 file changed, 2 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 682aebe96db781..550492a7a031d7 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1029,9 +1029,8 @@ EXPORT_SYMBOL(drm_sched_job_cleanup); + void drm_sched_wakeup(struct drm_gpu_scheduler *sched, + struct drm_sched_entity *entity) + { +- if (drm_sched_entity_is_ready(entity)) +- if (drm_sched_can_queue(sched, entity)) +- drm_sched_run_job_queue(sched); ++ if (drm_sched_can_queue(sched, entity)) ++ drm_sched_run_job_queue(sched); + } + + /** + +From 0db44f3df44a2acb57e4ed6c344113c427a4d051 Mon Sep 17 00:00:00 2001 +From: Markus Elfring +Date: Tue, 26 Dec 2023 16:30:25 +0100 +Subject: [PATCH 076/150] drm/sched: One function call less in drm_sched_init() + after error detection + +The kfree() function was called in one case by the +drm_sched_init() function during error handling +even if the passed data structure member contained a null pointer. +This issue was detected by using the Coccinelle software. + +Thus adjust a jump target. + +Signed-off-by: Markus Elfring +Link: https://patchwork.freedesktop.org/patch/msgid/85066512-983d-480c-a44d-32405ab1b80e@web.de +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 550492a7a031d7..b99d4e9ff10944 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1289,7 +1289,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq), + GFP_KERNEL | __GFP_ZERO); + if (!sched->sched_rq) +- goto Out_free; ++ goto Out_check_own; + sched->num_rqs = num_rqs; + for (i = DRM_SCHED_PRIORITY_KERNEL; i < sched->num_rqs; i++) { + sched->sched_rq[i] = kzalloc(sizeof(*sched->sched_rq[i]), GFP_KERNEL); +@@ -1314,9 +1314,10 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + Out_unroll: + for (--i ; i >= DRM_SCHED_PRIORITY_KERNEL; i--) + kfree(sched->sched_rq[i]); +-Out_free: ++ + kfree(sched->sched_rq); + sched->sched_rq = NULL; ++Out_check_own: + if (sched->own_submit_wq) + destroy_workqueue(sched->submit_wq); + drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); + +From 16cb274eca404e4ad4c86f62f36dd98f1cbf0b3b Mon Sep 17 00:00:00 2001 +From: Markus Elfring +Date: Tue, 26 Dec 2023 16:37:37 +0100 +Subject: [PATCH 077/150] drm/sched: Return an error code only as a constant in + drm_sched_init() + +Return an error code without storing it in an intermediate variable. + +Signed-off-by: Markus Elfring +Link: https://patchwork.freedesktop.org/patch/msgid/85f8004e-f0c9-42d9-8c59-30f1b4e0b89e@web.de +Reviewed-by: Luben Tuikov +Signed-off-by: Luben Tuikov +--- + drivers/gpu/drm/scheduler/sched_main.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index b99d4e9ff10944..1abbcdf38430e8 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1249,7 +1249,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, struct device *dev) + { +- int i, ret; ++ int i; + + sched->ops = ops; + sched->credit_limit = credit_limit; +@@ -1285,7 +1285,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + + sched->own_submit_wq = true; + } +- ret = -ENOMEM; ++ + sched->sched_rq = kmalloc_array(num_rqs, sizeof(*sched->sched_rq), + GFP_KERNEL | __GFP_ZERO); + if (!sched->sched_rq) +@@ -1321,7 +1321,7 @@ int drm_sched_init(struct drm_gpu_scheduler *sched, + if (sched->own_submit_wq) + destroy_workqueue(sched->submit_wq); + drm_err(sched, "%s: Failed to setup GPU scheduler--out of memory\n", __func__); +- return ret; ++ return -ENOMEM; + } + EXPORT_SYMBOL(drm_sched_init); + + +From afec7e2089125230fd84fec3ef4db8e277bd8eb8 Mon Sep 17 00:00:00 2001 +From: Matthew Brost +Date: Wed, 24 Jan 2024 13:08:11 -0800 +Subject: [PATCH 078/150] drm/sched: Drain all entities in DRM sched run job + worker + +All entities must be drained in the DRM scheduler run job worker to +avoid the following case. An entity found that is ready, no job found +ready on entity, and run job worker goes idle with other entities + jobs +ready. Draining all ready entities (i.e. loop over all ready entities) +in the run job worker ensures all job that are ready will be scheduled. + +Cc: Thorsten Leemhuis +Reported-by: Mikhail Gavrilov +Closes: https://lore.kernel.org/all/CABXGCsM2VLs489CH-vF-1539-s3in37=bwuOWtoeeE+q26zE+Q@mail.gmail.com/ +Reported-and-tested-by: Mario Limonciello +Closes: https://gitlab.freedesktop.org/drm/amd/-/issues/3124 +Link: https://lore.kernel.org/all/20240123021155.2775-1-mario.limonciello@amd.com/ +Reported-and-tested-by: Vlastimil Babka +Closes: https://lore.kernel.org/dri-devel/05ddb2da-b182-4791-8ef7-82179fd159a8@amd.com/T/#m0c31d4d1b9ae9995bb880974c4f1dbaddc33a48a +Signed-off-by: Matthew Brost +Signed-off-by: Dave Airlie +Link: https://patchwork.freedesktop.org/patch/msgid/20240124210811.1639040-1-matthew.brost@intel.com +--- + drivers/gpu/drm/scheduler/sched_main.c | 15 +++++++-------- + 1 file changed, 7 insertions(+), 8 deletions(-) + +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 1abbcdf38430e8..8acbef7ae53d11 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -1178,21 +1178,20 @@ static void drm_sched_run_job_work(struct work_struct *w) + struct drm_sched_entity *entity; + struct dma_fence *fence; + struct drm_sched_fence *s_fence; +- struct drm_sched_job *sched_job; ++ struct drm_sched_job *sched_job = NULL; + int r; + + if (READ_ONCE(sched->pause_submit)) + return; + +- entity = drm_sched_select_entity(sched); ++ /* Find entity with a ready job */ ++ while (!sched_job && (entity = drm_sched_select_entity(sched))) { ++ sched_job = drm_sched_entity_pop_job(entity); ++ if (!sched_job) ++ complete_all(&entity->entity_idle); ++ } + if (!entity) +- return; +- +- sched_job = drm_sched_entity_pop_job(entity); +- if (!sched_job) { +- complete_all(&entity->entity_idle); + return; /* No more work */ +- } + + s_fence = sched_job->s_fence; + + +From 534790e7a2a812facf8653378a82c22e8e7e22e7 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Thomas=20Hellstr=C3=B6m?= +Date: Wed, 6 Sep 2023 11:50:39 +0200 +Subject: [PATCH 079/150] drm/drm_exec: Work around a WW mutex lockdep oddity +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +If *any* object of a certain WW mutex class is locked, lockdep will +consider *all* mutexes of that class as locked. Also the lock allocation +tracking code will apparently register only the address of the first +mutex of a given class locked in a sequence. +This has the odd consequence that if that first mutex is unlocked while +other mutexes of the same class remain locked and then its memory then +freed, the lock alloc tracking code will incorrectly assume that memory +is freed with a held lock in there. + +For now, work around that for drm_exec by releasing the first grabbed +object lock last. + +v2: +- Fix a typo (Danilo Krummrich) +- Reword the commit message a bit. +- Add a Fixes: tag + +Related lock alloc tracking warning: +[ 322.660067] ========================= +[ 322.660070] WARNING: held lock freed! +[ 322.660074] 6.5.0-rc7+ #155 Tainted: G U N +[ 322.660078] ------------------------- +[ 322.660081] kunit_try_catch/4981 is freeing memory ffff888112adc000-ffff888112adc3ff, with a lock still held there! +[ 322.660089] ffff888112adc1a0 (reservation_ww_class_mutex){+.+.}-{3:3}, at: drm_exec_lock_obj+0x11a/0x600 [drm_exec] +[ 322.660104] 2 locks held by kunit_try_catch/4981: +[ 322.660108] #0: ffffc9000343fe18 (reservation_ww_class_acquire){+.+.}-{0:0}, at: test_early_put+0x22f/0x490 [drm_exec_test] +[ 322.660123] #1: ffff888112adc1a0 (reservation_ww_class_mutex){+.+.}-{3:3}, at: drm_exec_lock_obj+0x11a/0x600 [drm_exec] +[ 322.660135] + stack backtrace: +[ 322.660139] CPU: 7 PID: 4981 Comm: kunit_try_catch Tainted: G U N 6.5.0-rc7+ #155 +[ 322.660146] Hardware name: ASUS System Product Name/PRIME B560M-A AC, BIOS 0403 01/26/2021 +[ 322.660152] Call Trace: +[ 322.660155] +[ 322.660158] dump_stack_lvl+0x57/0x90 +[ 322.660164] debug_check_no_locks_freed+0x20b/0x2b0 +[ 322.660172] slab_free_freelist_hook+0xa1/0x160 +[ 322.660179] ? drm_exec_unlock_all+0x168/0x2a0 [drm_exec] +[ 322.660186] __kmem_cache_free+0xb2/0x290 +[ 322.660192] drm_exec_unlock_all+0x168/0x2a0 [drm_exec] +[ 322.660200] drm_exec_fini+0xf/0x1c0 [drm_exec] +[ 322.660206] test_early_put+0x289/0x490 [drm_exec_test] +[ 322.660215] ? __pfx_test_early_put+0x10/0x10 [drm_exec_test] +[ 322.660222] ? __kasan_check_byte+0xf/0x40 +[ 322.660227] ? __ksize+0x63/0x140 +[ 322.660233] ? drmm_add_final_kfree+0x3e/0xa0 [drm] +[ 322.660289] ? _raw_spin_unlock_irqrestore+0x30/0x60 +[ 322.660294] ? lockdep_hardirqs_on+0x7d/0x100 +[ 322.660301] ? __pfx_kunit_try_run_case+0x10/0x10 [kunit] +[ 322.660310] ? __pfx_kunit_generic_run_threadfn_adapter+0x10/0x10 [kunit] +[ 322.660319] kunit_generic_run_threadfn_adapter+0x4a/0x90 [kunit] +[ 322.660328] kthread+0x2e7/0x3c0 +[ 322.660334] ? __pfx_kthread+0x10/0x10 +[ 322.660339] ret_from_fork+0x2d/0x70 +[ 322.660345] ? __pfx_kthread+0x10/0x10 +[ 322.660349] ret_from_fork_asm+0x1b/0x30 +[ 322.660358] +[ 322.660818] ok 8 test_early_put + +Cc: Christian König +Cc: Boris Brezillon +Cc: Danilo Krummrich +Cc: dri-devel@lists.freedesktop.org +Fixes: 09593216bff1 ("drm: execution context for GEM buffers v7") +Signed-off-by: Thomas Hellström +Reviewed-by: Boris Brezillon +Reviewed-by: Danilo Krummrich +Reviewed-by: Christian König +Link: https://patchwork.freedesktop.org/patch/msgid/20230906095039.3320-4-thomas.hellstrom@linux.intel.com +--- + drivers/gpu/drm/drm_exec.c | 2 +- + include/drm/drm_exec.h | 35 +++++++++++++++++++++++++++++++---- + 2 files changed, 32 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/drm_exec.c b/drivers/gpu/drm/drm_exec.c +index 303013c23f3385..48ee851b61d904 100644 +--- a/drivers/gpu/drm/drm_exec.c ++++ b/drivers/gpu/drm/drm_exec.c +@@ -56,7 +56,7 @@ static void drm_exec_unlock_all(struct drm_exec *exec) + struct drm_gem_object *obj; + unsigned long index; + +- drm_exec_for_each_locked_object(exec, index, obj) { ++ drm_exec_for_each_locked_object_reverse(exec, index, obj) { + dma_resv_unlock(obj->resv); + drm_gem_object_put(obj); + } +diff --git a/include/drm/drm_exec.h b/include/drm/drm_exec.h +index ae52623e43c8ab..8069d57672d729 100644 +--- a/include/drm/drm_exec.h ++++ b/include/drm/drm_exec.h +@@ -50,6 +50,20 @@ struct drm_exec { + struct drm_gem_object *prelocked; + }; + ++/** ++ * drm_exec_obj() - Return the object for a give drm_exec index ++ * @exec: Pointer to the drm_exec context ++ * @index: The index. ++ * ++ * Return: Pointer to the locked object corresponding to @index if ++ * index is within the number of locked objects. NULL otherwise. ++ */ ++static inline struct drm_gem_object * ++drm_exec_obj(struct drm_exec *exec, unsigned long index) ++{ ++ return index < exec->num_objects ? exec->objects[index] : NULL; ++} ++ + /** + * drm_exec_for_each_locked_object - iterate over all the locked objects + * @exec: drm_exec object +@@ -58,10 +72,23 @@ struct drm_exec { + * + * Iterate over all the locked GEM objects inside the drm_exec object. + */ +-#define drm_exec_for_each_locked_object(exec, index, obj) \ +- for (index = 0, obj = (exec)->objects[0]; \ +- index < (exec)->num_objects; \ +- ++index, obj = (exec)->objects[index]) ++#define drm_exec_for_each_locked_object(exec, index, obj) \ ++ for ((index) = 0; ((obj) = drm_exec_obj(exec, index)); ++(index)) ++ ++/** ++ * drm_exec_for_each_locked_object_reverse - iterate over all the locked ++ * objects in reverse locking order ++ * @exec: drm_exec object ++ * @index: unsigned long index for the iteration ++ * @obj: the current GEM object ++ * ++ * Iterate over all the locked GEM objects inside the drm_exec object in ++ * reverse locking order. Note that @index may go below zero and wrap, ++ * but that will be caught by drm_exec_obj(), returning a NULL object. ++ */ ++#define drm_exec_for_each_locked_object_reverse(exec, index, obj) \ ++ for ((index) = (exec)->num_objects - 1; \ ++ ((obj) = drm_exec_obj(exec, index)); --(index)) + + /** + * drm_exec_until_all_locked - loop until all GEM objects are locked + +From b32ab126a9e7cd211e0cafc3c2188d6fac028b17 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:15 +0100 +Subject: [PATCH 080/150] drm/panthor: Add uAPI + +Panthor follows the lead of other recently submitted drivers with +ioctls allowing us to support modern Vulkan features, like sparse memory +binding: + +- Pretty standard GEM management ioctls (BO_CREATE and BO_MMAP_OFFSET), + with the 'exclusive-VM' bit to speed-up BO reservation on job +submission +- VM management ioctls (VM_CREATE, VM_DESTROY and VM_BIND). The VM_BIND + ioctl is loosely based on the Xe model, and can handle both + asynchronous and synchronous requests +- GPU execution context creation/destruction, tiler heap context +creation + and job submission. Those ioctls reflect how the hardware/scheduler + works and are thus driver specific. + +We also have a way to expose IO regions, such that the usermode driver +can directly access specific/well-isolate registers, like the +LATEST_FLUSH register used to implement cache-flush reduction. + +This uAPI intentionally keeps usermode queues out of the scope, which +explains why doorbell registers and command stream ring-buffers are not +directly exposed to userspace. + +v6: +- Add Maxime's and Heiko's acks + +v5: +- Fix typo +- Add Liviu's R-b + +v4: +- Add a VM_GET_STATE ioctl +- Fix doc +- Expose the CORE_FEATURES register so we can deal with variants in the + UMD +- Add Steve's R-b + +v3: +- Add the concept of sync-only VM operation +- Fix support for 32-bit userspace +- Rework drm_panthor_vm_create to pass the user VA size instead of + the kernel VA size (suggested by Robin Murphy) +- Typo fixes +- Explicitly cast enums with top bit set to avoid compiler warnings in + -pedantic mode. +- Drop property core_group_count as it can be easily calculated by the + number of bits set in l2_present. + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-2-boris.brezillon@collabora.com +--- + Documentation/gpu/driver-uapi.rst | 5 + + include/uapi/drm/panthor_drm.h | 945 ++++++++++++++++++++++++++++++ + 2 files changed, 950 insertions(+) + create mode 100644 include/uapi/drm/panthor_drm.h + +diff --git a/Documentation/gpu/driver-uapi.rst b/Documentation/gpu/driver-uapi.rst +index 4411e6919a3dfe..7b6d9006306cf9 100644 +--- a/Documentation/gpu/driver-uapi.rst ++++ b/Documentation/gpu/driver-uapi.rst +@@ -6,3 +6,8 @@ drm/i915 uAPI + ============= + + .. kernel-doc:: include/uapi/drm/i915_drm.h ++ ++drm/panthor uAPI ++================ ++ ++.. kernel-doc:: i +\ No newline at end of file +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +new file mode 100644 +index 00000000000000..373df80f41edf6 +--- /dev/null ++++ b/include/uapi/drm/panthor_drm.h +@@ -0,0 +1,945 @@ ++/* SPDX-License-Identifier: MIT */ ++/* Copyright (C) 2023 Collabora ltd. */ ++#ifndef _PANTHOR_DRM_H_ ++#define _PANTHOR_DRM_H_ ++ ++#include "drm.h" ++ ++#if defined(__cplusplus) ++extern "C" { ++#endif ++ ++/** ++ * DOC: Introduction ++ * ++ * This documentation describes the Panthor IOCTLs. ++ * ++ * Just a few generic rules about the data passed to the Panthor IOCTLs: ++ * ++ * - Structures must be aligned on 64-bit/8-byte. If the object is not ++ * naturally aligned, a padding field must be added. ++ * - Fields must be explicitly aligned to their natural type alignment with ++ * pad[0..N] fields. ++ * - All padding fields will be checked by the driver to make sure they are ++ * zeroed. ++ * - Flags can be added, but not removed/replaced. ++ * - New fields can be added to the main structures (the structures ++ * directly passed to the ioctl). Those fields can be added at the end of ++ * the structure, or replace existing padding fields. Any new field being ++ * added must preserve the behavior that existed before those fields were ++ * added when a value of zero is passed. ++ * - New fields can be added to indirect objects (objects pointed by the ++ * main structure), iff those objects are passed a size to reflect the ++ * size known by the userspace driver (see drm_panthor_obj_array::stride ++ * or drm_panthor_dev_query::size). ++ * - If the kernel driver is too old to know some fields, those will be ++ * ignored if zero, and otherwise rejected (and so will be zero on output). ++ * - If userspace is too old to know some fields, those will be zeroed ++ * (input) before the structure is parsed by the kernel driver. ++ * - Each new flag/field addition must come with a driver version update so ++ * the userspace driver doesn't have to trial and error to know which ++ * flags are supported. ++ * - Structures should not contain unions, as this would defeat the ++ * extensibility of such structures. ++ * - IOCTLs can't be removed or replaced. New IOCTL IDs should be placed ++ * at the end of the drm_panthor_ioctl_id enum. ++ */ ++ ++/** ++ * DOC: MMIO regions exposed to userspace. ++ * ++ * .. c:macro:: DRM_PANTHOR_USER_MMIO_OFFSET ++ * ++ * File offset for all MMIO regions being exposed to userspace. Don't use ++ * this value directly, use DRM_PANTHOR_USER__OFFSET values instead. ++ * pgoffset passed to mmap2() is an unsigned long, which forces us to use a ++ * different offset on 32-bit and 64-bit systems. ++ * ++ * .. c:macro:: DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET ++ * ++ * File offset for the LATEST_FLUSH_ID register. The Userspace driver controls ++ * GPU cache flushing through CS instructions, but the flush reduction ++ * mechanism requires a flush_id. This flush_id could be queried with an ++ * ioctl, but Arm provides a well-isolated register page containing only this ++ * read-only register, so let's expose this page through a static mmap offset ++ * and allow direct mapping of this MMIO region so we can avoid the ++ * user <-> kernel round-trip. ++ */ ++#define DRM_PANTHOR_USER_MMIO_OFFSET_32BIT (1ull << 43) ++#define DRM_PANTHOR_USER_MMIO_OFFSET_64BIT (1ull << 56) ++#define DRM_PANTHOR_USER_MMIO_OFFSET (sizeof(unsigned long) < 8 ? \ ++ DRM_PANTHOR_USER_MMIO_OFFSET_32BIT : \ ++ DRM_PANTHOR_USER_MMIO_OFFSET_64BIT) ++#define DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET (DRM_PANTHOR_USER_MMIO_OFFSET | 0) ++ ++/** ++ * DOC: IOCTL IDs ++ * ++ * enum drm_panthor_ioctl_id - IOCTL IDs ++ * ++ * Place new ioctls at the end, don't re-order, don't replace or remove entries. ++ * ++ * These IDs are not meant to be used directly. Use the DRM_IOCTL_PANTHOR_xxx ++ * definitions instead. ++ */ ++enum drm_panthor_ioctl_id { ++ /** @DRM_PANTHOR_DEV_QUERY: Query device information. */ ++ DRM_PANTHOR_DEV_QUERY = 0, ++ ++ /** @DRM_PANTHOR_VM_CREATE: Create a VM. */ ++ DRM_PANTHOR_VM_CREATE, ++ ++ /** @DRM_PANTHOR_VM_DESTROY: Destroy a VM. */ ++ DRM_PANTHOR_VM_DESTROY, ++ ++ /** @DRM_PANTHOR_VM_BIND: Bind/unbind memory to a VM. */ ++ DRM_PANTHOR_VM_BIND, ++ ++ /** @DRM_PANTHOR_VM_GET_STATE: Get VM state. */ ++ DRM_PANTHOR_VM_GET_STATE, ++ ++ /** @DRM_PANTHOR_BO_CREATE: Create a buffer object. */ ++ DRM_PANTHOR_BO_CREATE, ++ ++ /** ++ * @DRM_PANTHOR_BO_MMAP_OFFSET: Get the file offset to pass to ++ * mmap to map a GEM object. ++ */ ++ DRM_PANTHOR_BO_MMAP_OFFSET, ++ ++ /** @DRM_PANTHOR_GROUP_CREATE: Create a scheduling group. */ ++ DRM_PANTHOR_GROUP_CREATE, ++ ++ /** @DRM_PANTHOR_GROUP_DESTROY: Destroy a scheduling group. */ ++ DRM_PANTHOR_GROUP_DESTROY, ++ ++ /** ++ * @DRM_PANTHOR_GROUP_SUBMIT: Submit jobs to queues belonging ++ * to a specific scheduling group. ++ */ ++ DRM_PANTHOR_GROUP_SUBMIT, ++ ++ /** @DRM_PANTHOR_GROUP_GET_STATE: Get the state of a scheduling group. */ ++ DRM_PANTHOR_GROUP_GET_STATE, ++ ++ /** @DRM_PANTHOR_TILER_HEAP_CREATE: Create a tiler heap. */ ++ DRM_PANTHOR_TILER_HEAP_CREATE, ++ ++ /** @DRM_PANTHOR_TILER_HEAP_DESTROY: Destroy a tiler heap. */ ++ DRM_PANTHOR_TILER_HEAP_DESTROY, ++}; ++ ++/** ++ * DRM_IOCTL_PANTHOR() - Build a Panthor IOCTL number ++ * @__access: Access type. Must be R, W or RW. ++ * @__id: One of the DRM_PANTHOR_xxx id. ++ * @__type: Suffix of the type being passed to the IOCTL. ++ * ++ * Don't use this macro directly, use the DRM_IOCTL_PANTHOR_xxx ++ * values instead. ++ * ++ * Return: An IOCTL number to be passed to ioctl() from userspace. ++ */ ++#define DRM_IOCTL_PANTHOR(__access, __id, __type) \ ++ DRM_IO ## __access(DRM_COMMAND_BASE + DRM_PANTHOR_ ## __id, \ ++ struct drm_panthor_ ## __type) ++ ++#define DRM_IOCTL_PANTHOR_DEV_QUERY \ ++ DRM_IOCTL_PANTHOR(WR, DEV_QUERY, dev_query) ++#define DRM_IOCTL_PANTHOR_VM_CREATE \ ++ DRM_IOCTL_PANTHOR(WR, VM_CREATE, vm_create) ++#define DRM_IOCTL_PANTHOR_VM_DESTROY \ ++ DRM_IOCTL_PANTHOR(WR, VM_DESTROY, vm_destroy) ++#define DRM_IOCTL_PANTHOR_VM_BIND \ ++ DRM_IOCTL_PANTHOR(WR, VM_BIND, vm_bind) ++#define DRM_IOCTL_PANTHOR_VM_GET_STATE \ ++ DRM_IOCTL_PANTHOR(WR, VM_GET_STATE, vm_get_state) ++#define DRM_IOCTL_PANTHOR_BO_CREATE \ ++ DRM_IOCTL_PANTHOR(WR, BO_CREATE, bo_create) ++#define DRM_IOCTL_PANTHOR_BO_MMAP_OFFSET \ ++ DRM_IOCTL_PANTHOR(WR, BO_MMAP_OFFSET, bo_mmap_offset) ++#define DRM_IOCTL_PANTHOR_GROUP_CREATE \ ++ DRM_IOCTL_PANTHOR(WR, GROUP_CREATE, group_create) ++#define DRM_IOCTL_PANTHOR_GROUP_DESTROY \ ++ DRM_IOCTL_PANTHOR(WR, GROUP_DESTROY, group_destroy) ++#define DRM_IOCTL_PANTHOR_GROUP_SUBMIT \ ++ DRM_IOCTL_PANTHOR(WR, GROUP_SUBMIT, group_submit) ++#define DRM_IOCTL_PANTHOR_GROUP_GET_STATE \ ++ DRM_IOCTL_PANTHOR(WR, GROUP_GET_STATE, group_get_state) ++#define DRM_IOCTL_PANTHOR_TILER_HEAP_CREATE \ ++ DRM_IOCTL_PANTHOR(WR, TILER_HEAP_CREATE, tiler_heap_create) ++#define DRM_IOCTL_PANTHOR_TILER_HEAP_DESTROY \ ++ DRM_IOCTL_PANTHOR(WR, TILER_HEAP_DESTROY, tiler_heap_destroy) ++ ++/** ++ * DOC: IOCTL arguments ++ */ ++ ++/** ++ * struct drm_panthor_obj_array - Object array. ++ * ++ * This object is used to pass an array of objects whose size is subject to changes in ++ * future versions of the driver. In order to support this mutability, we pass a stride ++ * describing the size of the object as known by userspace. ++ * ++ * You shouldn't fill drm_panthor_obj_array fields directly. You should instead use ++ * the DRM_PANTHOR_OBJ_ARRAY() macro that takes care of initializing the stride to ++ * the object size. ++ */ ++struct drm_panthor_obj_array { ++ /** @stride: Stride of object struct. Used for versioning. */ ++ __u32 stride; ++ ++ /** @count: Number of objects in the array. */ ++ __u32 count; ++ ++ /** @array: User pointer to an array of objects. */ ++ __u64 array; ++}; ++ ++/** ++ * DRM_PANTHOR_OBJ_ARRAY() - Initialize a drm_panthor_obj_array field. ++ * @cnt: Number of elements in the array. ++ * @ptr: Pointer to the array to pass to the kernel. ++ * ++ * Macro initializing a drm_panthor_obj_array based on the object size as known ++ * by userspace. ++ */ ++#define DRM_PANTHOR_OBJ_ARRAY(cnt, ptr) \ ++ { .stride = sizeof((ptr)[0]), .count = (cnt), .array = (__u64)(uintptr_t)(ptr) } ++ ++/** ++ * enum drm_panthor_sync_op_flags - Synchronization operation flags. ++ */ ++enum drm_panthor_sync_op_flags { ++ /** @DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK: Synchronization handle type mask. */ ++ DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK = 0xff, ++ ++ /** @DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ: Synchronization object type. */ ++ DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ = 0, ++ ++ /** ++ * @DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ: Timeline synchronization ++ * object type. ++ */ ++ DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ = 1, ++ ++ /** @DRM_PANTHOR_SYNC_OP_WAIT: Wait operation. */ ++ DRM_PANTHOR_SYNC_OP_WAIT = 0 << 31, ++ ++ /** @DRM_PANTHOR_SYNC_OP_SIGNAL: Signal operation. */ ++ DRM_PANTHOR_SYNC_OP_SIGNAL = (int)(1u << 31), ++}; ++ ++/** ++ * struct drm_panthor_sync_op - Synchronization operation. ++ */ ++struct drm_panthor_sync_op { ++ /** @flags: Synchronization operation flags. Combination of DRM_PANTHOR_SYNC_OP values. */ ++ __u32 flags; ++ ++ /** @handle: Sync handle. */ ++ __u32 handle; ++ ++ /** ++ * @timeline_value: MBZ if ++ * (flags & DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK) != ++ * DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ. ++ */ ++ __u64 timeline_value; ++}; ++ ++/** ++ * enum drm_panthor_dev_query_type - Query type ++ * ++ * Place new types at the end, don't re-order, don't remove or replace. ++ */ ++enum drm_panthor_dev_query_type { ++ /** @DRM_PANTHOR_DEV_QUERY_GPU_INFO: Query GPU information. */ ++ DRM_PANTHOR_DEV_QUERY_GPU_INFO = 0, ++ ++ /** @DRM_PANTHOR_DEV_QUERY_CSIF_INFO: Query command-stream interface information. */ ++ DRM_PANTHOR_DEV_QUERY_CSIF_INFO, ++}; ++ ++/** ++ * struct drm_panthor_gpu_info - GPU information ++ * ++ * Structure grouping all queryable information relating to the GPU. ++ */ ++struct drm_panthor_gpu_info { ++ /** @gpu_id : GPU ID. */ ++ __u32 gpu_id; ++#define DRM_PANTHOR_ARCH_MAJOR(x) ((x) >> 28) ++#define DRM_PANTHOR_ARCH_MINOR(x) (((x) >> 24) & 0xf) ++#define DRM_PANTHOR_ARCH_REV(x) (((x) >> 20) & 0xf) ++#define DRM_PANTHOR_PRODUCT_MAJOR(x) (((x) >> 16) & 0xf) ++#define DRM_PANTHOR_VERSION_MAJOR(x) (((x) >> 12) & 0xf) ++#define DRM_PANTHOR_VERSION_MINOR(x) (((x) >> 4) & 0xff) ++#define DRM_PANTHOR_VERSION_STATUS(x) ((x) & 0xf) ++ ++ /** @gpu_rev: GPU revision. */ ++ __u32 gpu_rev; ++ ++ /** @csf_id: Command stream frontend ID. */ ++ __u32 csf_id; ++#define DRM_PANTHOR_CSHW_MAJOR(x) (((x) >> 26) & 0x3f) ++#define DRM_PANTHOR_CSHW_MINOR(x) (((x) >> 20) & 0x3f) ++#define DRM_PANTHOR_CSHW_REV(x) (((x) >> 16) & 0xf) ++#define DRM_PANTHOR_MCU_MAJOR(x) (((x) >> 10) & 0x3f) ++#define DRM_PANTHOR_MCU_MINOR(x) (((x) >> 4) & 0x3f) ++#define DRM_PANTHOR_MCU_REV(x) ((x) & 0xf) ++ ++ /** @l2_features: L2-cache features. */ ++ __u32 l2_features; ++ ++ /** @tiler_features: Tiler features. */ ++ __u32 tiler_features; ++ ++ /** @mem_features: Memory features. */ ++ __u32 mem_features; ++ ++ /** @mmu_features: MMU features. */ ++ __u32 mmu_features; ++#define DRM_PANTHOR_MMU_VA_BITS(x) ((x) & 0xff) ++ ++ /** @thread_features: Thread features. */ ++ __u32 thread_features; ++ ++ /** @max_threads: Maximum number of threads. */ ++ __u32 max_threads; ++ ++ /** @thread_max_workgroup_size: Maximum workgroup size. */ ++ __u32 thread_max_workgroup_size; ++ ++ /** ++ * @thread_max_barrier_size: Maximum number of threads that can wait ++ * simultaneously on a barrier. ++ */ ++ __u32 thread_max_barrier_size; ++ ++ /** @coherency_features: Coherency features. */ ++ __u32 coherency_features; ++ ++ /** @texture_features: Texture features. */ ++ __u32 texture_features[4]; ++ ++ /** @as_present: Bitmask encoding the number of address-space exposed by the MMU. */ ++ __u32 as_present; ++ ++ /** @shader_present: Bitmask encoding the shader cores exposed by the GPU. */ ++ __u64 shader_present; ++ ++ /** @l2_present: Bitmask encoding the L2 caches exposed by the GPU. */ ++ __u64 l2_present; ++ ++ /** @tiler_present: Bitmask encoding the tiler units exposed by the GPU. */ ++ __u64 tiler_present; ++ ++ /* @core_features: Used to discriminate core variants when they exist. */ ++ __u32 core_features; ++ ++ /* @pad: MBZ. */ ++ __u32 pad; ++}; ++ ++/** ++ * struct drm_panthor_csif_info - Command stream interface information ++ * ++ * Structure grouping all queryable information relating to the command stream interface. ++ */ ++struct drm_panthor_csif_info { ++ /** @csg_slot_count: Number of command stream group slots exposed by the firmware. */ ++ __u32 csg_slot_count; ++ ++ /** @cs_slot_count: Number of command stream slots per group. */ ++ __u32 cs_slot_count; ++ ++ /** @cs_reg_count: Number of command stream registers. */ ++ __u32 cs_reg_count; ++ ++ /** @scoreboard_slot_count: Number of scoreboard slots. */ ++ __u32 scoreboard_slot_count; ++ ++ /** ++ * @unpreserved_cs_reg_count: Number of command stream registers reserved by ++ * the kernel driver to call a userspace command stream. ++ * ++ * All registers can be used by a userspace command stream, but the ++ * [cs_slot_count - unpreserved_cs_reg_count .. cs_slot_count] registers are ++ * used by the kernel when DRM_PANTHOR_IOCTL_GROUP_SUBMIT is called. ++ */ ++ __u32 unpreserved_cs_reg_count; ++ ++ /** ++ * @pad: Padding field, set to zero. ++ */ ++ __u32 pad; ++}; ++ ++/** ++ * struct drm_panthor_dev_query - Arguments passed to DRM_PANTHOR_IOCTL_DEV_QUERY ++ */ ++struct drm_panthor_dev_query { ++ /** @type: the query type (see drm_panthor_dev_query_type). */ ++ __u32 type; ++ ++ /** ++ * @size: size of the type being queried. ++ * ++ * If pointer is NULL, size is updated by the driver to provide the ++ * output structure size. If pointer is not NULL, the driver will ++ * only copy min(size, actual_structure_size) bytes to the pointer, ++ * and update the size accordingly. This allows us to extend query ++ * types without breaking userspace. ++ */ ++ __u32 size; ++ ++ /** ++ * @pointer: user pointer to a query type struct. ++ * ++ * Pointer can be NULL, in which case, nothing is copied, but the ++ * actual structure size is returned. If not NULL, it must point to ++ * a location that's large enough to hold size bytes. ++ */ ++ __u64 pointer; ++}; ++ ++/** ++ * struct drm_panthor_vm_create - Arguments passed to DRM_PANTHOR_IOCTL_VM_CREATE ++ */ ++struct drm_panthor_vm_create { ++ /** @flags: VM flags, MBZ. */ ++ __u32 flags; ++ ++ /** @id: Returned VM ID. */ ++ __u32 id; ++ ++ /** ++ * @user_va_range: Size of the VA space reserved for user objects. ++ * ++ * The kernel will pick the remaining space to map kernel-only objects to the ++ * VM (heap chunks, heap context, ring buffers, kernel synchronization objects, ++ * ...). If the space left for kernel objects is too small, kernel object ++ * allocation will fail further down the road. One can use ++ * drm_panthor_gpu_info::mmu_features to extract the total virtual address ++ * range, and chose a user_va_range that leaves some space to the kernel. ++ * ++ * If user_va_range is zero, the kernel will pick a sensible value based on ++ * TASK_SIZE and the virtual range supported by the GPU MMU (the kernel/user ++ * split should leave enough VA space for userspace processes to support SVM, ++ * while still allowing the kernel to map some amount of kernel objects in ++ * the kernel VA range). The value chosen by the driver will be returned in ++ * @user_va_range. ++ * ++ * User VA space always starts at 0x0, kernel VA space is always placed after ++ * the user VA range. ++ */ ++ __u64 user_va_range; ++}; ++ ++/** ++ * struct drm_panthor_vm_destroy - Arguments passed to DRM_PANTHOR_IOCTL_VM_DESTROY ++ */ ++struct drm_panthor_vm_destroy { ++ /** @id: ID of the VM to destroy. */ ++ __u32 id; ++ ++ /** @pad: MBZ. */ ++ __u32 pad; ++}; ++ ++/** ++ * enum drm_panthor_vm_bind_op_flags - VM bind operation flags ++ */ ++enum drm_panthor_vm_bind_op_flags { ++ /** ++ * @DRM_PANTHOR_VM_BIND_OP_MAP_READONLY: Map the memory read-only. ++ * ++ * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. ++ */ ++ DRM_PANTHOR_VM_BIND_OP_MAP_READONLY = 1 << 0, ++ ++ /** ++ * @DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC: Map the memory not-executable. ++ * ++ * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. ++ */ ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC = 1 << 1, ++ ++ /** ++ * @DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED: Map the memory uncached. ++ * ++ * Only valid with DRM_PANTHOR_VM_BIND_OP_TYPE_MAP. ++ */ ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED = 1 << 2, ++ ++ /** ++ * @DRM_PANTHOR_VM_BIND_OP_TYPE_MASK: Mask used to determine the type of operation. ++ */ ++ DRM_PANTHOR_VM_BIND_OP_TYPE_MASK = (int)(0xfu << 28), ++ ++ /** @DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: Map operation. */ ++ DRM_PANTHOR_VM_BIND_OP_TYPE_MAP = 0 << 28, ++ ++ /** @DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP: Unmap operation. */ ++ DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP = 1 << 28, ++ ++ /** ++ * @DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY: No VM operation. ++ * ++ * Just serves as a synchronization point on a VM queue. ++ * ++ * Only valid if %DRM_PANTHOR_VM_BIND_ASYNC is set in drm_panthor_vm_bind::flags, ++ * and drm_panthor_vm_bind_op::syncs contains at least one element. ++ */ ++ DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY = 2 << 28, ++}; ++ ++/** ++ * struct drm_panthor_vm_bind_op - VM bind operation ++ */ ++struct drm_panthor_vm_bind_op { ++ /** @flags: Combination of drm_panthor_vm_bind_op_flags flags. */ ++ __u32 flags; ++ ++ /** ++ * @bo_handle: Handle of the buffer object to map. ++ * MBZ for unmap or sync-only operations. ++ */ ++ __u32 bo_handle; ++ ++ /** ++ * @bo_offset: Buffer object offset. ++ * MBZ for unmap or sync-only operations. ++ */ ++ __u64 bo_offset; ++ ++ /** ++ * @va: Virtual address to map/unmap. ++ * MBZ for sync-only operations. ++ */ ++ __u64 va; ++ ++ /** ++ * @size: Size to map/unmap. ++ * MBZ for sync-only operations. ++ */ ++ __u64 size; ++ ++ /** ++ * @syncs: Array of struct drm_panthor_sync_op synchronization ++ * operations. ++ * ++ * This array must be empty if %DRM_PANTHOR_VM_BIND_ASYNC is not set on ++ * the drm_panthor_vm_bind object containing this VM bind operation. ++ * ++ * This array shall not be empty for sync-only operations. ++ */ ++ struct drm_panthor_obj_array syncs; ++ ++}; ++ ++/** ++ * enum drm_panthor_vm_bind_flags - VM bind flags ++ */ ++enum drm_panthor_vm_bind_flags { ++ /** ++ * @DRM_PANTHOR_VM_BIND_ASYNC: VM bind operations are queued to the VM ++ * queue instead of being executed synchronously. ++ */ ++ DRM_PANTHOR_VM_BIND_ASYNC = 1 << 0, ++}; ++ ++/** ++ * struct drm_panthor_vm_bind - Arguments passed to DRM_IOCTL_PANTHOR_VM_BIND ++ */ ++struct drm_panthor_vm_bind { ++ /** @vm_id: VM targeted by the bind request. */ ++ __u32 vm_id; ++ ++ /** @flags: Combination of drm_panthor_vm_bind_flags flags. */ ++ __u32 flags; ++ ++ /** @ops: Array of struct drm_panthor_vm_bind_op bind operations. */ ++ struct drm_panthor_obj_array ops; ++}; ++ ++/** ++ * enum drm_panthor_vm_state - VM states. ++ */ ++enum drm_panthor_vm_state { ++ /** ++ * @DRM_PANTHOR_VM_STATE_USABLE: VM is usable. ++ * ++ * New VM operations will be accepted on this VM. ++ */ ++ DRM_PANTHOR_VM_STATE_USABLE, ++ ++ /** ++ * @DRM_PANTHOR_VM_STATE_UNUSABLE: VM is unusable. ++ * ++ * Something put the VM in an unusable state (like an asynchronous ++ * VM_BIND request failing for any reason). ++ * ++ * Once the VM is in this state, all new MAP operations will be ++ * rejected, and any GPU job targeting this VM will fail. ++ * UNMAP operations are still accepted. ++ * ++ * The only way to recover from an unusable VM is to create a new ++ * VM, and destroy the old one. ++ */ ++ DRM_PANTHOR_VM_STATE_UNUSABLE, ++}; ++ ++/** ++ * struct drm_panthor_vm_get_state - Get VM state. ++ */ ++struct drm_panthor_vm_get_state { ++ /** @vm_id: VM targeted by the get_state request. */ ++ __u32 vm_id; ++ ++ /** ++ * @state: state returned by the driver. ++ * ++ * Must be one of the enum drm_panthor_vm_state values. ++ */ ++ __u32 state; ++}; ++ ++/** ++ * enum drm_panthor_bo_flags - Buffer object flags, passed at creation time. ++ */ ++enum drm_panthor_bo_flags { ++ /** @DRM_PANTHOR_BO_NO_MMAP: The buffer object will never be CPU-mapped in userspace. */ ++ DRM_PANTHOR_BO_NO_MMAP = (1 << 0), ++}; ++ ++/** ++ * struct drm_panthor_bo_create - Arguments passed to DRM_IOCTL_PANTHOR_BO_CREATE. ++ */ ++struct drm_panthor_bo_create { ++ /** ++ * @size: Requested size for the object ++ * ++ * The (page-aligned) allocated size for the object will be returned. ++ */ ++ __u64 size; ++ ++ /** ++ * @flags: Flags. Must be a combination of drm_panthor_bo_flags flags. ++ */ ++ __u32 flags; ++ ++ /** ++ * @exclusive_vm_id: Exclusive VM this buffer object will be mapped to. ++ * ++ * If not zero, the field must refer to a valid VM ID, and implies that: ++ * - the buffer object will only ever be bound to that VM ++ * - cannot be exported as a PRIME fd ++ */ ++ __u32 exclusive_vm_id; ++ ++ /** ++ * @handle: Returned handle for the object. ++ * ++ * Object handles are nonzero. ++ */ ++ __u32 handle; ++ ++ /** @pad: MBZ. */ ++ __u32 pad; ++}; ++ ++/** ++ * struct drm_panthor_bo_mmap_offset - Arguments passed to DRM_IOCTL_PANTHOR_BO_MMAP_OFFSET. ++ */ ++struct drm_panthor_bo_mmap_offset { ++ /** @handle: Handle of the object we want an mmap offset for. */ ++ __u32 handle; ++ ++ /** @pad: MBZ. */ ++ __u32 pad; ++ ++ /** @offset: The fake offset to use for subsequent mmap calls. */ ++ __u64 offset; ++}; ++ ++/** ++ * struct drm_panthor_queue_create - Queue creation arguments. ++ */ ++struct drm_panthor_queue_create { ++ /** ++ * @priority: Defines the priority of queues inside a group. Goes from 0 to 15, ++ * 15 being the highest priority. ++ */ ++ __u8 priority; ++ ++ /** @pad: Padding fields, MBZ. */ ++ __u8 pad[3]; ++ ++ /** @ringbuf_size: Size of the ring buffer to allocate to this queue. */ ++ __u32 ringbuf_size; ++}; ++ ++/** ++ * enum drm_panthor_group_priority - Scheduling group priority ++ */ ++enum drm_panthor_group_priority { ++ /** @PANTHOR_GROUP_PRIORITY_LOW: Low priority group. */ ++ PANTHOR_GROUP_PRIORITY_LOW = 0, ++ ++ /** @PANTHOR_GROUP_PRIORITY_MEDIUM: Medium priority group. */ ++ PANTHOR_GROUP_PRIORITY_MEDIUM, ++ ++ /** @PANTHOR_GROUP_PRIORITY_HIGH: High priority group. */ ++ PANTHOR_GROUP_PRIORITY_HIGH, ++}; ++ ++/** ++ * struct drm_panthor_group_create - Arguments passed to DRM_IOCTL_PANTHOR_GROUP_CREATE ++ */ ++struct drm_panthor_group_create { ++ /** @queues: Array of drm_panthor_queue_create elements. */ ++ struct drm_panthor_obj_array queues; ++ ++ /** ++ * @max_compute_cores: Maximum number of cores that can be used by compute ++ * jobs across CS queues bound to this group. ++ * ++ * Must be less or equal to the number of bits set in @compute_core_mask. ++ */ ++ __u8 max_compute_cores; ++ ++ /** ++ * @max_fragment_cores: Maximum number of cores that can be used by fragment ++ * jobs across CS queues bound to this group. ++ * ++ * Must be less or equal to the number of bits set in @fragment_core_mask. ++ */ ++ __u8 max_fragment_cores; ++ ++ /** ++ * @max_tiler_cores: Maximum number of tilers that can be used by tiler jobs ++ * across CS queues bound to this group. ++ * ++ * Must be less or equal to the number of bits set in @tiler_core_mask. ++ */ ++ __u8 max_tiler_cores; ++ ++ /** @priority: Group priority (see enum drm_panthor_group_priority). */ ++ __u8 priority; ++ ++ /** @pad: Padding field, MBZ. */ ++ __u32 pad; ++ ++ /** ++ * @compute_core_mask: Mask encoding cores that can be used for compute jobs. ++ * ++ * This field must have at least @max_compute_cores bits set. ++ * ++ * The bits set here should also be set in drm_panthor_gpu_info::shader_present. ++ */ ++ __u64 compute_core_mask; ++ ++ /** ++ * @fragment_core_mask: Mask encoding cores that can be used for fragment jobs. ++ * ++ * This field must have at least @max_fragment_cores bits set. ++ * ++ * The bits set here should also be set in drm_panthor_gpu_info::shader_present. ++ */ ++ __u64 fragment_core_mask; ++ ++ /** ++ * @tiler_core_mask: Mask encoding cores that can be used for tiler jobs. ++ * ++ * This field must have at least @max_tiler_cores bits set. ++ * ++ * The bits set here should also be set in drm_panthor_gpu_info::tiler_present. ++ */ ++ __u64 tiler_core_mask; ++ ++ /** ++ * @vm_id: VM ID to bind this group to. ++ * ++ * All submission to queues bound to this group will use this VM. ++ */ ++ __u32 vm_id; ++ ++ /** ++ * @group_handle: Returned group handle. Passed back when submitting jobs or ++ * destroying a group. ++ */ ++ __u32 group_handle; ++}; ++ ++/** ++ * struct drm_panthor_group_destroy - Arguments passed to DRM_IOCTL_PANTHOR_GROUP_DESTROY ++ */ ++struct drm_panthor_group_destroy { ++ /** @group_handle: Group to destroy */ ++ __u32 group_handle; ++ ++ /** @pad: Padding field, MBZ. */ ++ __u32 pad; ++}; ++ ++/** ++ * struct drm_panthor_queue_submit - Job submission arguments. ++ * ++ * This is describing the userspace command stream to call from the kernel ++ * command stream ring-buffer. Queue submission is always part of a group ++ * submission, taking one or more jobs to submit to the underlying queues. ++ */ ++struct drm_panthor_queue_submit { ++ /** @queue_index: Index of the queue inside a group. */ ++ __u32 queue_index; ++ ++ /** ++ * @stream_size: Size of the command stream to execute. ++ * ++ * Must be 64-bit/8-byte aligned (the size of a CS instruction) ++ * ++ * Can be zero if stream_addr is zero too. ++ */ ++ __u32 stream_size; ++ ++ /** ++ * @stream_addr: GPU address of the command stream to execute. ++ * ++ * Must be aligned on 64-byte. ++ * ++ * Can be zero is stream_size is zero too. ++ */ ++ __u64 stream_addr; ++ ++ /** ++ * @latest_flush: FLUSH_ID read at the time the stream was built. ++ * ++ * This allows cache flush elimination for the automatic ++ * flush+invalidate(all) done at submission time, which is needed to ++ * ensure the GPU doesn't get garbage when reading the indirect command ++ * stream buffers. If you want the cache flush to happen ++ * unconditionally, pass a zero here. ++ */ ++ __u32 latest_flush; ++ ++ /** @pad: MBZ. */ ++ __u32 pad; ++ ++ /** @syncs: Array of struct drm_panthor_sync_op sync operations. */ ++ struct drm_panthor_obj_array syncs; ++}; ++ ++/** ++ * struct drm_panthor_group_submit - Arguments passed to DRM_IOCTL_PANTHOR_GROUP_SUBMIT ++ */ ++struct drm_panthor_group_submit { ++ /** @group_handle: Handle of the group to queue jobs to. */ ++ __u32 group_handle; ++ ++ /** @pad: MBZ. */ ++ __u32 pad; ++ ++ /** @queue_submits: Array of drm_panthor_queue_submit objects. */ ++ struct drm_panthor_obj_array queue_submits; ++}; ++ ++/** ++ * enum drm_panthor_group_state_flags - Group state flags ++ */ ++enum drm_panthor_group_state_flags { ++ /** ++ * @DRM_PANTHOR_GROUP_STATE_TIMEDOUT: Group had unfinished jobs. ++ * ++ * When a group ends up with this flag set, no jobs can be submitted to its queues. ++ */ ++ DRM_PANTHOR_GROUP_STATE_TIMEDOUT = 1 << 0, ++ ++ /** ++ * @DRM_PANTHOR_GROUP_STATE_FATAL_FAULT: Group had fatal faults. ++ * ++ * When a group ends up with this flag set, no jobs can be submitted to its queues. ++ */ ++ DRM_PANTHOR_GROUP_STATE_FATAL_FAULT = 1 << 1, ++}; ++ ++/** ++ * struct drm_panthor_group_get_state - Arguments passed to DRM_IOCTL_PANTHOR_GROUP_GET_STATE ++ * ++ * Used to query the state of a group and decide whether a new group should be created to ++ * replace it. ++ */ ++struct drm_panthor_group_get_state { ++ /** @group_handle: Handle of the group to query state on */ ++ __u32 group_handle; ++ ++ /** ++ * @state: Combination of DRM_PANTHOR_GROUP_STATE_* flags encoding the ++ * group state. ++ */ ++ __u32 state; ++ ++ /** @fatal_queues: Bitmask of queues that faced fatal faults. */ ++ __u32 fatal_queues; ++ ++ /** @pad: MBZ */ ++ __u32 pad; ++}; ++ ++/** ++ * struct drm_panthor_tiler_heap_create - Arguments passed to DRM_IOCTL_PANTHOR_TILER_HEAP_CREATE ++ */ ++struct drm_panthor_tiler_heap_create { ++ /** @vm_id: VM ID the tiler heap should be mapped to */ ++ __u32 vm_id; ++ ++ /** @initial_chunk_count: Initial number of chunks to allocate. */ ++ __u32 initial_chunk_count; ++ ++ /** @chunk_size: Chunk size. Must be a power of two at least 256KB large. */ ++ __u32 chunk_size; ++ ++ /** @max_chunks: Maximum number of chunks that can be allocated. */ ++ __u32 max_chunks; ++ ++ /** ++ * @target_in_flight: Maximum number of in-flight render passes. ++ * ++ * If the heap has more than tiler jobs in-flight, the FW will wait for render ++ * passes to finish before queuing new tiler jobs. ++ */ ++ __u32 target_in_flight; ++ ++ /** @handle: Returned heap handle. Passed back to DESTROY_TILER_HEAP. */ ++ __u32 handle; ++ ++ /** @tiler_heap_ctx_gpu_va: Returned heap GPU virtual address returned */ ++ __u64 tiler_heap_ctx_gpu_va; ++ ++ /** ++ * @first_heap_chunk_gpu_va: First heap chunk. ++ * ++ * The tiler heap is formed of heap chunks forming a single-link list. This ++ * is the first element in the list. ++ */ ++ __u64 first_heap_chunk_gpu_va; ++}; ++ ++/** ++ * struct drm_panthor_tiler_heap_destroy - Arguments passed to DRM_IOCTL_PANTHOR_TILER_HEAP_DESTROY ++ */ ++struct drm_panthor_tiler_heap_destroy { ++ /** @handle: Handle of the tiler heap to destroy */ ++ __u32 handle; ++ ++ /** @pad: Padding field, MBZ. */ ++ __u32 pad; ++}; ++ ++#if defined(__cplusplus) ++} ++#endif ++ ++#endif /* _PANTHOR_DRM_H_ */ + +From 3f20d7ecce484f3d75a8a355a53049f2e4070d2f Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:16 +0100 +Subject: [PATCH 081/150] drm/panthor: Add GPU register definitions + +Those are the registers directly accessible through the MMIO range. + +FW registers are exposed in panthor_fw.h. + +v6: +- Add Maxime's and Heiko's acks + +v4: +- Add the CORE_FEATURES register (needed for GPU variants) +- Add Steve's R-b + +v3: +- Add macros to extract GPU ID info +- Formatting changes +- Remove AS_TRANSCFG_ADRMODE_LEGACY - it doesn't exist post-CSF +- Remove CSF_GPU_LATEST_FLUSH_ID_DEFAULT +- Add GPU_L2_FEATURES_LINE_SIZE for extracting the GPU cache line size + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_regs.h | 239 +++++++++++++++++++++++++ + 1 file changed, 239 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_regs.h + +diff --git a/drivers/gpu/drm/panthor/panthor_regs.h b/drivers/gpu/drm/panthor/panthor_regs.h +new file mode 100644 +index 00000000000000..b7b3b3add16627 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_regs.h +@@ -0,0 +1,239 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++/* ++ * Register definitions based on mali_kbase_gpu_regmap.h and ++ * mali_kbase_gpu_regmap_csf.h ++ * (C) COPYRIGHT 2010-2022 ARM Limited. All rights reserved. ++ */ ++#ifndef __PANTHOR_REGS_H__ ++#define __PANTHOR_REGS_H__ ++ ++#define GPU_ID 0x0 ++#define GPU_ARCH_MAJOR(x) ((x) >> 28) ++#define GPU_ARCH_MINOR(x) (((x) & GENMASK(27, 24)) >> 24) ++#define GPU_ARCH_REV(x) (((x) & GENMASK(23, 20)) >> 20) ++#define GPU_PROD_MAJOR(x) (((x) & GENMASK(19, 16)) >> 16) ++#define GPU_VER_MAJOR(x) (((x) & GENMASK(15, 12)) >> 12) ++#define GPU_VER_MINOR(x) (((x) & GENMASK(11, 4)) >> 4) ++#define GPU_VER_STATUS(x) ((x) & GENMASK(3, 0)) ++ ++#define GPU_L2_FEATURES 0x4 ++#define GPU_L2_FEATURES_LINE_SIZE(x) (1 << ((x) & GENMASK(7, 0))) ++ ++#define GPU_CORE_FEATURES 0x8 ++ ++#define GPU_TILER_FEATURES 0xC ++#define GPU_MEM_FEATURES 0x10 ++#define GROUPS_L2_COHERENT BIT(0) ++ ++#define GPU_MMU_FEATURES 0x14 ++#define GPU_MMU_FEATURES_VA_BITS(x) ((x) & GENMASK(7, 0)) ++#define GPU_MMU_FEATURES_PA_BITS(x) (((x) >> 8) & GENMASK(7, 0)) ++#define GPU_AS_PRESENT 0x18 ++#define GPU_CSF_ID 0x1C ++ ++#define GPU_INT_RAWSTAT 0x20 ++#define GPU_INT_CLEAR 0x24 ++#define GPU_INT_MASK 0x28 ++#define GPU_INT_STAT 0x2c ++#define GPU_IRQ_FAULT BIT(0) ++#define GPU_IRQ_PROTM_FAULT BIT(1) ++#define GPU_IRQ_RESET_COMPLETED BIT(8) ++#define GPU_IRQ_POWER_CHANGED BIT(9) ++#define GPU_IRQ_POWER_CHANGED_ALL BIT(10) ++#define GPU_IRQ_CLEAN_CACHES_COMPLETED BIT(17) ++#define GPU_IRQ_DOORBELL_MIRROR BIT(18) ++#define GPU_IRQ_MCU_STATUS_CHANGED BIT(19) ++#define GPU_CMD 0x30 ++#define GPU_CMD_DEF(type, payload) ((type) | ((payload) << 8)) ++#define GPU_SOFT_RESET GPU_CMD_DEF(1, 1) ++#define GPU_HARD_RESET GPU_CMD_DEF(1, 2) ++#define CACHE_CLEAN BIT(0) ++#define CACHE_INV BIT(1) ++#define GPU_FLUSH_CACHES(l2, lsc, oth) \ ++ GPU_CMD_DEF(4, ((l2) << 0) | ((lsc) << 4) | ((oth) << 8)) ++ ++#define GPU_STATUS 0x34 ++#define GPU_STATUS_ACTIVE BIT(0) ++#define GPU_STATUS_PWR_ACTIVE BIT(1) ++#define GPU_STATUS_PAGE_FAULT BIT(4) ++#define GPU_STATUS_PROTM_ACTIVE BIT(7) ++#define GPU_STATUS_DBG_ENABLED BIT(8) ++ ++#define GPU_FAULT_STATUS 0x3C ++#define GPU_FAULT_ADDR_LO 0x40 ++#define GPU_FAULT_ADDR_HI 0x44 ++ ++#define GPU_PWR_KEY 0x50 ++#define GPU_PWR_KEY_UNLOCK 0x2968A819 ++#define GPU_PWR_OVERRIDE0 0x54 ++#define GPU_PWR_OVERRIDE1 0x58 ++ ++#define GPU_TIMESTAMP_OFFSET_LO 0x88 ++#define GPU_TIMESTAMP_OFFSET_HI 0x8C ++#define GPU_CYCLE_COUNT_LO 0x90 ++#define GPU_CYCLE_COUNT_HI 0x94 ++#define GPU_TIMESTAMP_LO 0x98 ++#define GPU_TIMESTAMP_HI 0x9C ++ ++#define GPU_THREAD_MAX_THREADS 0xA0 ++#define GPU_THREAD_MAX_WORKGROUP_SIZE 0xA4 ++#define GPU_THREAD_MAX_BARRIER_SIZE 0xA8 ++#define GPU_THREAD_FEATURES 0xAC ++ ++#define GPU_TEXTURE_FEATURES(n) (0xB0 + ((n) * 4)) ++ ++#define GPU_SHADER_PRESENT_LO 0x100 ++#define GPU_SHADER_PRESENT_HI 0x104 ++#define GPU_TILER_PRESENT_LO 0x110 ++#define GPU_TILER_PRESENT_HI 0x114 ++#define GPU_L2_PRESENT_LO 0x120 ++#define GPU_L2_PRESENT_HI 0x124 ++ ++#define SHADER_READY_LO 0x140 ++#define SHADER_READY_HI 0x144 ++#define TILER_READY_LO 0x150 ++#define TILER_READY_HI 0x154 ++#define L2_READY_LO 0x160 ++#define L2_READY_HI 0x164 ++ ++#define SHADER_PWRON_LO 0x180 ++#define SHADER_PWRON_HI 0x184 ++#define TILER_PWRON_LO 0x190 ++#define TILER_PWRON_HI 0x194 ++#define L2_PWRON_LO 0x1A0 ++#define L2_PWRON_HI 0x1A4 ++ ++#define SHADER_PWROFF_LO 0x1C0 ++#define SHADER_PWROFF_HI 0x1C4 ++#define TILER_PWROFF_LO 0x1D0 ++#define TILER_PWROFF_HI 0x1D4 ++#define L2_PWROFF_LO 0x1E0 ++#define L2_PWROFF_HI 0x1E4 ++ ++#define SHADER_PWRTRANS_LO 0x200 ++#define SHADER_PWRTRANS_HI 0x204 ++#define TILER_PWRTRANS_LO 0x210 ++#define TILER_PWRTRANS_HI 0x214 ++#define L2_PWRTRANS_LO 0x220 ++#define L2_PWRTRANS_HI 0x224 ++ ++#define SHADER_PWRACTIVE_LO 0x240 ++#define SHADER_PWRACTIVE_HI 0x244 ++#define TILER_PWRACTIVE_LO 0x250 ++#define TILER_PWRACTIVE_HI 0x254 ++#define L2_PWRACTIVE_LO 0x260 ++#define L2_PWRACTIVE_HI 0x264 ++ ++#define GPU_REVID 0x280 ++ ++#define GPU_COHERENCY_FEATURES 0x300 ++#define GPU_COHERENCY_PROT_BIT(name) BIT(GPU_COHERENCY_ ## name) ++ ++#define GPU_COHERENCY_PROTOCOL 0x304 ++#define GPU_COHERENCY_ACE 0 ++#define GPU_COHERENCY_ACE_LITE 1 ++#define GPU_COHERENCY_NONE 31 ++ ++#define MCU_CONTROL 0x700 ++#define MCU_CONTROL_ENABLE 1 ++#define MCU_CONTROL_AUTO 2 ++#define MCU_CONTROL_DISABLE 0 ++ ++#define MCU_STATUS 0x704 ++#define MCU_STATUS_DISABLED 0 ++#define MCU_STATUS_ENABLED 1 ++#define MCU_STATUS_HALT 2 ++#define MCU_STATUS_FATAL 3 ++ ++/* Job Control regs */ ++#define JOB_INT_RAWSTAT 0x1000 ++#define JOB_INT_CLEAR 0x1004 ++#define JOB_INT_MASK 0x1008 ++#define JOB_INT_STAT 0x100c ++#define JOB_INT_GLOBAL_IF BIT(31) ++#define JOB_INT_CSG_IF(x) BIT(x) ++ ++/* MMU regs */ ++#define MMU_INT_RAWSTAT 0x2000 ++#define MMU_INT_CLEAR 0x2004 ++#define MMU_INT_MASK 0x2008 ++#define MMU_INT_STAT 0x200c ++ ++/* AS_COMMAND register commands */ ++ ++#define MMU_BASE 0x2400 ++#define MMU_AS_SHIFT 6 ++#define MMU_AS(as) (MMU_BASE + ((as) << MMU_AS_SHIFT)) ++ ++#define AS_TRANSTAB_LO(as) (MMU_AS(as) + 0x0) ++#define AS_TRANSTAB_HI(as) (MMU_AS(as) + 0x4) ++#define AS_MEMATTR_LO(as) (MMU_AS(as) + 0x8) ++#define AS_MEMATTR_HI(as) (MMU_AS(as) + 0xC) ++#define AS_MEMATTR_AARCH64_INNER_ALLOC_IMPL (2 << 2) ++#define AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(w, r) ((3 << 2) | \ ++ ((w) ? BIT(0) : 0) | \ ++ ((r) ? BIT(1) : 0)) ++#define AS_MEMATTR_AARCH64_SH_MIDGARD_INNER (0 << 4) ++#define AS_MEMATTR_AARCH64_SH_CPU_INNER (1 << 4) ++#define AS_MEMATTR_AARCH64_SH_CPU_INNER_SHADER_COH (2 << 4) ++#define AS_MEMATTR_AARCH64_SHARED (0 << 6) ++#define AS_MEMATTR_AARCH64_INNER_OUTER_NC (1 << 6) ++#define AS_MEMATTR_AARCH64_INNER_OUTER_WB (2 << 6) ++#define AS_MEMATTR_AARCH64_FAULT (3 << 6) ++#define AS_LOCKADDR_LO(as) (MMU_AS(as) + 0x10) ++#define AS_LOCKADDR_HI(as) (MMU_AS(as) + 0x14) ++#define AS_COMMAND(as) (MMU_AS(as) + 0x18) ++#define AS_COMMAND_NOP 0 ++#define AS_COMMAND_UPDATE 1 ++#define AS_COMMAND_LOCK 2 ++#define AS_COMMAND_UNLOCK 3 ++#define AS_COMMAND_FLUSH_PT 4 ++#define AS_COMMAND_FLUSH_MEM 5 ++#define AS_LOCK_REGION_MIN_SIZE (1ULL << 15) ++#define AS_FAULTSTATUS(as) (MMU_AS(as) + 0x1C) ++#define AS_FAULTSTATUS_ACCESS_TYPE_MASK (0x3 << 8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC (0x0 << 8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_EX (0x1 << 8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_READ (0x2 << 8) ++#define AS_FAULTSTATUS_ACCESS_TYPE_WRITE (0x3 << 8) ++#define AS_FAULTADDRESS_LO(as) (MMU_AS(as) + 0x20) ++#define AS_FAULTADDRESS_HI(as) (MMU_AS(as) + 0x24) ++#define AS_STATUS(as) (MMU_AS(as) + 0x28) ++#define AS_STATUS_AS_ACTIVE BIT(0) ++#define AS_TRANSCFG_LO(as) (MMU_AS(as) + 0x30) ++#define AS_TRANSCFG_HI(as) (MMU_AS(as) + 0x34) ++#define AS_TRANSCFG_ADRMODE_UNMAPPED (1 << 0) ++#define AS_TRANSCFG_ADRMODE_IDENTITY (2 << 0) ++#define AS_TRANSCFG_ADRMODE_AARCH64_4K (6 << 0) ++#define AS_TRANSCFG_ADRMODE_AARCH64_64K (8 << 0) ++#define AS_TRANSCFG_INA_BITS(x) ((x) << 6) ++#define AS_TRANSCFG_OUTA_BITS(x) ((x) << 14) ++#define AS_TRANSCFG_SL_CONCAT BIT(22) ++#define AS_TRANSCFG_PTW_MEMATTR_NC (1 << 24) ++#define AS_TRANSCFG_PTW_MEMATTR_WB (2 << 24) ++#define AS_TRANSCFG_PTW_SH_NS (0 << 28) ++#define AS_TRANSCFG_PTW_SH_OS (2 << 28) ++#define AS_TRANSCFG_PTW_SH_IS (3 << 28) ++#define AS_TRANSCFG_PTW_RA BIT(30) ++#define AS_TRANSCFG_DISABLE_HIER_AP BIT(33) ++#define AS_TRANSCFG_DISABLE_AF_FAULT BIT(34) ++#define AS_TRANSCFG_WXN BIT(35) ++#define AS_TRANSCFG_XREADABLE BIT(36) ++#define AS_FAULTEXTRA_LO(as) (MMU_AS(as) + 0x38) ++#define AS_FAULTEXTRA_HI(as) (MMU_AS(as) + 0x3C) ++ ++#define CSF_GPU_LATEST_FLUSH_ID 0x10000 ++ ++#define CSF_DOORBELL(i) (0x80000 + ((i) * 0x10000)) ++#define CSF_GLB_DOORBELL_ID 0 ++ ++#define gpu_write(dev, reg, data) \ ++ writel(data, (dev)->iomem + (reg)) ++ ++#define gpu_read(dev, reg) \ ++ readl((dev)->iomem + (reg)) ++ ++#endif + +From cb93a21d0a66a553c22c7558ad73757f732cfff6 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:17 +0100 +Subject: [PATCH 082/150] [MOD]drm/panthor: Add the device logical block + +The panthor driver is designed in a modular way, where each logical +block is dealing with a specific HW-block or software feature. In order +for those blocks to communicate with each other, we need a central +panthor_device collecting all the blocks, and exposing some common +features, like interrupt handling, power management, reset, ... + +This what this panthor_device logical block is about. + +v6: +- Add Maxime's and Heiko's acks +- Keep header inclusion alphabetically ordered + +v5: +- Suspend the MMU/GPU blocks if panthor_fw_resume() fails in + panthor_device_resume() +- Move the pm_runtime_use_autosuspend() call before drm_dev_register() +- Add Liviu's R-b + +v4: +- Check drmm_mutex_init() return code +- Fix panthor_device_reset_work() out path +- Fix the race in the unplug logic +- Fix typos +- Unplug blocks when something fails in panthor_device_init() +- Add Steve's R-b + +v3: +- Add acks for the MIT+GPL2 relicensing +- Fix 32-bit support +- Shorten the sections protected by panthor_device::pm::mmio_lock to fix + lock ordering issues. +- Rename panthor_device::pm::lock into panthor_device::pm::mmio_lock to + better reflect what this lock is protecting +- Use dev_err_probe() +- Make sure we call drm_dev_exit() when something fails half-way in + panthor_device_reset_work() +- Replace CSF_GPU_LATEST_FLUSH_ID_DEFAULT with a constant '1' and a + comment to explain. Also remove setting the dummy flush ID on suspend. +- Remove drm_WARN_ON() in panthor_exception_name() +- Check pirq->suspended in panthor_xxx_irq_raw_handler() + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-4-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 548 +++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_device.h | 394 ++++++++++++++++ + 2 files changed, 942 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_device.c + create mode 100644 drivers/gpu/drm/panthor/panthor_device.h + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +new file mode 100644 +index 00000000000000..5be34f4821c59e +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -0,0 +1,548 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "panthor_devfreq.h" ++#include "panthor_device.h" ++#include "panthor_fw.h" ++#include "panthor_gpu.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++#include "panthor_sched.h" ++ ++static int panthor_clk_init(struct panthor_device *ptdev) ++{ ++ ptdev->clks.core = devm_clk_get(ptdev->base.dev, NULL); ++ if (IS_ERR(ptdev->clks.core)) ++ return dev_err_probe(ptdev->base.dev, ++ PTR_ERR(ptdev->clks.core), ++ "get 'core' clock failed"); ++ ++ ptdev->clks.stacks = devm_clk_get_optional(ptdev->base.dev, "stacks"); ++ if (IS_ERR(ptdev->clks.stacks)) ++ return dev_err_probe(ptdev->base.dev, ++ PTR_ERR(ptdev->clks.stacks), ++ "get 'stacks' clock failed"); ++ ++ ptdev->clks.coregroup = devm_clk_get_optional(ptdev->base.dev, "coregroup"); ++ if (IS_ERR(ptdev->clks.coregroup)) ++ return dev_err_probe(ptdev->base.dev, ++ PTR_ERR(ptdev->clks.coregroup), ++ "get 'coregroup' clock failed"); ++ ++ drm_info(&ptdev->base, "clock rate = %lu\n", clk_get_rate(ptdev->clks.core)); ++ return 0; ++} ++ ++void panthor_device_unplug(struct panthor_device *ptdev) ++{ ++ /* This function can be called from two different path: the reset work ++ * and the platform device remove callback. drm_dev_unplug() doesn't ++ * deal with concurrent callers, so we have to protect drm_dev_unplug() ++ * calls with our own lock, and bail out if the device is already ++ * unplugged. ++ */ ++ mutex_lock(&ptdev->unplug.lock); ++ if (drm_dev_is_unplugged(&ptdev->base)) { ++ /* Someone beat us, release the lock and wait for the unplug ++ * operation to be reported as done. ++ **/ ++ mutex_unlock(&ptdev->unplug.lock); ++ wait_for_completion(&ptdev->unplug.done); ++ return; ++ } ++ ++ /* Call drm_dev_unplug() so any access to HW blocks happening after ++ * that point get rejected. ++ */ ++ drm_dev_unplug(&ptdev->base); ++ ++ /* We do the rest of the unplug with the unplug lock released, ++ * future callers will wait on ptdev->unplug.done anyway. ++ */ ++ mutex_unlock(&ptdev->unplug.lock); ++ ++ drm_WARN_ON(&ptdev->base, pm_runtime_get_sync(ptdev->base.dev) < 0); ++ ++ /* Now, try to cleanly shutdown the GPU before the device resources ++ * get reclaimed. ++ */ ++ panthor_sched_unplug(ptdev); ++ panthor_fw_unplug(ptdev); ++ panthor_mmu_unplug(ptdev); ++ panthor_gpu_unplug(ptdev); ++ ++ pm_runtime_dont_use_autosuspend(ptdev->base.dev); ++ pm_runtime_put_sync_suspend(ptdev->base.dev); ++ ++ /* Report the unplug operation as done to unblock concurrent ++ * panthor_device_unplug() callers. ++ */ ++ complete_all(&ptdev->unplug.done); ++} ++ ++static void panthor_device_reset_cleanup(struct drm_device *ddev, void *data) ++{ ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ ++ cancel_work_sync(&ptdev->reset.work); ++ destroy_workqueue(ptdev->reset.wq); ++} ++ ++static void panthor_device_reset_work(struct work_struct *work) ++{ ++ struct panthor_device *ptdev = container_of(work, struct panthor_device, reset.work); ++ int ret = 0, cookie; ++ ++ if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) { ++ /* ++ * No need for a reset as the device has been (or will be) ++ * powered down ++ */ ++ atomic_set(&ptdev->reset.pending, 0); ++ return; ++ } ++ ++ if (!drm_dev_enter(&ptdev->base, &cookie)) ++ return; ++ ++ panthor_sched_pre_reset(ptdev); ++ panthor_fw_pre_reset(ptdev, true); ++ panthor_mmu_pre_reset(ptdev); ++ panthor_gpu_soft_reset(ptdev); ++ panthor_gpu_l2_power_on(ptdev); ++ panthor_mmu_post_reset(ptdev); ++ ret = panthor_fw_post_reset(ptdev); ++ if (ret) ++ goto out_dev_exit; ++ ++ atomic_set(&ptdev->reset.pending, 0); ++ panthor_sched_post_reset(ptdev); ++ ++out_dev_exit: ++ drm_dev_exit(cookie); ++ ++ if (ret) { ++ panthor_device_unplug(ptdev); ++ drm_err(&ptdev->base, "Failed to boot MCU after reset, making device unusable."); ++ } ++} ++ ++static bool panthor_device_is_initialized(struct panthor_device *ptdev) ++{ ++ return !!ptdev->scheduler; ++} ++ ++static void panthor_device_free_page(struct drm_device *ddev, void *data) ++{ ++ free_page((unsigned long)data); ++} ++ ++int panthor_device_init(struct panthor_device *ptdev) ++{ ++ struct resource *res; ++ struct page *p; ++ int ret; ++ ++ ptdev->coherent = device_get_dma_attr(ptdev->base.dev) == DEV_DMA_COHERENT; ++ ++ init_completion(&ptdev->unplug.done); ++ ret = drmm_mutex_init(&ptdev->base, &ptdev->unplug.lock); ++ if (ret) ++ return ret; ++ ++ ret = drmm_mutex_init(&ptdev->base, &ptdev->pm.mmio_lock); ++ if (ret) ++ return ret; ++ ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); ++ p = alloc_page(GFP_KERNEL | __GFP_ZERO); ++ if (!p) ++ return -ENOMEM; ++ ++ ptdev->pm.dummy_latest_flush = page_address(p); ++ ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page, ++ ptdev->pm.dummy_latest_flush); ++ if (ret) ++ return ret; ++ ++ /* ++ * Set the dummy page holding the latest flush to 1. This will cause the ++ * flush to avoided as we know it isn't necessary if the submission ++ * happens while the dummy page is mapped. Zero cannot be used because ++ * that means 'always flush'. ++ */ ++ *ptdev->pm.dummy_latest_flush = 1; ++ ++ INIT_WORK(&ptdev->reset.work, panthor_device_reset_work); ++ ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0); ++ if (!ptdev->reset.wq) ++ return -ENOMEM; ++ ++ ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_reset_cleanup, NULL); ++ if (ret) ++ return ret; ++ ++ ret = panthor_clk_init(ptdev); ++ if (ret) ++ return ret; ++ ++ ret = panthor_devfreq_init(ptdev); ++ if (ret) ++ return ret; ++ ++ ptdev->iomem = devm_platform_get_and_ioremap_resource(to_platform_device(ptdev->base.dev), ++ 0, &res); ++ if (IS_ERR(ptdev->iomem)) ++ return PTR_ERR(ptdev->iomem); ++ ++ ptdev->phys_addr = res->start; ++ ++ ret = devm_pm_runtime_enable(ptdev->base.dev); ++ if (ret) ++ return ret; ++ ++ ret = pm_runtime_resume_and_get(ptdev->base.dev); ++ if (ret) ++ return ret; ++ ++ ret = panthor_gpu_init(ptdev); ++ if (ret) ++ goto err_rpm_put; ++ ++ ret = panthor_mmu_init(ptdev); ++ if (ret) ++ goto err_unplug_gpu; ++ ++ ret = panthor_fw_init(ptdev); ++ if (ret) ++ goto err_unplug_mmu; ++ ++ ret = panthor_sched_init(ptdev); ++ if (ret) ++ goto err_unplug_fw; ++ ++ /* ~3 frames */ ++ pm_runtime_set_autosuspend_delay(ptdev->base.dev, 50); ++ pm_runtime_use_autosuspend(ptdev->base.dev); ++ ++ ret = drm_dev_register(&ptdev->base, 0); ++ if (ret) ++ goto err_disable_autosuspend; ++ ++ pm_runtime_put_autosuspend(ptdev->base.dev); ++ return 0; ++ ++err_disable_autosuspend: ++ pm_runtime_dont_use_autosuspend(ptdev->base.dev); ++ panthor_sched_unplug(ptdev); ++ ++err_unplug_fw: ++ panthor_fw_unplug(ptdev); ++ ++err_unplug_mmu: ++ panthor_mmu_unplug(ptdev); ++ ++err_unplug_gpu: ++ panthor_gpu_unplug(ptdev); ++ ++err_rpm_put: ++ pm_runtime_put_sync_suspend(ptdev->base.dev); ++ return ret; ++} ++ ++#define PANTHOR_EXCEPTION(id) \ ++ [DRM_PANTHOR_EXCEPTION_ ## id] = { \ ++ .name = #id, \ ++ } ++ ++struct panthor_exception_info { ++ const char *name; ++}; ++ ++static const struct panthor_exception_info panthor_exception_infos[] = { ++ PANTHOR_EXCEPTION(OK), ++ PANTHOR_EXCEPTION(TERMINATED), ++ PANTHOR_EXCEPTION(KABOOM), ++ PANTHOR_EXCEPTION(EUREKA), ++ PANTHOR_EXCEPTION(ACTIVE), ++ PANTHOR_EXCEPTION(CS_RES_TERM), ++ PANTHOR_EXCEPTION(CS_CONFIG_FAULT), ++ PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT), ++ PANTHOR_EXCEPTION(CS_BUS_FAULT), ++ PANTHOR_EXCEPTION(CS_INSTR_INVALID), ++ PANTHOR_EXCEPTION(CS_CALL_STACK_OVERFLOW), ++ PANTHOR_EXCEPTION(CS_INHERIT_FAULT), ++ PANTHOR_EXCEPTION(INSTR_INVALID_PC), ++ PANTHOR_EXCEPTION(INSTR_INVALID_ENC), ++ PANTHOR_EXCEPTION(INSTR_BARRIER_FAULT), ++ PANTHOR_EXCEPTION(DATA_INVALID_FAULT), ++ PANTHOR_EXCEPTION(TILE_RANGE_FAULT), ++ PANTHOR_EXCEPTION(ADDR_RANGE_FAULT), ++ PANTHOR_EXCEPTION(IMPRECISE_FAULT), ++ PANTHOR_EXCEPTION(OOM), ++ PANTHOR_EXCEPTION(CSF_FW_INTERNAL_ERROR), ++ PANTHOR_EXCEPTION(CSF_RES_EVICTION_TIMEOUT), ++ PANTHOR_EXCEPTION(GPU_BUS_FAULT), ++ PANTHOR_EXCEPTION(GPU_SHAREABILITY_FAULT), ++ PANTHOR_EXCEPTION(SYS_SHAREABILITY_FAULT), ++ PANTHOR_EXCEPTION(GPU_CACHEABILITY_FAULT), ++ PANTHOR_EXCEPTION(TRANSLATION_FAULT_0), ++ PANTHOR_EXCEPTION(TRANSLATION_FAULT_1), ++ PANTHOR_EXCEPTION(TRANSLATION_FAULT_2), ++ PANTHOR_EXCEPTION(TRANSLATION_FAULT_3), ++ PANTHOR_EXCEPTION(TRANSLATION_FAULT_4), ++ PANTHOR_EXCEPTION(PERM_FAULT_0), ++ PANTHOR_EXCEPTION(PERM_FAULT_1), ++ PANTHOR_EXCEPTION(PERM_FAULT_2), ++ PANTHOR_EXCEPTION(PERM_FAULT_3), ++ PANTHOR_EXCEPTION(ACCESS_FLAG_1), ++ PANTHOR_EXCEPTION(ACCESS_FLAG_2), ++ PANTHOR_EXCEPTION(ACCESS_FLAG_3), ++ PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_IN), ++ PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT0), ++ PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT1), ++ PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT2), ++ PANTHOR_EXCEPTION(ADDR_SIZE_FAULT_OUT3), ++ PANTHOR_EXCEPTION(MEM_ATTR_FAULT_0), ++ PANTHOR_EXCEPTION(MEM_ATTR_FAULT_1), ++ PANTHOR_EXCEPTION(MEM_ATTR_FAULT_2), ++ PANTHOR_EXCEPTION(MEM_ATTR_FAULT_3), ++}; ++ ++const char *panthor_exception_name(struct panthor_device *ptdev, u32 exception_code) ++{ ++ if (exception_code >= ARRAY_SIZE(panthor_exception_infos) || ++ !panthor_exception_infos[exception_code].name) ++ return "Unknown exception type"; ++ ++ return panthor_exception_infos[exception_code].name; ++} ++ ++static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf) ++{ ++ struct vm_area_struct *vma = vmf->vma; ++ struct panthor_device *ptdev = vma->vm_private_data; ++ u64 id = (u64)vma->vm_pgoff << PAGE_SHIFT; ++ unsigned long pfn; ++ pgprot_t pgprot; ++ vm_fault_t ret; ++ bool active; ++ int cookie; ++ ++ if (!drm_dev_enter(&ptdev->base, &cookie)) ++ return VM_FAULT_SIGBUS; ++ ++ mutex_lock(&ptdev->pm.mmio_lock); ++ active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE; ++ ++ switch (panthor_device_mmio_offset(id)) { ++ case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: ++ if (active) ++ pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID); ++ else ++ pfn = virt_to_pfn(ptdev->pm.dummy_latest_flush); ++ break; ++ ++ default: ++ ret = VM_FAULT_SIGBUS; ++ goto out_unlock; ++ } ++ ++ pgprot = vma->vm_page_prot; ++ if (active) ++ pgprot = pgprot_noncached(pgprot); ++ ++ ret = vmf_insert_pfn_prot(vma, vmf->address, pfn, pgprot); ++ ++out_unlock: ++ mutex_unlock(&ptdev->pm.mmio_lock); ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static const struct vm_operations_struct panthor_mmio_vm_ops = { ++ .fault = panthor_mmio_vm_fault, ++}; ++ ++int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma) ++{ ++ u64 id = (u64)vma->vm_pgoff << PAGE_SHIFT; ++ ++ switch (panthor_device_mmio_offset(id)) { ++ case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: ++ if (vma->vm_end - vma->vm_start != PAGE_SIZE || ++ (vma->vm_flags & (VM_WRITE | VM_EXEC))) ++ return -EINVAL; ++ ++ break; ++ ++ default: ++ return -EINVAL; ++ } ++ ++ /* Defer actual mapping to the fault handler. */ ++ vma->vm_private_data = ptdev; ++ vma->vm_ops = &panthor_mmio_vm_ops; ++ vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND | ++ VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP; ++ return 0; ++} ++ ++#ifdef CONFIG_PM ++int panthor_device_resume(struct device *dev) ++{ ++ struct panthor_device *ptdev = dev_get_drvdata(dev); ++ int ret, cookie; ++ ++ if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_SUSPENDED) ++ return -EINVAL; ++ ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_RESUMING); ++ ++ ret = clk_prepare_enable(ptdev->clks.core); ++ if (ret) ++ goto err_set_suspended; ++ ++ ret = clk_prepare_enable(ptdev->clks.stacks); ++ if (ret) ++ goto err_disable_core_clk; ++ ++ ret = clk_prepare_enable(ptdev->clks.coregroup); ++ if (ret) ++ goto err_disable_stacks_clk; ++ ++ ret = panthor_devfreq_resume(ptdev); ++ if (ret) ++ goto err_disable_coregroup_clk; ++ ++ if (panthor_device_is_initialized(ptdev) && ++ drm_dev_enter(&ptdev->base, &cookie)) { ++ panthor_gpu_resume(ptdev); ++ panthor_mmu_resume(ptdev); ++ ret = drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev)); ++ if (!ret) { ++ panthor_sched_resume(ptdev); ++ } else { ++ panthor_mmu_suspend(ptdev); ++ panthor_gpu_suspend(ptdev); ++ } ++ ++ drm_dev_exit(cookie); ++ ++ if (ret) ++ goto err_suspend_devfreq; ++ } ++ ++ if (atomic_read(&ptdev->reset.pending)) ++ queue_work(ptdev->reset.wq, &ptdev->reset.work); ++ ++ /* Clear all IOMEM mappings pointing to this device after we've ++ * resumed. This way the fake mappings pointing to the dummy pages ++ * are removed and the real iomem mapping will be restored on next ++ * access. ++ */ ++ mutex_lock(&ptdev->pm.mmio_lock); ++ unmap_mapping_range(ptdev->base.anon_inode->i_mapping, ++ DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE); ++ mutex_unlock(&ptdev->pm.mmio_lock); ++ return 0; ++ ++err_suspend_devfreq: ++ panthor_devfreq_suspend(ptdev); ++ ++err_disable_coregroup_clk: ++ clk_disable_unprepare(ptdev->clks.coregroup); ++ ++err_disable_stacks_clk: ++ clk_disable_unprepare(ptdev->clks.stacks); ++ ++err_disable_core_clk: ++ clk_disable_unprepare(ptdev->clks.core); ++ ++err_set_suspended: ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); ++ return ret; ++} ++ ++int panthor_device_suspend(struct device *dev) ++{ ++ struct panthor_device *ptdev = dev_get_drvdata(dev); ++ int ret, cookie; ++ ++ if (atomic_read(&ptdev->pm.state) != PANTHOR_DEVICE_PM_STATE_ACTIVE) ++ return -EINVAL; ++ ++ /* Clear all IOMEM mappings pointing to this device before we ++ * shutdown the power-domain and clocks. Failing to do that results ++ * in external aborts when the process accesses the iomem region. ++ * We change the state and call unmap_mapping_range() with the ++ * mmio_lock held to make sure the vm_fault handler won't set up ++ * invalid mappings. ++ */ ++ mutex_lock(&ptdev->pm.mmio_lock); ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDING); ++ unmap_mapping_range(ptdev->base.anon_inode->i_mapping, ++ DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); ++ mutex_unlock(&ptdev->pm.mmio_lock); ++ ++ if (panthor_device_is_initialized(ptdev) && ++ drm_dev_enter(&ptdev->base, &cookie)) { ++ cancel_work_sync(&ptdev->reset.work); ++ ++ /* We prepare everything as if we were resetting the GPU. ++ * The end of the reset will happen in the resume path though. ++ */ ++ panthor_sched_suspend(ptdev); ++ panthor_fw_suspend(ptdev); ++ panthor_mmu_suspend(ptdev); ++ panthor_gpu_suspend(ptdev); ++ drm_dev_exit(cookie); ++ } ++ ++ ret = panthor_devfreq_suspend(ptdev); ++ if (ret) { ++ if (panthor_device_is_initialized(ptdev) && ++ drm_dev_enter(&ptdev->base, &cookie)) { ++ panthor_gpu_resume(ptdev); ++ panthor_mmu_resume(ptdev); ++ drm_WARN_ON(&ptdev->base, panthor_fw_resume(ptdev)); ++ panthor_sched_resume(ptdev); ++ drm_dev_exit(cookie); ++ } ++ ++ goto err_set_active; ++ } ++ ++ clk_disable_unprepare(ptdev->clks.coregroup); ++ clk_disable_unprepare(ptdev->clks.stacks); ++ clk_disable_unprepare(ptdev->clks.core); ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_SUSPENDED); ++ return 0; ++ ++err_set_active: ++ /* If something failed and we have to revert back to an ++ * active state, we also need to clear the MMIO userspace ++ * mappings, so any dumb pages that were mapped while we ++ * were trying to suspend gets invalidated. ++ */ ++ mutex_lock(&ptdev->pm.mmio_lock); ++ atomic_set(&ptdev->pm.state, PANTHOR_DEVICE_PM_STATE_ACTIVE); ++ unmap_mapping_range(ptdev->base.anon_inode->i_mapping, ++ DRM_PANTHOR_USER_MMIO_OFFSET, 0, 1); ++ mutex_unlock(&ptdev->pm.mmio_lock); ++ return ret; ++} ++#endif +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +new file mode 100644 +index 00000000000000..51c9d61b67963d +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -0,0 +1,394 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_DEVICE_H__ ++#define __PANTHOR_DEVICE_H__ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++ ++struct panthor_csf; ++struct panthor_csf_ctx; ++struct panthor_device; ++struct panthor_gpu; ++struct panthor_group_pool; ++struct panthor_heap_pool; ++struct panthor_job; ++struct panthor_mmu; ++struct panthor_fw; ++struct panthor_perfcnt; ++struct panthor_vm; ++struct panthor_vm_pool; ++ ++/** ++ * enum panthor_device_pm_state - PM state ++ */ ++enum panthor_device_pm_state { ++ /** @PANTHOR_DEVICE_PM_STATE_SUSPENDED: Device is suspended. */ ++ PANTHOR_DEVICE_PM_STATE_SUSPENDED = 0, ++ ++ /** @PANTHOR_DEVICE_PM_STATE_RESUMING: Device is being resumed. */ ++ PANTHOR_DEVICE_PM_STATE_RESUMING, ++ ++ /** @PANTHOR_DEVICE_PM_STATE_ACTIVE: Device is active. */ ++ PANTHOR_DEVICE_PM_STATE_ACTIVE, ++ ++ /** @PANTHOR_DEVICE_PM_STATE_SUSPENDING: Device is being suspended. */ ++ PANTHOR_DEVICE_PM_STATE_SUSPENDING, ++}; ++ ++/** ++ * struct panthor_irq - IRQ data ++ * ++ * Used to automate IRQ handling for the 3 different IRQs we have in this driver. ++ */ ++struct panthor_irq { ++ /** @ptdev: Panthor device */ ++ struct panthor_device *ptdev; ++ ++ /** @irq: IRQ number. */ ++ int irq; ++ ++ /** @mask: Current mask being applied to xxx_INT_MASK. */ ++ u32 mask; ++ ++ /** @suspended: Set to true when the IRQ is suspended. */ ++ atomic_t suspended; ++}; ++ ++/** ++ * struct panthor_device - Panthor device ++ */ ++struct panthor_device { ++ /** @base: Base drm_device. */ ++ struct drm_device base; ++ ++ /** @phys_addr: Physical address of the iomem region. */ ++ phys_addr_t phys_addr; ++ ++ /** @iomem: CPU mapping of the IOMEM region. */ ++ void __iomem *iomem; ++ ++ /** @clks: GPU clocks. */ ++ struct { ++ /** @core: Core clock. */ ++ struct clk *core; ++ ++ /** @stacks: Stacks clock. This clock is optional. */ ++ struct clk *stacks; ++ ++ /** @coregroup: Core group clock. This clock is optional. */ ++ struct clk *coregroup; ++ } clks; ++ ++ /** @coherent: True if the CPU/GPU are memory coherent. */ ++ bool coherent; ++ ++ /** @gpu_info: GPU information. */ ++ struct drm_panthor_gpu_info gpu_info; ++ ++ /** @csif_info: Command stream interface information. */ ++ struct drm_panthor_csif_info csif_info; ++ ++ /** @gpu: GPU management data. */ ++ struct panthor_gpu *gpu; ++ ++ /** @fw: FW management data. */ ++ struct panthor_fw *fw; ++ ++ /** @mmu: MMU management data. */ ++ struct panthor_mmu *mmu; ++ ++ /** @scheduler: Scheduler management data. */ ++ struct panthor_scheduler *scheduler; ++ ++ /** @devfreq: Device frequency scaling management data. */ ++ struct panthor_devfreq *devfreq; ++ ++ /** @unplug: Device unplug related fields. */ ++ struct { ++ /** @lock: Lock used to serialize unplug operations. */ ++ struct mutex lock; ++ ++ /** ++ * @done: Completion object signaled when the unplug ++ * operation is done. ++ */ ++ struct completion done; ++ } unplug; ++ ++ /** @reset: Reset related fields. */ ++ struct { ++ /** @wq: Ordered worqueud used to schedule reset operations. */ ++ struct workqueue_struct *wq; ++ ++ /** @work: Reset work. */ ++ struct work_struct work; ++ ++ /** @pending: Set to true if a reset is pending. */ ++ atomic_t pending; ++ } reset; ++ ++ /** @pm: Power management related data. */ ++ struct { ++ /** @state: Power state. */ ++ atomic_t state; ++ ++ /** ++ * @mmio_lock: Lock protecting MMIO userspace CPU mappings. ++ * ++ * This is needed to ensure we map the dummy IO pages when ++ * the device is being suspended, and the real IO pages when ++ * the device is being resumed. We can't just do with the ++ * state atomicity to deal with this race. ++ */ ++ struct mutex mmio_lock; ++ ++ /** ++ * @dummy_latest_flush: Dummy LATEST_FLUSH page. ++ * ++ * Used to replace the real LATEST_FLUSH page when the GPU ++ * is suspended. ++ */ ++ u32 *dummy_latest_flush; ++ } pm; ++}; ++ ++/** ++ * struct panthor_file - Panthor file ++ */ ++struct panthor_file { ++ /** @ptdev: Device attached to this file. */ ++ struct panthor_device *ptdev; ++ ++ /** @vms: VM pool attached to this file. */ ++ struct panthor_vm_pool *vms; ++ ++ /** @groups: Scheduling group pool attached to this file. */ ++ struct panthor_group_pool *groups; ++}; ++ ++int panthor_device_init(struct panthor_device *ptdev); ++void panthor_device_unplug(struct panthor_device *ptdev); ++ ++/** ++ * panthor_device_schedule_reset() - Schedules a reset operation ++ */ ++static inline void panthor_device_schedule_reset(struct panthor_device *ptdev) ++{ ++ if (!atomic_cmpxchg(&ptdev->reset.pending, 0, 1) && ++ atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE) ++ queue_work(ptdev->reset.wq, &ptdev->reset.work); ++} ++ ++/** ++ * panthor_device_reset_is_pending() - Checks if a reset is pending. ++ * ++ * Return: true if a reset is pending, false otherwise. ++ */ ++static inline bool panthor_device_reset_is_pending(struct panthor_device *ptdev) ++{ ++ return atomic_read(&ptdev->reset.pending) != 0; ++} ++ ++int panthor_device_mmap_io(struct panthor_device *ptdev, ++ struct vm_area_struct *vma); ++ ++int panthor_device_resume(struct device *dev); ++int panthor_device_suspend(struct device *dev); ++ ++enum drm_panthor_exception_type { ++ DRM_PANTHOR_EXCEPTION_OK = 0x00, ++ DRM_PANTHOR_EXCEPTION_TERMINATED = 0x04, ++ DRM_PANTHOR_EXCEPTION_KABOOM = 0x05, ++ DRM_PANTHOR_EXCEPTION_EUREKA = 0x06, ++ DRM_PANTHOR_EXCEPTION_ACTIVE = 0x08, ++ DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f, ++ DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f, ++ DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40, ++ DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44, ++ DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48, ++ DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49, ++ DRM_PANTHOR_EXCEPTION_CS_CALL_STACK_OVERFLOW = 0x4a, ++ DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT = 0x4b, ++ DRM_PANTHOR_EXCEPTION_INSTR_INVALID_PC = 0x50, ++ DRM_PANTHOR_EXCEPTION_INSTR_INVALID_ENC = 0x51, ++ DRM_PANTHOR_EXCEPTION_INSTR_BARRIER_FAULT = 0x55, ++ DRM_PANTHOR_EXCEPTION_DATA_INVALID_FAULT = 0x58, ++ DRM_PANTHOR_EXCEPTION_TILE_RANGE_FAULT = 0x59, ++ DRM_PANTHOR_EXCEPTION_ADDR_RANGE_FAULT = 0x5a, ++ DRM_PANTHOR_EXCEPTION_IMPRECISE_FAULT = 0x5b, ++ DRM_PANTHOR_EXCEPTION_OOM = 0x60, ++ DRM_PANTHOR_EXCEPTION_CSF_FW_INTERNAL_ERROR = 0x68, ++ DRM_PANTHOR_EXCEPTION_CSF_RES_EVICTION_TIMEOUT = 0x69, ++ DRM_PANTHOR_EXCEPTION_GPU_BUS_FAULT = 0x80, ++ DRM_PANTHOR_EXCEPTION_GPU_SHAREABILITY_FAULT = 0x88, ++ DRM_PANTHOR_EXCEPTION_SYS_SHAREABILITY_FAULT = 0x89, ++ DRM_PANTHOR_EXCEPTION_GPU_CACHEABILITY_FAULT = 0x8a, ++ DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_0 = 0xc0, ++ DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_1 = 0xc1, ++ DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_2 = 0xc2, ++ DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_3 = 0xc3, ++ DRM_PANTHOR_EXCEPTION_TRANSLATION_FAULT_4 = 0xc4, ++ DRM_PANTHOR_EXCEPTION_PERM_FAULT_0 = 0xc8, ++ DRM_PANTHOR_EXCEPTION_PERM_FAULT_1 = 0xc9, ++ DRM_PANTHOR_EXCEPTION_PERM_FAULT_2 = 0xca, ++ DRM_PANTHOR_EXCEPTION_PERM_FAULT_3 = 0xcb, ++ DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_1 = 0xd9, ++ DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_2 = 0xda, ++ DRM_PANTHOR_EXCEPTION_ACCESS_FLAG_3 = 0xdb, ++ DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_IN = 0xe0, ++ DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT0 = 0xe4, ++ DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT1 = 0xe5, ++ DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT2 = 0xe6, ++ DRM_PANTHOR_EXCEPTION_ADDR_SIZE_FAULT_OUT3 = 0xe7, ++ DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_0 = 0xe8, ++ DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_1 = 0xe9, ++ DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_2 = 0xea, ++ DRM_PANTHOR_EXCEPTION_MEM_ATTR_FAULT_3 = 0xeb, ++}; ++ ++/** ++ * panthor_exception_is_fault() - Checks if an exception is a fault. ++ * ++ * Return: true if the exception is a fault, false otherwise. ++ */ ++static inline bool ++panthor_exception_is_fault(u32 exception_code) ++{ ++ return exception_code > DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT; ++} ++ ++const char *panthor_exception_name(struct panthor_device *ptdev, ++ u32 exception_code); ++ ++/** ++ * PANTHOR_IRQ_HANDLER() - Define interrupt handlers and the interrupt ++ * registration function. ++ * ++ * The boiler-plate to gracefully deal with shared interrupts is ++ * auto-generated. All you have to do is call PANTHOR_IRQ_HANDLER() ++ * just after the actual handler. The handler prototype is: ++ * ++ * void (*handler)(struct panthor_device *, u32 status); ++ */ ++#define PANTHOR_IRQ_HANDLER(__name, __reg_prefix, __handler) \ ++static irqreturn_t panthor_ ## __name ## _irq_raw_handler(int irq, void *data) \ ++{ \ ++ struct panthor_irq *pirq = data; \ ++ struct panthor_device *ptdev = pirq->ptdev; \ ++ \ ++ if (atomic_read(&pirq->suspended)) \ ++ return IRQ_NONE; \ ++ if (!gpu_read(ptdev, __reg_prefix ## _INT_STAT)) \ ++ return IRQ_NONE; \ ++ \ ++ gpu_write(ptdev, __reg_prefix ## _INT_MASK, 0); \ ++ return IRQ_WAKE_THREAD; \ ++} \ ++ \ ++static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *data) \ ++{ \ ++ struct panthor_irq *pirq = data; \ ++ struct panthor_device *ptdev = pirq->ptdev; \ ++ irqreturn_t ret = IRQ_NONE; \ ++ \ ++ while (true) { \ ++ u32 status = gpu_read(ptdev, __reg_prefix ## _INT_RAWSTAT) & pirq->mask; \ ++ \ ++ if (!status) \ ++ break; \ ++ \ ++ gpu_write(ptdev, __reg_prefix ## _INT_CLEAR, status); \ ++ \ ++ __handler(ptdev, status); \ ++ ret = IRQ_HANDLED; \ ++ } \ ++ \ ++ if (!atomic_read(&pirq->suspended)) \ ++ gpu_write(ptdev, __reg_prefix ## _INT_MASK, pirq->mask); \ ++ \ ++ return ret; \ ++} \ ++ \ ++static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \ ++{ \ ++ int cookie; \ ++ \ ++ atomic_set(&pirq->suspended, true); \ ++ \ ++ if (drm_dev_enter(&pirq->ptdev->base, &cookie)) { \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ ++ synchronize_irq(pirq->irq); \ ++ drm_dev_exit(cookie); \ ++ } \ ++ \ ++ pirq->mask = 0; \ ++} \ ++ \ ++static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask) \ ++{ \ ++ int cookie; \ ++ \ ++ atomic_set(&pirq->suspended, false); \ ++ pirq->mask = mask; \ ++ \ ++ if (drm_dev_enter(&pirq->ptdev->base, &cookie)) { \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask); \ ++ drm_dev_exit(cookie); \ ++ } \ ++} \ ++ \ ++static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ ++ struct panthor_irq *pirq, \ ++ int irq, u32 mask) \ ++{ \ ++ pirq->ptdev = ptdev; \ ++ pirq->irq = irq; \ ++ panthor_ ## __name ## _irq_resume(pirq, mask); \ ++ \ ++ return devm_request_threaded_irq(ptdev->base.dev, irq, \ ++ panthor_ ## __name ## _irq_raw_handler, \ ++ panthor_ ## __name ## _irq_threaded_handler, \ ++ IRQF_SHARED, KBUILD_MODNAME "-" # __name, \ ++ pirq); \ ++} ++ ++/** ++ * panthor_device_mmio_offset() - Turn a user MMIO offset into a kernel one ++ * @offset: Offset to convert. ++ * ++ * With 32-bit systems being limited by the 32-bit representation of mmap2's ++ * pgoffset field, we need to make the MMIO offset arch specific. This function ++ * converts a user MMIO offset into something the kernel driver understands. ++ * ++ * If the kernel and userspace architecture match, the offset is unchanged. If ++ * the kernel is 64-bit and userspace is 32-bit, the offset is adjusted to match ++ * 64-bit offsets. 32-bit kernel with 64-bit userspace is impossible. ++ * ++ * Return: Adjusted offset. ++ */ ++static inline u64 panthor_device_mmio_offset(u64 offset) ++{ ++#ifdef CONFIG_ARM64 ++ if (test_tsk_thread_flag(current, TIF_32BIT)) ++ offset += DRM_PANTHOR_USER_MMIO_OFFSET_64BIT - DRM_PANTHOR_USER_MMIO_OFFSET_32BIT; ++#endif ++ ++ return offset; ++} ++ ++extern struct workqueue_struct *panthor_cleanup_wq; ++ ++#endif + +From 45e08b89af8f0e8a92fafb3bf579e57057fb9c73 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:18 +0100 +Subject: [PATCH 083/150] drm/panthor: Add the GPU logical block + +Handles everything that's not related to the FW, the MMU or the +scheduler. This is the block dealing with the GPU property retrieval, +the GPU block power on/off logic, and some global operations, like +global cache flushing. + +v6: +- Add Maxime's and Heiko's acks + +v5: +- Fix GPU_MODEL() kernel doc +- Fix test in panthor_gpu_block_power_off() +- Add Steve's R-b + +v4: +- Expose CORE_FEATURES through DEV_QUERY + +v3: +- Add acks for the MIT/GPL2 relicensing +- Use macros to extract GPU ID info +- Make sure we reset clear pending_reqs bits when wait_event_timeout() + times out but the corresponding bit is cleared in GPU_INT_RAWSTAT + (can happen if the IRQ is masked or HW takes to long to call the IRQ + handler) +- GPU_MODEL now takes separate arch and product majors to be more + readable. +- Drop GPU_IRQ_MCU_STATUS_CHANGED from interrupt mask. +- Handle GPU_IRQ_PROTM_FAULT correctly (don't output registers that are + not updated for protected interrupts). +- Minor code tidy ups + +Cc: Alexey Sheplyakov # MIT+GPL2 relicensing +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-5-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_gpu.c | 482 ++++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_gpu.h | 52 +++ + 2 files changed, 534 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_gpu.c + create mode 100644 drivers/gpu/drm/panthor/panthor_gpu.h + +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c +new file mode 100644 +index 00000000000000..6dbbc4cfbe7e6a +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_gpu.c +@@ -0,0 +1,482 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Linaro, Ltd., Rob Herring */ ++/* Copyright 2019 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "panthor_device.h" ++#include "panthor_gpu.h" ++#include "panthor_regs.h" ++ ++/** ++ * struct panthor_gpu - GPU block management data. ++ */ ++struct panthor_gpu { ++ /** @irq: GPU irq. */ ++ struct panthor_irq irq; ++ ++ /** @reqs_lock: Lock protecting access to pending_reqs. */ ++ spinlock_t reqs_lock; ++ ++ /** @pending_reqs: Pending GPU requests. */ ++ u32 pending_reqs; ++ ++ /** @reqs_acked: GPU request wait queue. */ ++ wait_queue_head_t reqs_acked; ++}; ++ ++/** ++ * struct panthor_model - GPU model description ++ */ ++struct panthor_model { ++ /** @name: Model name. */ ++ const char *name; ++ ++ /** @arch_major: Major version number of architecture. */ ++ u8 arch_major; ++ ++ /** @product_major: Major version number of product. */ ++ u8 product_major; ++}; ++ ++/** ++ * GPU_MODEL() - Define a GPU model. A GPU product can be uniquely identified ++ * by a combination of the major architecture version and the major product ++ * version. ++ * @_name: Name for the GPU model. ++ * @_arch_major: Architecture major. ++ * @_product_major: Product major. ++ */ ++#define GPU_MODEL(_name, _arch_major, _product_major) \ ++{\ ++ .name = __stringify(_name), \ ++ .arch_major = _arch_major, \ ++ .product_major = _product_major, \ ++} ++ ++static const struct panthor_model gpu_models[] = { ++ GPU_MODEL(g610, 10, 7), ++ {}, ++}; ++ ++#define GPU_INTERRUPTS_MASK \ ++ (GPU_IRQ_FAULT | \ ++ GPU_IRQ_PROTM_FAULT | \ ++ GPU_IRQ_RESET_COMPLETED | \ ++ GPU_IRQ_CLEAN_CACHES_COMPLETED) ++ ++static void panthor_gpu_init_info(struct panthor_device *ptdev) ++{ ++ const struct panthor_model *model; ++ u32 arch_major, product_major; ++ u32 major, minor, status; ++ unsigned int i; ++ ++ ptdev->gpu_info.gpu_id = gpu_read(ptdev, GPU_ID); ++ ptdev->gpu_info.csf_id = gpu_read(ptdev, GPU_CSF_ID); ++ ptdev->gpu_info.gpu_rev = gpu_read(ptdev, GPU_REVID); ++ ptdev->gpu_info.core_features = gpu_read(ptdev, GPU_CORE_FEATURES); ++ ptdev->gpu_info.l2_features = gpu_read(ptdev, GPU_L2_FEATURES); ++ ptdev->gpu_info.tiler_features = gpu_read(ptdev, GPU_TILER_FEATURES); ++ ptdev->gpu_info.mem_features = gpu_read(ptdev, GPU_MEM_FEATURES); ++ ptdev->gpu_info.mmu_features = gpu_read(ptdev, GPU_MMU_FEATURES); ++ ptdev->gpu_info.thread_features = gpu_read(ptdev, GPU_THREAD_FEATURES); ++ ptdev->gpu_info.max_threads = gpu_read(ptdev, GPU_THREAD_MAX_THREADS); ++ ptdev->gpu_info.thread_max_workgroup_size = gpu_read(ptdev, GPU_THREAD_MAX_WORKGROUP_SIZE); ++ ptdev->gpu_info.thread_max_barrier_size = gpu_read(ptdev, GPU_THREAD_MAX_BARRIER_SIZE); ++ ptdev->gpu_info.coherency_features = gpu_read(ptdev, GPU_COHERENCY_FEATURES); ++ for (i = 0; i < 4; i++) ++ ptdev->gpu_info.texture_features[i] = gpu_read(ptdev, GPU_TEXTURE_FEATURES(i)); ++ ++ ptdev->gpu_info.as_present = gpu_read(ptdev, GPU_AS_PRESENT); ++ ++ ptdev->gpu_info.shader_present = gpu_read(ptdev, GPU_SHADER_PRESENT_LO); ++ ptdev->gpu_info.shader_present |= (u64)gpu_read(ptdev, GPU_SHADER_PRESENT_HI) << 32; ++ ++ ptdev->gpu_info.tiler_present = gpu_read(ptdev, GPU_TILER_PRESENT_LO); ++ ptdev->gpu_info.tiler_present |= (u64)gpu_read(ptdev, GPU_TILER_PRESENT_HI) << 32; ++ ++ ptdev->gpu_info.l2_present = gpu_read(ptdev, GPU_L2_PRESENT_LO); ++ ptdev->gpu_info.l2_present |= (u64)gpu_read(ptdev, GPU_L2_PRESENT_HI) << 32; ++ ++ arch_major = GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id); ++ product_major = GPU_PROD_MAJOR(ptdev->gpu_info.gpu_id); ++ major = GPU_VER_MAJOR(ptdev->gpu_info.gpu_id); ++ minor = GPU_VER_MINOR(ptdev->gpu_info.gpu_id); ++ status = GPU_VER_STATUS(ptdev->gpu_info.gpu_id); ++ ++ for (model = gpu_models; model->name; model++) { ++ if (model->arch_major == arch_major && ++ model->product_major == product_major) ++ break; ++ } ++ ++ drm_info(&ptdev->base, ++ "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x", ++ model->name ?: "unknown", ptdev->gpu_info.gpu_id >> 16, ++ major, minor, status); ++ ++ drm_info(&ptdev->base, ++ "Features: L2:%#x Tiler:%#x Mem:%#x MMU:%#x AS:%#x", ++ ptdev->gpu_info.l2_features, ++ ptdev->gpu_info.tiler_features, ++ ptdev->gpu_info.mem_features, ++ ptdev->gpu_info.mmu_features, ++ ptdev->gpu_info.as_present); ++ ++ drm_info(&ptdev->base, ++ "shader_present=0x%0llx l2_present=0x%0llx tiler_present=0x%0llx", ++ ptdev->gpu_info.shader_present, ptdev->gpu_info.l2_present, ++ ptdev->gpu_info.tiler_present); ++} ++ ++static void panthor_gpu_irq_handler(struct panthor_device *ptdev, u32 status) ++{ ++ if (status & GPU_IRQ_FAULT) { ++ u32 fault_status = gpu_read(ptdev, GPU_FAULT_STATUS); ++ u64 address = ((u64)gpu_read(ptdev, GPU_FAULT_ADDR_HI) << 32) | ++ gpu_read(ptdev, GPU_FAULT_ADDR_LO); ++ ++ drm_warn(&ptdev->base, "GPU Fault 0x%08x (%s) at 0x%016llx\n", ++ fault_status, panthor_exception_name(ptdev, fault_status & 0xFF), ++ address); ++ } ++ if (status & GPU_IRQ_PROTM_FAULT) ++ drm_warn(&ptdev->base, "GPU Fault in protected mode\n"); ++ ++ spin_lock(&ptdev->gpu->reqs_lock); ++ if (status & ptdev->gpu->pending_reqs) { ++ ptdev->gpu->pending_reqs &= ~status; ++ wake_up_all(&ptdev->gpu->reqs_acked); ++ } ++ spin_unlock(&ptdev->gpu->reqs_lock); ++} ++PANTHOR_IRQ_HANDLER(gpu, GPU, panthor_gpu_irq_handler); ++ ++/** ++ * panthor_gpu_unplug() - Called when the GPU is unplugged. ++ * @ptdev: Device to unplug. ++ */ ++void panthor_gpu_unplug(struct panthor_device *ptdev) ++{ ++ unsigned long flags; ++ ++ /* Make sure the IRQ handler is not running after that point. */ ++ panthor_gpu_irq_suspend(&ptdev->gpu->irq); ++ ++ /* Wake-up all waiters. */ ++ spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); ++ ptdev->gpu->pending_reqs = 0; ++ wake_up_all(&ptdev->gpu->reqs_acked); ++ spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); ++} ++ ++/** ++ * panthor_gpu_init() - Initialize the GPU block ++ * @ptdev: Device. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_init(struct panthor_device *ptdev) ++{ ++ struct panthor_gpu *gpu; ++ u32 pa_bits; ++ int ret, irq; ++ ++ gpu = drmm_kzalloc(&ptdev->base, sizeof(*gpu), GFP_KERNEL); ++ if (!gpu) ++ return -ENOMEM; ++ ++ spin_lock_init(&gpu->reqs_lock); ++ init_waitqueue_head(&gpu->reqs_acked); ++ ptdev->gpu = gpu; ++ panthor_gpu_init_info(ptdev); ++ ++ dma_set_max_seg_size(ptdev->base.dev, UINT_MAX); ++ pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features); ++ ret = dma_set_mask_and_coherent(ptdev->base.dev, DMA_BIT_MASK(pa_bits)); ++ if (ret) ++ return ret; ++ ++ irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); ++ if (irq <= 0) ++ return ret; ++ ++ ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); ++ if (ret) ++ return ret; ++ ++ return 0; ++} ++ ++/** ++ * panthor_gpu_block_power_off() - Power-off a specific block of the GPU ++ * @ptdev: Device. ++ * @blk_name: Block name. ++ * @pwroff_reg: Power-off register for this block. ++ * @pwrtrans_reg: Power transition register for this block. ++ * @mask: Sub-elements to power-off. ++ * @timeout_us: Timeout in microseconds. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_block_power_off(struct panthor_device *ptdev, ++ const char *blk_name, ++ u32 pwroff_reg, u32 pwrtrans_reg, ++ u64 mask, u32 timeout_us) ++{ ++ u32 val, i; ++ int ret; ++ ++ for (i = 0; i < 2; i++) { ++ u32 mask32 = mask >> (i * 32); ++ ++ if (!mask32) ++ continue; ++ ++ ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4), ++ val, !(mask32 & val), ++ 100, timeout_us); ++ if (ret) { ++ drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition", ++ blk_name, mask); ++ return ret; ++ } ++ } ++ ++ if (mask & GENMASK(31, 0)) ++ gpu_write(ptdev, pwroff_reg, mask); ++ ++ if (mask >> 32) ++ gpu_write(ptdev, pwroff_reg + 4, mask >> 32); ++ ++ for (i = 0; i < 2; i++) { ++ u32 mask32 = mask >> (i * 32); ++ ++ if (!mask32) ++ continue; ++ ++ ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4), ++ val, !(mask32 & val), ++ 100, timeout_us); ++ if (ret) { ++ drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition", ++ blk_name, mask); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_gpu_block_power_on() - Power-on a specific block of the GPU ++ * @ptdev: Device. ++ * @blk_name: Block name. ++ * @pwron_reg: Power-on register for this block. ++ * @pwrtrans_reg: Power transition register for this block. ++ * @rdy_reg: Power transition ready register. ++ * @mask: Sub-elements to power-on. ++ * @timeout_us: Timeout in microseconds. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_block_power_on(struct panthor_device *ptdev, ++ const char *blk_name, ++ u32 pwron_reg, u32 pwrtrans_reg, ++ u32 rdy_reg, u64 mask, u32 timeout_us) ++{ ++ u32 val, i; ++ int ret; ++ ++ for (i = 0; i < 2; i++) { ++ u32 mask32 = mask >> (i * 32); ++ ++ if (!mask32) ++ continue; ++ ++ ret = readl_relaxed_poll_timeout(ptdev->iomem + pwrtrans_reg + (i * 4), ++ val, !(mask32 & val), ++ 100, timeout_us); ++ if (ret) { ++ drm_err(&ptdev->base, "timeout waiting on %s:%llx power transition", ++ blk_name, mask); ++ return ret; ++ } ++ } ++ ++ if (mask & GENMASK(31, 0)) ++ gpu_write(ptdev, pwron_reg, mask); ++ ++ if (mask >> 32) ++ gpu_write(ptdev, pwron_reg + 4, mask >> 32); ++ ++ for (i = 0; i < 2; i++) { ++ u32 mask32 = mask >> (i * 32); ++ ++ if (!mask32) ++ continue; ++ ++ ret = readl_relaxed_poll_timeout(ptdev->iomem + rdy_reg + (i * 4), ++ val, (mask32 & val) == mask32, ++ 100, timeout_us); ++ if (ret) { ++ drm_err(&ptdev->base, "timeout waiting on %s:%llx readyness", ++ blk_name, mask); ++ return ret; ++ } ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_gpu_l2_power_on() - Power-on the L2-cache ++ * @ptdev: Device. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_l2_power_on(struct panthor_device *ptdev) ++{ ++ if (ptdev->gpu_info.l2_present != 1) { ++ /* ++ * Only support one core group now. ++ * ~(l2_present - 1) unsets all bits in l2_present except ++ * the bottom bit. (l2_present - 2) has all the bits in ++ * the first core group set. AND them together to generate ++ * a mask of cores in the first core group. ++ */ ++ u64 core_mask = ~(ptdev->gpu_info.l2_present - 1) & ++ (ptdev->gpu_info.l2_present - 2); ++ drm_info_once(&ptdev->base, "using only 1st core group (%lu cores from %lu)\n", ++ hweight64(core_mask), ++ hweight64(ptdev->gpu_info.shader_present)); ++ } ++ ++ return panthor_gpu_power_on(ptdev, L2, 1, 20000); ++} ++ ++/** ++ * panthor_gpu_flush_caches() - Flush caches ++ * @ptdev: Device. ++ * @l2: L2 flush type. ++ * @lsc: LSC flush type. ++ * @other: Other flush type. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_flush_caches(struct panthor_device *ptdev, ++ u32 l2, u32 lsc, u32 other) ++{ ++ bool timedout = false; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); ++ if (!drm_WARN_ON(&ptdev->base, ++ ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED)) { ++ ptdev->gpu->pending_reqs |= GPU_IRQ_CLEAN_CACHES_COMPLETED; ++ gpu_write(ptdev, GPU_CMD, GPU_FLUSH_CACHES(l2, lsc, other)); ++ } ++ spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); ++ ++ if (!wait_event_timeout(ptdev->gpu->reqs_acked, ++ !(ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED), ++ msecs_to_jiffies(100))) { ++ spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); ++ if ((ptdev->gpu->pending_reqs & GPU_IRQ_CLEAN_CACHES_COMPLETED) != 0 && ++ !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_CLEAN_CACHES_COMPLETED)) ++ timedout = true; ++ else ++ ptdev->gpu->pending_reqs &= ~GPU_IRQ_CLEAN_CACHES_COMPLETED; ++ spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); ++ } ++ ++ if (timedout) { ++ drm_err(&ptdev->base, "Flush caches timeout"); ++ return -ETIMEDOUT; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_gpu_soft_reset() - Issue a soft-reset ++ * @ptdev: Device. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_gpu_soft_reset(struct panthor_device *ptdev) ++{ ++ bool timedout = false; ++ unsigned long flags; ++ ++ spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); ++ if (!drm_WARN_ON(&ptdev->base, ++ ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED)) { ++ ptdev->gpu->pending_reqs |= GPU_IRQ_RESET_COMPLETED; ++ gpu_write(ptdev, GPU_INT_CLEAR, GPU_IRQ_RESET_COMPLETED); ++ gpu_write(ptdev, GPU_CMD, GPU_SOFT_RESET); ++ } ++ spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); ++ ++ if (!wait_event_timeout(ptdev->gpu->reqs_acked, ++ !(ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED), ++ msecs_to_jiffies(100))) { ++ spin_lock_irqsave(&ptdev->gpu->reqs_lock, flags); ++ if ((ptdev->gpu->pending_reqs & GPU_IRQ_RESET_COMPLETED) != 0 && ++ !(gpu_read(ptdev, GPU_INT_RAWSTAT) & GPU_IRQ_RESET_COMPLETED)) ++ timedout = true; ++ else ++ ptdev->gpu->pending_reqs &= ~GPU_IRQ_RESET_COMPLETED; ++ spin_unlock_irqrestore(&ptdev->gpu->reqs_lock, flags); ++ } ++ ++ if (timedout) { ++ drm_err(&ptdev->base, "Soft reset timeout"); ++ return -ETIMEDOUT; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_gpu_suspend() - Suspend the GPU block. ++ * @ptdev: Device. ++ * ++ * Suspend the GPU irq. This should be called last in the suspend procedure, ++ * after all other blocks have been suspented. ++ */ ++void panthor_gpu_suspend(struct panthor_device *ptdev) ++{ ++ /* ++ * It may be preferable to simply power down the L2, but for now just ++ * soft-reset which will leave the L2 powered down. ++ */ ++ panthor_gpu_soft_reset(ptdev); ++ panthor_gpu_irq_suspend(&ptdev->gpu->irq); ++} ++ ++/** ++ * panthor_gpu_resume() - Resume the GPU block. ++ * @ptdev: Device. ++ * ++ * Resume the IRQ handler and power-on the L2-cache. ++ * The FW takes care of powering the other blocks. ++ */ ++void panthor_gpu_resume(struct panthor_device *ptdev) ++{ ++ panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK); ++ panthor_gpu_l2_power_on(ptdev); ++} +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.h b/drivers/gpu/drm/panthor/panthor_gpu.h +new file mode 100644 +index 00000000000000..bba7555dd3c649 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_gpu.h +@@ -0,0 +1,52 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Collabora ltd. */ ++ ++#ifndef __PANTHOR_GPU_H__ ++#define __PANTHOR_GPU_H__ ++ ++struct panthor_device; ++ ++int panthor_gpu_init(struct panthor_device *ptdev); ++void panthor_gpu_unplug(struct panthor_device *ptdev); ++void panthor_gpu_suspend(struct panthor_device *ptdev); ++void panthor_gpu_resume(struct panthor_device *ptdev); ++ ++int panthor_gpu_block_power_on(struct panthor_device *ptdev, ++ const char *blk_name, ++ u32 pwron_reg, u32 pwrtrans_reg, ++ u32 rdy_reg, u64 mask, u32 timeout_us); ++int panthor_gpu_block_power_off(struct panthor_device *ptdev, ++ const char *blk_name, ++ u32 pwroff_reg, u32 pwrtrans_reg, ++ u64 mask, u32 timeout_us); ++ ++/** ++ * panthor_gpu_power_on() - Power on the GPU block. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++#define panthor_gpu_power_on(ptdev, type, mask, timeout_us) \ ++ panthor_gpu_block_power_on(ptdev, #type, \ ++ type ## _PWRON_LO, \ ++ type ## _PWRTRANS_LO, \ ++ type ## _READY_LO, \ ++ mask, timeout_us) ++ ++/** ++ * panthor_gpu_power_off() - Power off the GPU block. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++#define panthor_gpu_power_off(ptdev, type, mask, timeout_us) \ ++ panthor_gpu_block_power_off(ptdev, #type, \ ++ type ## _PWROFF_LO, \ ++ type ## _PWRTRANS_LO, \ ++ mask, timeout_us) ++ ++int panthor_gpu_l2_power_on(struct panthor_device *ptdev); ++int panthor_gpu_flush_caches(struct panthor_device *ptdev, ++ u32 l2, u32 lsc, u32 other); ++int panthor_gpu_soft_reset(struct panthor_device *ptdev); ++ ++#endif + +From 6febe9b31121e1e8b390b28a76600b62bf84a602 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:19 +0100 +Subject: [PATCH 084/150] drm/panthor: Add GEM logical block + +Anything relating to GEM object management is placed here. Nothing +particularly interesting here, given the implementation is based on +drm_gem_shmem_object, which is doing most of the work. + +v6: +- Add Maxime's and Heiko's acks +- Return a page-aligned BO size to userspace when creating a BO +- Keep header inclusion alphabetically ordered + +v5: +- Add Liviu's and Steve's R-b + +v4: +- Force kernel BOs to be GPU mapped +- Make panthor_kernel_bo_destroy() robust against ERR/NULL BO pointers + to simplify the call sites + +v3: +- Add acks for the MIT/GPL2 relicensing +- Provide a panthor_kernel_bo abstraction for buffer objects managed by + the kernel (will replace panthor_fw_mem and be used everywhere we were + using panthor_gem_create_and_map() before) +- Adjust things to match drm_gpuvm changes +- Change return of panthor_gem_create_with_handle() to int + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-6-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_gem.c | 230 ++++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_gem.h | 142 ++++++++++++++++ + 2 files changed, 372 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_gem.c + create mode 100644 drivers/gpu/drm/panthor/panthor_gem.h + +diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c +new file mode 100644 +index 00000000000000..d6483266d0c202 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_gem.c +@@ -0,0 +1,230 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "panthor_device.h" ++#include "panthor_gem.h" ++#include "panthor_mmu.h" ++ ++static void panthor_gem_free_object(struct drm_gem_object *obj) ++{ ++ struct panthor_gem_object *bo = to_panthor_bo(obj); ++ struct drm_gem_object *vm_root_gem = bo->exclusive_vm_root_gem; ++ ++ drm_gem_free_mmap_offset(&bo->base.base); ++ mutex_destroy(&bo->gpuva_list_lock); ++ drm_gem_shmem_free(&bo->base); ++ drm_gem_object_put(vm_root_gem); ++} ++ ++/** ++ * panthor_kernel_bo_destroy() - Destroy a kernel buffer object ++ * @vm: The VM this BO was mapped to. ++ * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction ++ * is skipped. ++ */ ++void panthor_kernel_bo_destroy(struct panthor_vm *vm, ++ struct panthor_kernel_bo *bo) ++{ ++ int ret; ++ ++ if (IS_ERR_OR_NULL(bo)) ++ return; ++ ++ panthor_kernel_bo_vunmap(bo); ++ ++ if (drm_WARN_ON(bo->obj->dev, ++ to_panthor_bo(bo->obj)->exclusive_vm_root_gem != panthor_vm_root_gem(vm))) ++ goto out_free_bo; ++ ++ ret = panthor_vm_unmap_range(vm, bo->va_node.start, ++ panthor_kernel_bo_size(bo)); ++ if (ret) ++ goto out_free_bo; ++ ++ panthor_vm_free_va(vm, &bo->va_node); ++ drm_gem_object_put(bo->obj); ++ ++out_free_bo: ++ kfree(bo); ++} ++ ++/** ++ * panthor_kernel_bo_create() - Create and map a GEM object to a VM ++ * @ptdev: Device. ++ * @vm: VM to map the GEM to. If NULL, the kernel object is not GPU mapped. ++ * @size: Size of the buffer object. ++ * @bo_flags: Combination of drm_panthor_bo_flags flags. ++ * @vm_map_flags: Combination of drm_panthor_vm_bind_op_flags (only those ++ * that are related to map operations). ++ * @gpu_va: GPU address assigned when mapping to the VM. ++ * If gpu_va == PANTHOR_VM_KERNEL_AUTO_VA, the virtual address will be ++ * automatically allocated. ++ * ++ * Return: A valid pointer in case of success, an ERR_PTR() otherwise. ++ */ ++struct panthor_kernel_bo * ++panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, ++ size_t size, u32 bo_flags, u32 vm_map_flags, ++ u64 gpu_va) ++{ ++ struct drm_gem_shmem_object *obj; ++ struct panthor_kernel_bo *kbo; ++ struct panthor_gem_object *bo; ++ int ret; ++ ++ if (drm_WARN_ON(&ptdev->base, !vm)) ++ return ERR_PTR(-EINVAL); ++ ++ kbo = kzalloc(sizeof(*kbo), GFP_KERNEL); ++ if (!kbo) ++ return ERR_PTR(-ENOMEM); ++ ++ obj = drm_gem_shmem_create(&ptdev->base, size); ++ if (IS_ERR(obj)) { ++ ret = PTR_ERR(obj); ++ goto err_free_bo; ++ } ++ ++ bo = to_panthor_bo(&obj->base); ++ size = obj->base.size; ++ kbo->obj = &obj->base; ++ bo->flags = bo_flags; ++ ++ ret = panthor_vm_alloc_va(vm, gpu_va, size, &kbo->va_node); ++ if (ret) ++ goto err_put_obj; ++ ++ ret = panthor_vm_map_bo_range(vm, bo, 0, size, kbo->va_node.start, vm_map_flags); ++ if (ret) ++ goto err_free_va; ++ ++ bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm); ++ drm_gem_object_get(bo->exclusive_vm_root_gem); ++ bo->base.base.resv = bo->exclusive_vm_root_gem->resv; ++ return kbo; ++ ++err_free_va: ++ panthor_vm_free_va(vm, &kbo->va_node); ++ ++err_put_obj: ++ drm_gem_object_put(&obj->base); ++ ++err_free_bo: ++ kfree(kbo); ++ return ERR_PTR(ret); ++} ++ ++static int panthor_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) ++{ ++ struct panthor_gem_object *bo = to_panthor_bo(obj); ++ ++ /* Don't allow mmap on objects that have the NO_MMAP flag set. */ ++ if (bo->flags & DRM_PANTHOR_BO_NO_MMAP) ++ return -EINVAL; ++ ++ return drm_gem_shmem_object_mmap(obj, vma); ++} ++ ++static struct dma_buf * ++panthor_gem_prime_export(struct drm_gem_object *obj, int flags) ++{ ++ /* We can't export GEMs that have an exclusive VM. */ ++ if (to_panthor_bo(obj)->exclusive_vm_root_gem) ++ return ERR_PTR(-EINVAL); ++ ++ return drm_gem_prime_export(obj, flags); ++} ++ ++static const struct drm_gem_object_funcs panthor_gem_funcs = { ++ .free = panthor_gem_free_object, ++ .print_info = drm_gem_shmem_object_print_info, ++ .pin = drm_gem_shmem_object_pin, ++ .unpin = drm_gem_shmem_object_unpin, ++ .get_sg_table = drm_gem_shmem_object_get_sg_table, ++ .vmap = drm_gem_shmem_object_vmap, ++ .vunmap = drm_gem_shmem_object_vunmap, ++ .mmap = panthor_gem_mmap, ++ .export = panthor_gem_prime_export, ++ .vm_ops = &drm_gem_shmem_vm_ops, ++}; ++ ++/** ++ * panthor_gem_create_object - Implementation of driver->gem_create_object. ++ * @ddev: DRM device ++ * @size: Size in bytes of the memory the object will reference ++ * ++ * This lets the GEM helpers allocate object structs for us, and keep ++ * our BO stats correct. ++ */ ++struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size) ++{ ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ struct panthor_gem_object *obj; ++ ++ obj = kzalloc(sizeof(*obj), GFP_KERNEL); ++ if (!obj) ++ return ERR_PTR(-ENOMEM); ++ ++ obj->base.base.funcs = &panthor_gem_funcs; ++ obj->base.map_wc = !ptdev->coherent; ++ mutex_init(&obj->gpuva_list_lock); ++ drm_gem_gpuva_set_lock(&obj->base.base, &obj->gpuva_list_lock); ++ ++ return &obj->base.base; ++} ++ ++/** ++ * panthor_gem_create_with_handle() - Create a GEM object and attach it to a handle. ++ * @file: DRM file. ++ * @ddev: DRM device. ++ * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. ++ * @size: Size of the GEM object to allocate. ++ * @flags: Combination of drm_panthor_bo_flags flags. ++ * @handle: Pointer holding the handle pointing to the new GEM object. ++ * ++ * Return: Zero on success ++ */ ++int ++panthor_gem_create_with_handle(struct drm_file *file, ++ struct drm_device *ddev, ++ struct panthor_vm *exclusive_vm, ++ u64 *size, u32 flags, u32 *handle) ++{ ++ int ret; ++ struct drm_gem_shmem_object *shmem; ++ struct panthor_gem_object *bo; ++ ++ shmem = drm_gem_shmem_create(ddev, *size); ++ if (IS_ERR(shmem)) ++ return PTR_ERR(shmem); ++ ++ bo = to_panthor_bo(&shmem->base); ++ bo->flags = flags; ++ ++ if (exclusive_vm) { ++ bo->exclusive_vm_root_gem = panthor_vm_root_gem(exclusive_vm); ++ drm_gem_object_get(bo->exclusive_vm_root_gem); ++ bo->base.base.resv = bo->exclusive_vm_root_gem->resv; ++ } ++ ++ /* ++ * Allocate an id of idr table where the obj is registered ++ * and handle has the id what user can see. ++ */ ++ ret = drm_gem_handle_create(file, &shmem->base, handle); ++ if (!ret) ++ *size = bo->base.base.size; ++ ++ /* drop reference from allocate - handle holds it now. */ ++ drm_gem_object_put(&shmem->base); ++ ++ return ret; ++} +diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h +new file mode 100644 +index 00000000000000..3bccba394d00a4 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_gem.h +@@ -0,0 +1,142 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_GEM_H__ ++#define __PANTHOR_GEM_H__ ++ ++#include ++#include ++ ++#include ++#include ++ ++struct panthor_vm; ++ ++/** ++ * struct panthor_gem_object - Driver specific GEM object. ++ */ ++struct panthor_gem_object { ++ /** @base: Inherit from drm_gem_shmem_object. */ ++ struct drm_gem_shmem_object base; ++ ++ /** ++ * @exclusive_vm_root_gem: Root GEM of the exclusive VM this GEM object ++ * is attached to. ++ * ++ * If @exclusive_vm_root_gem != NULL, any attempt to bind the GEM to a ++ * different VM will fail. ++ * ++ * All FW memory objects have this field set to the root GEM of the MCU ++ * VM. ++ */ ++ struct drm_gem_object *exclusive_vm_root_gem; ++ ++ /** ++ * @gpuva_list_lock: Custom GPUVA lock. ++ * ++ * Used to protect insertion of drm_gpuva elements to the ++ * drm_gem_object.gpuva.list list. ++ * ++ * We can't use the GEM resv for that, because drm_gpuva_link() is ++ * called in a dma-signaling path, where we're not allowed to take ++ * resv locks. ++ */ ++ struct mutex gpuva_list_lock; ++ ++ /** @flags: Combination of drm_panthor_bo_flags flags. */ ++ u32 flags; ++}; ++ ++/** ++ * struct panthor_kernel_bo - Kernel buffer object. ++ * ++ * These objects are only manipulated by the kernel driver and not ++ * directly exposed to the userspace. The GPU address of a kernel ++ * BO might be passed to userspace though. ++ */ ++struct panthor_kernel_bo { ++ /** ++ * @obj: The GEM object backing this kernel buffer object. ++ */ ++ struct drm_gem_object *obj; ++ ++ /** ++ * @va_node: VA space allocated to this GEM. ++ */ ++ struct drm_mm_node va_node; ++ ++ /** ++ * @kmap: Kernel CPU mapping of @gem. ++ */ ++ void *kmap; ++}; ++ ++static inline ++struct panthor_gem_object *to_panthor_bo(struct drm_gem_object *obj) ++{ ++ return container_of(to_drm_gem_shmem_obj(obj), struct panthor_gem_object, base); ++} ++ ++struct drm_gem_object *panthor_gem_create_object(struct drm_device *ddev, size_t size); ++ ++struct drm_gem_object * ++panthor_gem_prime_import_sg_table(struct drm_device *ddev, ++ struct dma_buf_attachment *attach, ++ struct sg_table *sgt); ++ ++int ++panthor_gem_create_with_handle(struct drm_file *file, ++ struct drm_device *ddev, ++ struct panthor_vm *exclusive_vm, ++ u64 *size, u32 flags, uint32_t *handle); ++ ++static inline u64 ++panthor_kernel_bo_gpuva(struct panthor_kernel_bo *bo) ++{ ++ return bo->va_node.start; ++} ++ ++static inline size_t ++panthor_kernel_bo_size(struct panthor_kernel_bo *bo) ++{ ++ return bo->obj->size; ++} ++ ++static inline int ++panthor_kernel_bo_vmap(struct panthor_kernel_bo *bo) ++{ ++ struct iosys_map map; ++ int ret; ++ ++ if (bo->kmap) ++ return 0; ++ ++ ret = drm_gem_vmap_unlocked(bo->obj, &map); ++ if (ret) ++ return ret; ++ ++ bo->kmap = map.vaddr; ++ return 0; ++} ++ ++static inline void ++panthor_kernel_bo_vunmap(struct panthor_kernel_bo *bo) ++{ ++ if (bo->kmap) { ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(bo->kmap); ++ ++ drm_gem_vunmap_unlocked(bo->obj, &map); ++ bo->kmap = NULL; ++ } ++} ++ ++struct panthor_kernel_bo * ++panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, ++ size_t size, u32 bo_flags, u32 vm_map_flags, ++ u64 gpu_va); ++ ++void panthor_kernel_bo_destroy(struct panthor_vm *vm, ++ struct panthor_kernel_bo *bo); ++ ++#endif /* __PANTHOR_GEM_H__ */ + +From 0a9ea3c13e2b044c4769b6ddfbb238eb6d203574 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:20 +0100 +Subject: [PATCH 085/150] drm/panthor: Add the devfreq logical block +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Every thing related to devfreq in placed in panthor_devfreq.c, and +helpers that can be called by other logical blocks are exposed through +panthor_devfreq.h. + +This implementation is loosely based on the panfrost implementation, +the only difference being that we don't count device users, because +the idle/active state will be managed by the scheduler logic. + +v6: +- Add Maxime's and Heiko's acks +- Keep header inclusion alphabetically ordered + +v4: +- Add Clément's A-b for the relicensing + +v3: +- Add acks for the MIT/GPL2 relicensing + +v2: +- Added in v2 + +Cc: Clément Péron # MIT+GPL2 relicensing +Reviewed-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Acked-by: Clément Péron # MIT+GPL2 relicensing +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-7-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_devfreq.c | 283 ++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_devfreq.h | 21 ++ + 2 files changed, 304 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_devfreq.c + create mode 100644 drivers/gpu/drm/panthor/panthor_devfreq.h + +diff --git a/drivers/gpu/drm/panthor/panthor_devfreq.c b/drivers/gpu/drm/panthor/panthor_devfreq.c +new file mode 100644 +index 00000000000000..7ac4fa290f2729 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_devfreq.c +@@ -0,0 +1,283 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2019 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include ++ ++#include "panthor_devfreq.h" ++#include "panthor_device.h" ++ ++/** ++ * struct panthor_devfreq - Device frequency management ++ */ ++struct panthor_devfreq { ++ /** @devfreq: devfreq device. */ ++ struct devfreq *devfreq; ++ ++ /** @gov_data: Governor data. */ ++ struct devfreq_simple_ondemand_data gov_data; ++ ++ /** @busy_time: Busy time. */ ++ ktime_t busy_time; ++ ++ /** @idle_time: Idle time. */ ++ ktime_t idle_time; ++ ++ /** @time_last_update: Last update time. */ ++ ktime_t time_last_update; ++ ++ /** @last_busy_state: True if the GPU was busy last time we updated the state. */ ++ bool last_busy_state; ++ ++ /* ++ * @lock: Lock used to protect busy_time, idle_time, time_last_update and ++ * last_busy_state. ++ * ++ * These fields can be accessed concurrently by panthor_devfreq_get_dev_status() ++ * and panthor_devfreq_record_{busy,idle}(). ++ */ ++ spinlock_t lock; ++}; ++ ++static void panthor_devfreq_update_utilization(struct panthor_devfreq *pdevfreq) ++{ ++ ktime_t now, last; ++ ++ now = ktime_get(); ++ last = pdevfreq->time_last_update; ++ ++ if (pdevfreq->last_busy_state) ++ pdevfreq->busy_time += ktime_sub(now, last); ++ else ++ pdevfreq->idle_time += ktime_sub(now, last); ++ ++ pdevfreq->time_last_update = now; ++} ++ ++static int panthor_devfreq_target(struct device *dev, unsigned long *freq, ++ u32 flags) ++{ ++ struct dev_pm_opp *opp; ++ ++ opp = devfreq_recommended_opp(dev, freq, flags); ++ if (IS_ERR(opp)) ++ return PTR_ERR(opp); ++ dev_pm_opp_put(opp); ++ ++ return dev_pm_opp_set_rate(dev, *freq); ++} ++ ++static void panthor_devfreq_reset(struct panthor_devfreq *pdevfreq) ++{ ++ pdevfreq->busy_time = 0; ++ pdevfreq->idle_time = 0; ++ pdevfreq->time_last_update = ktime_get(); ++} ++ ++static int panthor_devfreq_get_dev_status(struct device *dev, ++ struct devfreq_dev_status *status) ++{ ++ struct panthor_device *ptdev = dev_get_drvdata(dev); ++ struct panthor_devfreq *pdevfreq = ptdev->devfreq; ++ unsigned long irqflags; ++ ++ status->current_frequency = clk_get_rate(ptdev->clks.core); ++ ++ spin_lock_irqsave(&pdevfreq->lock, irqflags); ++ ++ panthor_devfreq_update_utilization(pdevfreq); ++ ++ status->total_time = ktime_to_ns(ktime_add(pdevfreq->busy_time, ++ pdevfreq->idle_time)); ++ ++ status->busy_time = ktime_to_ns(pdevfreq->busy_time); ++ ++ panthor_devfreq_reset(pdevfreq); ++ ++ spin_unlock_irqrestore(&pdevfreq->lock, irqflags); ++ ++ drm_dbg(&ptdev->base, "busy %lu total %lu %lu %% freq %lu MHz\n", ++ status->busy_time, status->total_time, ++ status->busy_time / (status->total_time / 100), ++ status->current_frequency / 1000 / 1000); ++ ++ return 0; ++} ++ ++static struct devfreq_dev_profile panthor_devfreq_profile = { ++ .timer = DEVFREQ_TIMER_DELAYED, ++ .polling_ms = 50, /* ~3 frames */ ++ .target = panthor_devfreq_target, ++ .get_dev_status = panthor_devfreq_get_dev_status, ++}; ++ ++int panthor_devfreq_init(struct panthor_device *ptdev) ++{ ++ /* There's actually 2 regulators (mali and sram), but the OPP core only ++ * supports one. ++ * ++ * We assume the sram regulator is coupled with the mali one and let ++ * the coupling logic deal with voltage updates. ++ */ ++ static const char * const reg_names[] = { "mali", NULL }; ++ struct thermal_cooling_device *cooling; ++ struct device *dev = ptdev->base.dev; ++ struct panthor_devfreq *pdevfreq; ++ struct dev_pm_opp *opp; ++ unsigned long cur_freq; ++ int ret; ++ ++ pdevfreq = drmm_kzalloc(&ptdev->base, sizeof(*ptdev->devfreq), GFP_KERNEL); ++ if (!pdevfreq) ++ return -ENOMEM; ++ ++ ptdev->devfreq = pdevfreq; ++ ++ ret = devm_pm_opp_set_regulators(dev, reg_names); ++ if (ret) { ++ if (ret != -EPROBE_DEFER) ++ DRM_DEV_ERROR(dev, "Couldn't set OPP regulators\n"); ++ ++ return ret; ++ } ++ ++ ret = devm_pm_opp_of_add_table(dev); ++ if (ret) ++ return ret; ++ ++ spin_lock_init(&pdevfreq->lock); ++ ++ panthor_devfreq_reset(pdevfreq); ++ ++ cur_freq = clk_get_rate(ptdev->clks.core); ++ ++ opp = devfreq_recommended_opp(dev, &cur_freq, 0); ++ if (IS_ERR(opp)) ++ return PTR_ERR(opp); ++ ++ panthor_devfreq_profile.initial_freq = cur_freq; ++ ++ /* Regulator coupling only takes care of synchronizing/balancing voltage ++ * updates, but the coupled regulator needs to be enabled manually. ++ * ++ * We use devm_regulator_get_enable_optional() and keep the sram supply ++ * enabled until the device is removed, just like we do for the mali ++ * supply, which is enabled when dev_pm_opp_set_opp(dev, opp) is called, ++ * and disabled when the opp_table is torn down, using the devm action. ++ * ++ * If we really care about disabling regulators on suspend, we should: ++ * - use devm_regulator_get_optional() here ++ * - call dev_pm_opp_set_opp(dev, NULL) before leaving this function ++ * (this disables the regulator passed to the OPP layer) ++ * - call dev_pm_opp_set_opp(dev, NULL) and ++ * regulator_disable(ptdev->regulators.sram) in ++ * panthor_devfreq_suspend() ++ * - call dev_pm_opp_set_opp(dev, default_opp) and ++ * regulator_enable(ptdev->regulators.sram) in ++ * panthor_devfreq_resume() ++ * ++ * But without knowing if it's beneficial or not (in term of power ++ * consumption), or how much it slows down the suspend/resume steps, ++ * let's just keep regulators enabled for the device lifetime. ++ */ ++ ret = devm_regulator_get_enable_optional(dev, "sram"); ++ if (ret && ret != -ENODEV) { ++ if (ret != -EPROBE_DEFER) ++ DRM_DEV_ERROR(dev, "Couldn't retrieve/enable sram supply\n"); ++ return ret; ++ } ++ ++ /* ++ * Set the recommend OPP this will enable and configure the regulator ++ * if any and will avoid a switch off by regulator_late_cleanup() ++ */ ++ ret = dev_pm_opp_set_opp(dev, opp); ++ if (ret) { ++ DRM_DEV_ERROR(dev, "Couldn't set recommended OPP\n"); ++ return ret; ++ } ++ ++ dev_pm_opp_put(opp); ++ ++ /* ++ * Setup default thresholds for the simple_ondemand governor. ++ * The values are chosen based on experiments. ++ */ ++ pdevfreq->gov_data.upthreshold = 45; ++ pdevfreq->gov_data.downdifferential = 5; ++ ++ pdevfreq->devfreq = devm_devfreq_add_device(dev, &panthor_devfreq_profile, ++ DEVFREQ_GOV_SIMPLE_ONDEMAND, ++ &pdevfreq->gov_data); ++ if (IS_ERR(pdevfreq->devfreq)) { ++ DRM_DEV_ERROR(dev, "Couldn't initialize GPU devfreq\n"); ++ ret = PTR_ERR(pdevfreq->devfreq); ++ pdevfreq->devfreq = NULL; ++ return ret; ++ } ++ ++ cooling = devfreq_cooling_em_register(pdevfreq->devfreq, NULL); ++ if (IS_ERR(cooling)) ++ DRM_DEV_INFO(dev, "Failed to register cooling device\n"); ++ ++ return 0; ++} ++ ++int panthor_devfreq_resume(struct panthor_device *ptdev) ++{ ++ struct panthor_devfreq *pdevfreq = ptdev->devfreq; ++ ++ if (!pdevfreq->devfreq) ++ return 0; ++ ++ panthor_devfreq_reset(pdevfreq); ++ ++ return devfreq_resume_device(pdevfreq->devfreq); ++} ++ ++int panthor_devfreq_suspend(struct panthor_device *ptdev) ++{ ++ struct panthor_devfreq *pdevfreq = ptdev->devfreq; ++ ++ if (!pdevfreq->devfreq) ++ return 0; ++ ++ return devfreq_suspend_device(pdevfreq->devfreq); ++} ++ ++void panthor_devfreq_record_busy(struct panthor_device *ptdev) ++{ ++ struct panthor_devfreq *pdevfreq = ptdev->devfreq; ++ unsigned long irqflags; ++ ++ if (!pdevfreq->devfreq) ++ return; ++ ++ spin_lock_irqsave(&pdevfreq->lock, irqflags); ++ ++ panthor_devfreq_update_utilization(pdevfreq); ++ pdevfreq->last_busy_state = true; ++ ++ spin_unlock_irqrestore(&pdevfreq->lock, irqflags); ++} ++ ++void panthor_devfreq_record_idle(struct panthor_device *ptdev) ++{ ++ struct panthor_devfreq *pdevfreq = ptdev->devfreq; ++ unsigned long irqflags; ++ ++ if (!pdevfreq->devfreq) ++ return; ++ ++ spin_lock_irqsave(&pdevfreq->lock, irqflags); ++ ++ panthor_devfreq_update_utilization(pdevfreq); ++ pdevfreq->last_busy_state = false; ++ ++ spin_unlock_irqrestore(&pdevfreq->lock, irqflags); ++} +diff --git a/drivers/gpu/drm/panthor/panthor_devfreq.h b/drivers/gpu/drm/panthor/panthor_devfreq.h +new file mode 100644 +index 00000000000000..83a5c952249347 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_devfreq.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2019 Collabora ltd. */ ++ ++#ifndef __PANTHOR_DEVFREQ_H__ ++#define __PANTHOR_DEVFREQ_H__ ++ ++struct devfreq; ++struct thermal_cooling_device; ++ ++struct panthor_device; ++struct panthor_devfreq; ++ ++int panthor_devfreq_init(struct panthor_device *ptdev); ++ ++int panthor_devfreq_resume(struct panthor_device *ptdev); ++int panthor_devfreq_suspend(struct panthor_device *ptdev); ++ ++void panthor_devfreq_record_busy(struct panthor_device *ptdev); ++void panthor_devfreq_record_idle(struct panthor_device *ptdev); ++ ++#endif /* __PANTHOR_DEVFREQ_H__ */ + +From 213f168ba8009b63ba982b06adcaf30633f678d4 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:21 +0100 +Subject: [PATCH 086/150] drm/panthor: Add the MMU/VM logical block + +MMU and VM management is related and placed in the same source file. + +Page table updates are delegated to the io-pgtable-arm driver that's in +the iommu subsystem. + +The VM management logic is based on drm_gpuva_mgr, and is assuming the +VA space is mostly managed by the usermode driver, except for a reserved +portion of this VA-space that's used for kernel objects (like the heap +contexts/chunks). + +Both asynchronous and synchronous VM operations are supported, and +internal helpers are exposed to allow other logical blocks to map their +buffers in the GPU VA space. + +There's one VM_BIND queue per-VM (meaning the Vulkan driver can only +expose one sparse-binding queue), and this bind queue is managed with +a 1:1 drm_sched_entity:drm_gpu_scheduler, such that each VM gets its own +independent execution queue, avoiding VM operation serialization at the +device level (things are still serialized at the VM level). + +The rest is just implementation details that are hopefully well explained +in the documentation. + +v6: +- Add Maxime's and Heiko's acks +- Add Steve's R-b +- Adjust the TRANSCFG value to account for SW VA space limitation on + 32-bit systems +- Keep header inclusion alphabetically ordered + +v5: +- Fix a double panthor_vm_cleanup_op_ctx() call +- Fix a race between panthor_vm_prepare_map_op_ctx() and + panthor_vm_bo_put() +- Fix panthor_vm_pool_destroy_vm() kernel doc +- Fix paddr adjustment in panthor_vm_map_pages() +- Fix bo_offset calculation in panthor_vm_get_bo_for_va() + +v4: +- Add an helper to return the VM state +- Check drmm_mutex_init() return code +- Remove the VM from the AS reclaim list when panthor_vm_active() is + called +- Count the number of active VM users instead of considering there's + at most one user (several scheduling groups can point to the same + vM) +- Pre-allocate a VMA object for unmap operations (unmaps can trigger + a sm_step_remap() call) +- Check vm->root_page_table instead of vm->pgtbl_ops to detect if + the io-pgtable is trying to allocate the root page table +- Don't memset() the va_node in panthor_vm_alloc_va(), make it a + caller requirement +- Fix the kernel doc in a few places +- Drop the panthor_vm::base offset constraint and modify + panthor_vm_put() to explicitly check for a NULL value +- Fix unbalanced vm_bo refcount in panthor_gpuva_sm_step_remap() +- Drop stale comments about the shared_bos list +- Patch mmu_features::va_bits on 32-bit builds to reflect the + io_pgtable limitation and let the UMD know about it + +v3: +- Add acks for the MIT/GPL2 relicensing +- Propagate MMU faults to the scheduler +- Move pages pinning/unpinning out of the dma_signalling path +- Fix 32-bit support +- Rework the user/kernel VA range calculation +- Make the auto-VA range explicit (auto-VA range doesn't cover the full + kernel-VA range on the MCU VM) +- Let callers of panthor_vm_alloc_va() allocate the drm_mm_node + (embedded in panthor_kernel_bo now) +- Adjust things to match the latest drm_gpuvm changes (extobj tracking, + resv prep and more) +- Drop the per-AS lock and use slots_lock (fixes a race on vm->as.id) +- Set as.id to -1 when reusing an address space from the LRU list +- Drop misleading comment about page faults +- Remove check for irq being assigned in panthor_mmu_unplug() + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-8-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_mmu.c | 2768 +++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_mmu.h | 102 + + 2 files changed, 2870 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.c + create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.h + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +new file mode 100644 +index 00000000000000..fdd35249169fdb +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -0,0 +1,2768 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "panthor_device.h" ++#include "panthor_gem.h" ++#include "panthor_heap.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++#include "panthor_sched.h" ++ ++#define MAX_AS_SLOTS 32 ++ ++struct panthor_vm; ++ ++/** ++ * struct panthor_as_slot - Address space slot ++ */ ++struct panthor_as_slot { ++ /** @vm: VM bound to this slot. NULL is no VM is bound. */ ++ struct panthor_vm *vm; ++}; ++ ++/** ++ * struct panthor_mmu - MMU related data ++ */ ++struct panthor_mmu { ++ /** @irq: The MMU irq. */ ++ struct panthor_irq irq; ++ ++ /** @as: Address space related fields. ++ * ++ * The GPU has a limited number of address spaces (AS) slots, forcing ++ * us to re-assign them to re-assign slots on-demand. ++ */ ++ struct { ++ /** @slots_lock: Lock protecting access to all other AS fields. */ ++ struct mutex slots_lock; ++ ++ /** @alloc_mask: Bitmask encoding the allocated slots. */ ++ unsigned long alloc_mask; ++ ++ /** @faulty_mask: Bitmask encoding the faulty slots. */ ++ unsigned long faulty_mask; ++ ++ /** @slots: VMs currently bound to the AS slots. */ ++ struct panthor_as_slot slots[MAX_AS_SLOTS]; ++ ++ /** ++ * @lru_list: List of least recently used VMs. ++ * ++ * We use this list to pick a VM to evict when all slots are ++ * used. ++ * ++ * There should be no more active VMs than there are AS slots, ++ * so this LRU is just here to keep VMs bound until there's ++ * a need to release a slot, thus avoid unnecessary TLB/cache ++ * flushes. ++ */ ++ struct list_head lru_list; ++ } as; ++ ++ /** @vm: VMs management fields */ ++ struct { ++ /** @lock: Lock protecting access to list. */ ++ struct mutex lock; ++ ++ /** @list: List containing all VMs. */ ++ struct list_head list; ++ ++ /** @reset_in_progress: True if a reset is in progress. */ ++ bool reset_in_progress; ++ ++ /** @wq: Workqueue used for the VM_BIND queues. */ ++ struct workqueue_struct *wq; ++ } vm; ++}; ++ ++/** ++ * struct panthor_vm_pool - VM pool object ++ */ ++struct panthor_vm_pool { ++ /** @xa: Array used for VM handle tracking. */ ++ struct xarray xa; ++}; ++ ++/** ++ * struct panthor_vma - GPU mapping object ++ * ++ * This is used to track GEM mappings in GPU space. ++ */ ++struct panthor_vma { ++ /** @base: Inherits from drm_gpuva. */ ++ struct drm_gpuva base; ++ ++ /** @node: Used to implement deferred release of VMAs. */ ++ struct list_head node; ++ ++ /** ++ * @flags: Combination of drm_panthor_vm_bind_op_flags. ++ * ++ * Only map related flags are accepted. ++ */ ++ u32 flags; ++}; ++ ++/** ++ * struct panthor_vm_op_ctx - VM operation context ++ * ++ * With VM operations potentially taking place in a dma-signaling path, we ++ * need to make sure everything that might require resource allocation is ++ * pre-allocated upfront. This is what this operation context is far. ++ * ++ * We also collect resources that have been freed, so we can release them ++ * asynchronously, and let the VM_BIND scheduler process the next VM_BIND ++ * request. ++ */ ++struct panthor_vm_op_ctx { ++ /** @rsvd_page_tables: Pages reserved for the MMU page table update. */ ++ struct { ++ /** @count: Number of pages reserved. */ ++ u32 count; ++ ++ /** @ptr: Point to the first unused page in the @pages table. */ ++ u32 ptr; ++ ++ /** ++ * @page: Array of pages that can be used for an MMU page table update. ++ * ++ * After an VM operation, there might be free pages left in this array. ++ * They should be returned to the pt_cache as part of the op_ctx cleanup. ++ */ ++ void **pages; ++ } rsvd_page_tables; ++ ++ /** ++ * @preallocated_vmas: Pre-allocated VMAs to handle the remap case. ++ * ++ * Partial unmap requests or map requests overlapping existing mappings will ++ * trigger a remap call, which need to register up to three panthor_vma objects ++ * (one for the new mapping, and two for the previous and next mappings). ++ */ ++ struct panthor_vma *preallocated_vmas[3]; ++ ++ /** @flags: Combination of drm_panthor_vm_bind_op_flags. */ ++ u32 flags; ++ ++ /** @va: Virtual range targeted by the VM operation. */ ++ struct { ++ /** @addr: Start address. */ ++ u64 addr; ++ ++ /** @range: Range size. */ ++ u64 range; ++ } va; ++ ++ /** ++ * @returned_vmas: List of panthor_vma objects returned after a VM operation. ++ * ++ * For unmap operations, this will contain all VMAs that were covered by the ++ * specified VA range. ++ * ++ * For map operations, this will contain all VMAs that previously mapped to ++ * the specified VA range. ++ * ++ * Those VMAs, and the resources they point to will be released as part of ++ * the op_ctx cleanup operation. ++ */ ++ struct list_head returned_vmas; ++ ++ /** @map: Fields specific to a map operation. */ ++ struct { ++ /** @vm_bo: Buffer object to map. */ ++ struct drm_gpuvm_bo *vm_bo; ++ ++ /** @bo_offset: Offset in the buffer object. */ ++ u64 bo_offset; ++ ++ /** ++ * @sgt: sg-table pointing to pages backing the GEM object. ++ * ++ * This is gathered at job creation time, such that we don't have ++ * to allocate in ::run_job(). ++ */ ++ struct sg_table *sgt; ++ ++ /** ++ * @new_vma: The new VMA object that will be inserted to the VA tree. ++ */ ++ struct panthor_vma *new_vma; ++ } map; ++}; ++ ++/** ++ * struct panthor_vm - VM object ++ * ++ * A VM is an object representing a GPU (or MCU) virtual address space. ++ * It embeds the MMU page table for this address space, a tree containing ++ * all the virtual mappings of GEM objects, and other things needed to manage ++ * the VM. ++ * ++ * Except for the MCU VM, which is managed by the kernel, all other VMs are ++ * created by userspace and mostly managed by userspace, using the ++ * %DRM_IOCTL_PANTHOR_VM_BIND ioctl. ++ * ++ * A portion of the virtual address space is reserved for kernel objects, ++ * like heap chunks, and userspace gets to decide how much of the virtual ++ * address space is left to the kernel (half of the virtual address space ++ * by default). ++ */ ++struct panthor_vm { ++ /** ++ * @base: Inherit from drm_gpuvm. ++ * ++ * We delegate all the VA management to the common drm_gpuvm framework ++ * and only implement hooks to update the MMU page table. ++ */ ++ struct drm_gpuvm base; ++ ++ /** ++ * @sched: Scheduler used for asynchronous VM_BIND request. ++ * ++ * We use a 1:1 scheduler here. ++ */ ++ struct drm_gpu_scheduler sched; ++ ++ /** ++ * @entity: Scheduling entity representing the VM_BIND queue. ++ * ++ * There's currently one bind queue per VM. It doesn't make sense to ++ * allow more given the VM operations are serialized anyway. ++ */ ++ struct drm_sched_entity entity; ++ ++ /** @ptdev: Device. */ ++ struct panthor_device *ptdev; ++ ++ /** @memattr: Value to program to the AS_MEMATTR register. */ ++ u64 memattr; ++ ++ /** @pgtbl_ops: Page table operations. */ ++ struct io_pgtable_ops *pgtbl_ops; ++ ++ /** @root_page_table: Stores the root page table pointer. */ ++ void *root_page_table; ++ ++ /** ++ * @op_lock: Lock used to serialize operations on a VM. ++ * ++ * The serialization of jobs queued to the VM_BIND queue is already ++ * taken care of by drm_sched, but we need to serialize synchronous ++ * and asynchronous VM_BIND request. This is what this lock is for. ++ */ ++ struct mutex op_lock; ++ ++ /** ++ * @op_ctx: The context attached to the currently executing VM operation. ++ * ++ * NULL when no operation is in progress. ++ */ ++ struct panthor_vm_op_ctx *op_ctx; ++ ++ /** ++ * @mm: Memory management object representing the auto-VA/kernel-VA. ++ * ++ * Used to auto-allocate VA space for kernel-managed objects (tiler ++ * heaps, ...). ++ * ++ * For the MCU VM, this is managing the VA range that's used to map ++ * all shared interfaces. ++ * ++ * For user VMs, the range is specified by userspace, and must not ++ * exceed half of the VA space addressable. ++ */ ++ struct drm_mm mm; ++ ++ /** @mm_lock: Lock protecting the @mm field. */ ++ struct mutex mm_lock; ++ ++ /** @kernel_auto_va: Automatic VA-range for kernel BOs. */ ++ struct { ++ /** @start: Start of the automatic VA-range for kernel BOs. */ ++ u64 start; ++ ++ /** @size: Size of the automatic VA-range for kernel BOs. */ ++ u64 end; ++ } kernel_auto_va; ++ ++ /** @as: Address space related fields. */ ++ struct { ++ /** ++ * @id: ID of the address space this VM is bound to. ++ * ++ * A value of -1 means the VM is inactive/not bound. ++ */ ++ int id; ++ ++ /** @active_cnt: Number of active users of this VM. */ ++ refcount_t active_cnt; ++ ++ /** ++ * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list. ++ * ++ * Active VMs should not be inserted in the LRU list. ++ */ ++ struct list_head lru_node; ++ } as; ++ ++ /** ++ * @heaps: Tiler heap related fields. ++ */ ++ struct { ++ /** ++ * @pool: The heap pool attached to this VM. ++ * ++ * Will stay NULL until someone creates a heap context on this VM. ++ */ ++ struct panthor_heap_pool *pool; ++ ++ /** @lock: Lock used to protect access to @pool. */ ++ struct mutex lock; ++ } heaps; ++ ++ /** @node: Used to insert the VM in the panthor_mmu::vm::list. */ ++ struct list_head node; ++ ++ /** @for_mcu: True if this is the MCU VM. */ ++ bool for_mcu; ++ ++ /** ++ * @destroyed: True if the VM was destroyed. ++ * ++ * No further bind requests should be queued to a destroyed VM. ++ */ ++ bool destroyed; ++ ++ /** ++ * @unusable: True if the VM has turned unusable because something ++ * bad happened during an asynchronous request. ++ * ++ * We don't try to recover from such failures, because this implies ++ * informing userspace about the specific operation that failed, and ++ * hoping the userspace driver can replay things from there. This all ++ * sounds very complicated for little gain. ++ * ++ * Instead, we should just flag the VM as unusable, and fail any ++ * further request targeting this VM. ++ * ++ * We also provide a way to query a VM state, so userspace can destroy ++ * it and create a new one. ++ * ++ * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST ++ * situation, where the logical device needs to be re-created. ++ */ ++ bool unusable; ++ ++ /** ++ * @unhandled_fault: Unhandled fault happened. ++ * ++ * This should be reported to the scheduler, and the queue/group be ++ * flagged as faulty as a result. ++ */ ++ bool unhandled_fault; ++}; ++ ++/** ++ * struct panthor_vm_bind_job - VM bind job ++ */ ++struct panthor_vm_bind_job { ++ /** @base: Inherit from drm_sched_job. */ ++ struct drm_sched_job base; ++ ++ /** @refcount: Reference count. */ ++ struct kref refcount; ++ ++ /** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */ ++ struct work_struct cleanup_op_ctx_work; ++ ++ /** @vm: VM targeted by the VM operation. */ ++ struct panthor_vm *vm; ++ ++ /** @ctx: Operation context. */ ++ struct panthor_vm_op_ctx ctx; ++}; ++ ++/** ++ * @pt_cache: Cache used to allocate MMU page tables. ++ * ++ * The pre-allocation pattern forces us to over-allocate to plan for ++ * the worst case scenario, and return the pages we didn't use. ++ * ++ * Having a kmem_cache allows us to speed allocations. ++ */ ++static struct kmem_cache *pt_cache; ++ ++/** ++ * alloc_pt() - Custom page table allocator ++ * @cookie: Cookie passed at page table allocation time. ++ * @size: Size of the page table. This size should be fixed, ++ * and determined at creation time based on the granule size. ++ * @gfp: GFP flags. ++ * ++ * We want a custom allocator so we can use a cache for page table ++ * allocations and amortize the cost of the over-reservation that's ++ * done to allow asynchronous VM operations. ++ * ++ * Return: non-NULL on success, NULL if the allocation failed for any ++ * reason. ++ */ ++static void *alloc_pt(void *cookie, size_t size, gfp_t gfp) ++{ ++ struct panthor_vm *vm = cookie; ++ void *page; ++ ++ /* Allocation of the root page table happening during init. */ ++ if (unlikely(!vm->root_page_table)) { ++ struct page *p; ++ ++ drm_WARN_ON(&vm->ptdev->base, vm->op_ctx); ++ p = alloc_pages_node(dev_to_node(vm->ptdev->base.dev), ++ gfp | __GFP_ZERO, get_order(size)); ++ page = p ? page_address(p) : NULL; ++ vm->root_page_table = page; ++ return page; ++ } ++ ++ /* We're not supposed to have anything bigger than 4k here, because we picked a ++ * 4k granule size at init time. ++ */ ++ if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K)) ++ return NULL; ++ ++ /* We must have some op_ctx attached to the VM and it must have at least one ++ * free page. ++ */ ++ if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) || ++ drm_WARN_ON(&vm->ptdev->base, ++ vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count)) ++ return NULL; ++ ++ page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++]; ++ memset(page, 0, SZ_4K); ++ ++ /* Page table entries don't use virtual addresses, which trips out ++ * kmemleak. kmemleak_alloc_phys() might work, but physical addresses ++ * are mixed with other fields, and I fear kmemleak won't detect that ++ * either. ++ * ++ * Let's just ignore memory passed to the page-table driver for now. ++ */ ++ kmemleak_ignore(page); ++ return page; ++} ++ ++/** ++ * @free_pt() - Custom page table free function ++ * @cookie: Cookie passed at page table allocation time. ++ * @data: Page table to free. ++ * @size: Size of the page table. This size should be fixed, ++ * and determined at creation time based on the granule size. ++ */ ++static void free_pt(void *cookie, void *data, size_t size) ++{ ++ struct panthor_vm *vm = cookie; ++ ++ if (unlikely(vm->root_page_table == data)) { ++ free_pages((unsigned long)data, get_order(size)); ++ vm->root_page_table = NULL; ++ return; ++ } ++ ++ if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K)) ++ return; ++ ++ /* Return the page to the pt_cache. */ ++ kmem_cache_free(pt_cache, data); ++} ++ ++static int wait_ready(struct panthor_device *ptdev, u32 as_nr) ++{ ++ int ret; ++ u32 val; ++ ++ /* Wait for the MMU status to indicate there is no active command, in ++ * case one is pending. ++ */ ++ ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr), ++ val, !(val & AS_STATUS_AS_ACTIVE), ++ 10, 100000); ++ ++ if (ret) { ++ panthor_device_schedule_reset(ptdev); ++ drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n"); ++ } ++ ++ return ret; ++} ++ ++static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd) ++{ ++ int status; ++ ++ /* write AS_COMMAND when MMU is ready to accept another command */ ++ status = wait_ready(ptdev, as_nr); ++ if (!status) ++ gpu_write(ptdev, AS_COMMAND(as_nr), cmd); ++ ++ return status; ++} ++ ++static void lock_region(struct panthor_device *ptdev, u32 as_nr, ++ u64 region_start, u64 size) ++{ ++ u8 region_width; ++ u64 region; ++ u64 region_end = region_start + size; ++ ++ if (!size) ++ return; ++ ++ /* ++ * The locked region is a naturally aligned power of 2 block encoded as ++ * log2 minus(1). ++ * Calculate the desired start/end and look for the highest bit which ++ * differs. The smallest naturally aligned block must include this bit ++ * change, the desired region starts with this bit (and subsequent bits) ++ * zeroed and ends with the bit (and subsequent bits) set to one. ++ */ ++ region_width = max(fls64(region_start ^ (region_end - 1)), ++ const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1; ++ ++ /* ++ * Mask off the low bits of region_start (which would be ignored by ++ * the hardware anyway) ++ */ ++ region_start &= GENMASK_ULL(63, region_width); ++ ++ region = region_width | region_start; ++ ++ /* Lock the region that needs to be updated */ ++ gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region)); ++ gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region)); ++ write_cmd(ptdev, as_nr, AS_COMMAND_LOCK); ++} ++ ++static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, ++ u64 iova, u64 size, u32 op) ++{ ++ lockdep_assert_held(&ptdev->mmu->as.slots_lock); ++ ++ if (as_nr < 0) ++ return 0; ++ ++ if (op != AS_COMMAND_UNLOCK) ++ lock_region(ptdev, as_nr, iova, size); ++ ++ /* Run the MMU operation */ ++ write_cmd(ptdev, as_nr, op); ++ ++ /* Wait for the flush to complete */ ++ return wait_ready(ptdev, as_nr); ++} ++ ++static int mmu_hw_do_operation(struct panthor_vm *vm, ++ u64 iova, u64 size, u32 op) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ int ret; ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op); ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ return ret; ++} ++ ++static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr, ++ u64 transtab, u64 transcfg, u64 memattr) ++{ ++ int ret; ++ ++ ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM); ++ if (ret) ++ return ret; ++ ++ gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab)); ++ gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab)); ++ ++ gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr)); ++ gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr)); ++ ++ gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg)); ++ gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg)); ++ ++ return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); ++} ++ ++static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr) ++{ ++ int ret; ++ ++ ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM); ++ if (ret) ++ return ret; ++ ++ gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0); ++ gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0); ++ ++ gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0); ++ gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0); ++ ++ gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED); ++ gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0); ++ ++ return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE); ++} ++ ++static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value) ++{ ++ /* Bits 16 to 31 mean REQ_COMPLETE. */ ++ return value & GENMASK(15, 0); ++} ++ ++static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as) ++{ ++ return BIT(as); ++} ++ ++/** ++ * panthor_vm_has_unhandled_faults() - Check if a VM has unhandled faults ++ * @vm: VM to check. ++ * ++ * Return: true if the VM has unhandled faults, false otherwise. ++ */ ++bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm) ++{ ++ return vm->unhandled_fault; ++} ++ ++/** ++ * panthor_vm_is_unusable() - Check if the VM is still usable ++ * @vm: VM to check. ++ * ++ * Return: true if the VM is unusable, false otherwise. ++ */ ++bool panthor_vm_is_unusable(struct panthor_vm *vm) ++{ ++ return vm->unusable; ++} ++ ++static void panthor_vm_release_as_locked(struct panthor_vm *vm) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ ++ lockdep_assert_held(&ptdev->mmu->as.slots_lock); ++ ++ if (drm_WARN_ON(&ptdev->base, vm->as.id < 0)) ++ return; ++ ++ ptdev->mmu->as.slots[vm->as.id].vm = NULL; ++ clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask); ++ refcount_set(&vm->as.active_cnt, 0); ++ list_del_init(&vm->as.lru_node); ++ vm->as.id = -1; ++} ++ ++/** ++ * panthor_vm_active() - Flag a VM as active ++ * @VM: VM to flag as active. ++ * ++ * Assigns an address space to a VM so it can be used by the GPU/MCU. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_active(struct panthor_vm *vm) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features); ++ struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg; ++ int ret = 0, as, cookie; ++ u64 transtab, transcfg; ++ ++ if (!drm_dev_enter(&ptdev->base, &cookie)) ++ return -ENODEV; ++ ++ if (refcount_inc_not_zero(&vm->as.active_cnt)) ++ goto out_dev_exit; ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ ++ if (refcount_inc_not_zero(&vm->as.active_cnt)) ++ goto out_unlock; ++ ++ as = vm->as.id; ++ if (as >= 0) { ++ /* Unhandled pagefault on this AS, the MMU was disabled. We need to ++ * re-enable the MMU after clearing+unmasking the AS interrupts. ++ */ ++ if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) ++ goto out_enable_as; ++ ++ goto out_make_active; ++ } ++ ++ /* Check for a free AS */ ++ if (vm->for_mcu) { ++ drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0)); ++ as = 0; ++ } else { ++ as = ffz(ptdev->mmu->as.alloc_mask | BIT(0)); ++ } ++ ++ if (!(BIT(as) & ptdev->gpu_info.as_present)) { ++ struct panthor_vm *lru_vm; ++ ++ lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list, ++ struct panthor_vm, ++ as.lru_node); ++ if (drm_WARN_ON(&ptdev->base, !lru_vm)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } ++ ++ drm_WARN_ON(&ptdev->base, refcount_read(&lru_vm->as.active_cnt)); ++ as = lru_vm->as.id; ++ panthor_vm_release_as_locked(lru_vm); ++ } ++ ++ /* Assign the free or reclaimed AS to the FD */ ++ vm->as.id = as; ++ set_bit(as, &ptdev->mmu->as.alloc_mask); ++ ptdev->mmu->as.slots[as].vm = vm; ++ ++out_enable_as: ++ transtab = cfg->arm_lpae_s1_cfg.ttbr; ++ transcfg = AS_TRANSCFG_PTW_MEMATTR_WB | ++ AS_TRANSCFG_PTW_RA | ++ AS_TRANSCFG_ADRMODE_AARCH64_4K | ++ AS_TRANSCFG_INA_BITS(55 - va_bits); ++ if (ptdev->coherent) ++ transcfg |= AS_TRANSCFG_PTW_SH_OS; ++ ++ /* If the VM is re-activated, we clear the fault. */ ++ vm->unhandled_fault = false; ++ ++ /* Unhandled pagefault on this AS, clear the fault and re-enable interrupts ++ * before enabling the AS. ++ */ ++ if (ptdev->mmu->as.faulty_mask & panthor_mmu_as_fault_mask(ptdev, as)) { ++ gpu_write(ptdev, MMU_INT_CLEAR, panthor_mmu_as_fault_mask(ptdev, as)); ++ ptdev->mmu->as.faulty_mask &= ~panthor_mmu_as_fault_mask(ptdev, as); ++ gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask); ++ } ++ ++ ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr); ++ ++out_make_active: ++ if (!ret) { ++ refcount_set(&vm->as.active_cnt, 1); ++ list_del_init(&vm->as.lru_node); ++ } ++ ++out_unlock: ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++out_dev_exit: ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++/** ++ * panthor_vm_idle() - Flag a VM idle ++ * @VM: VM to flag as idle. ++ * ++ * When we know the GPU is done with the VM (no more jobs to process), ++ * we can relinquish the AS slot attached to this VM, if any. ++ * ++ * We don't release the slot immediately, but instead place the VM in ++ * the LRU list, so it can be evicted if another VM needs an AS slot. ++ * This way, VMs keep attached to the AS they were given until we run ++ * out of free slot, limiting the number of MMU operations (TLB flush ++ * and other AS updates). ++ */ ++void panthor_vm_idle(struct panthor_vm *vm) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ ++ if (!refcount_dec_and_mutex_lock(&vm->as.active_cnt, &ptdev->mmu->as.slots_lock)) ++ return; ++ ++ if (!drm_WARN_ON(&ptdev->base, vm->as.id == -1 || !list_empty(&vm->as.lru_node))) ++ list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list); ++ ++ refcount_set(&vm->as.active_cnt, 0); ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++} ++ ++static void panthor_vm_stop(struct panthor_vm *vm) ++{ ++ drm_sched_stop(&vm->sched, NULL); ++} ++ ++static void panthor_vm_start(struct panthor_vm *vm) ++{ ++ drm_sched_start(&vm->sched, true); ++} ++ ++/** ++ * panthor_vm_as() - Get the AS slot attached to a VM ++ * @vm: VM to get the AS slot of. ++ * ++ * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise. ++ */ ++int panthor_vm_as(struct panthor_vm *vm) ++{ ++ return vm->as.id; ++} ++ ++static size_t get_pgsize(u64 addr, size_t size, size_t *count) ++{ ++ /* ++ * io-pgtable only operates on multiple pages within a single table ++ * entry, so we need to split at boundaries of the table size, i.e. ++ * the next block size up. The distance from address A to the next ++ * boundary of block size B is logically B - A % B, but in unsigned ++ * two's complement where B is a power of two we get the equivalence ++ * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :) ++ */ ++ size_t blk_offset = -addr % SZ_2M; ++ ++ if (blk_offset || size < SZ_2M) { ++ *count = min_not_zero(blk_offset, size) / SZ_4K; ++ return SZ_4K; ++ } ++ blk_offset = -addr % SZ_1G ?: SZ_1G; ++ *count = min(blk_offset, size) / SZ_2M; ++ return SZ_2M; ++} ++ ++static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ int ret = 0, cookie; ++ ++ if (vm->as.id < 0) ++ return 0; ++ ++ /* If the device is unplugged, we just silently skip the flush. */ ++ if (!drm_dev_enter(&ptdev->base, &cookie)) ++ return 0; ++ ++ /* Flush the PTs only if we're already awake */ ++ if (pm_runtime_active(ptdev->base.dev)) ++ ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT); ++ ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ struct io_pgtable_ops *ops = vm->pgtbl_ops; ++ u64 offset = 0; ++ ++ drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%llx", vm->as.id, iova, size); ++ ++ while (offset < size) { ++ size_t unmapped_sz = 0, pgcount; ++ size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount); ++ ++ unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL); ++ ++ if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) { ++ drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n", ++ iova + offset + unmapped_sz, ++ iova + offset + pgsize * pgcount, ++ iova, iova + size); ++ panthor_vm_flush_range(vm, iova, offset + unmapped_sz); ++ return -EINVAL; ++ } ++ offset += unmapped_sz; ++ } ++ ++ return panthor_vm_flush_range(vm, iova, size); ++} ++ ++static int ++panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot, ++ struct sg_table *sgt, u64 offset, u64 size) ++{ ++ struct panthor_device *ptdev = vm->ptdev; ++ unsigned int count; ++ struct scatterlist *sgl; ++ struct io_pgtable_ops *ops = vm->pgtbl_ops; ++ u64 start_iova = iova; ++ int ret; ++ ++ if (!size) ++ return 0; ++ ++ for_each_sgtable_dma_sg(sgt, sgl, count) { ++ dma_addr_t paddr = sg_dma_address(sgl); ++ size_t len = sg_dma_len(sgl); ++ ++ if (len <= offset) { ++ offset -= len; ++ continue; ++ } ++ ++ paddr += offset; ++ len -= offset; ++ len = min_t(size_t, len, size); ++ size -= len; ++ ++ drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%pad, len=%zx", ++ vm->as.id, iova, &paddr, len); ++ ++ while (len) { ++ size_t pgcount, mapped = 0; ++ size_t pgsize = get_pgsize(iova | paddr, len, &pgcount); ++ ++ ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, ++ GFP_KERNEL, &mapped); ++ iova += mapped; ++ paddr += mapped; ++ len -= mapped; ++ ++ if (drm_WARN_ON(&ptdev->base, !ret && !mapped)) ++ ret = -ENOMEM; ++ ++ if (ret) { ++ /* If something failed, unmap what we've already mapped before ++ * returning. The unmap call is not supposed to fail. ++ */ ++ drm_WARN_ON(&ptdev->base, ++ panthor_vm_unmap_pages(vm, start_iova, ++ iova - start_iova)); ++ return ret; ++ } ++ } ++ ++ if (!size) ++ break; ++ } ++ ++ return panthor_vm_flush_range(vm, start_iova, iova - start_iova); ++} ++ ++static int flags_to_prot(u32 flags) ++{ ++ int prot = 0; ++ ++ if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC) ++ prot |= IOMMU_NOEXEC; ++ ++ if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)) ++ prot |= IOMMU_CACHE; ++ ++ if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY) ++ prot |= IOMMU_READ; ++ else ++ prot |= IOMMU_READ | IOMMU_WRITE; ++ ++ return prot; ++} ++ ++/** ++ * panthor_vm_alloc_va() - Allocate a region in the auto-va space ++ * @VM: VM to allocate a region on. ++ * @va: start of the VA range. Can be PANTHOR_VM_KERNEL_AUTO_VA if the user ++ * wants the VA to be automatically allocated from the auto-VA range. ++ * @size: size of the VA range. ++ * @va_node: drm_mm_node to initialize. Must be zero-initialized. ++ * ++ * Some GPU objects, like heap chunks, are fully managed by the kernel and ++ * need to be mapped to the userspace VM, in the region reserved for kernel ++ * objects. ++ * ++ * This function takes care of allocating a region in the kernel auto-VA space. ++ * ++ * Return: 0 on success, an error code otherwise. ++ */ ++int ++panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size, ++ struct drm_mm_node *va_node) ++{ ++ int ret; ++ ++ if (!size || (size & ~PAGE_MASK)) ++ return -EINVAL; ++ ++ if (va != PANTHOR_VM_KERNEL_AUTO_VA && (va & ~PAGE_MASK)) ++ return -EINVAL; ++ ++ mutex_lock(&vm->mm_lock); ++ if (va != PANTHOR_VM_KERNEL_AUTO_VA) { ++ va_node->start = va; ++ va_node->size = size; ++ ret = drm_mm_reserve_node(&vm->mm, va_node); ++ } else { ++ ret = drm_mm_insert_node_in_range(&vm->mm, va_node, size, ++ size >= SZ_2M ? SZ_2M : SZ_4K, ++ 0, vm->kernel_auto_va.start, ++ vm->kernel_auto_va.end, ++ DRM_MM_INSERT_BEST); ++ } ++ mutex_unlock(&vm->mm_lock); ++ ++ return ret; ++} ++ ++/** ++ * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va() ++ * @VM: VM to free the region on. ++ * @va_node: Memory node representing the region to free. ++ */ ++void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node) ++{ ++ mutex_lock(&vm->mm_lock); ++ drm_mm_remove_node(va_node); ++ mutex_unlock(&vm->mm_lock); ++} ++ ++static void panthor_vm_bo_put(struct drm_gpuvm_bo *vm_bo) ++{ ++ struct panthor_gem_object *bo = to_panthor_bo(vm_bo->obj); ++ struct drm_gpuvm *vm = vm_bo->vm; ++ bool unpin; ++ ++ /* We must retain the GEM before calling drm_gpuvm_bo_put(), ++ * otherwise the mutex might be destroyed while we hold it. ++ * Same goes for the VM, since we take the VM resv lock. ++ */ ++ drm_gem_object_get(&bo->base.base); ++ drm_gpuvm_get(vm); ++ ++ /* We take the resv lock to protect against concurrent accesses to the ++ * gpuvm evicted/extobj lists that are modified in ++ * drm_gpuvm_bo_destroy(), which is called if drm_gpuvm_bo_put() ++ * releases sthe last vm_bo reference. ++ * We take the BO GPUVA list lock to protect the vm_bo removal from the ++ * GEM vm_bo list. ++ */ ++ dma_resv_lock(drm_gpuvm_resv(vm), NULL); ++ mutex_lock(&bo->gpuva_list_lock); ++ unpin = drm_gpuvm_bo_put(vm_bo); ++ mutex_unlock(&bo->gpuva_list_lock); ++ dma_resv_unlock(drm_gpuvm_resv(vm)); ++ ++ /* If the vm_bo object was destroyed, release the pin reference that ++ * was hold by this object. ++ */ ++ if (unpin && !bo->base.base.import_attach) ++ drm_gem_shmem_unpin(&bo->base); ++ ++ drm_gpuvm_put(vm); ++ drm_gem_object_put(&bo->base.base); ++} ++ ++static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx, ++ struct panthor_vm *vm) ++{ ++ struct panthor_vma *vma, *tmp_vma; ++ ++ u32 remaining_pt_count = op_ctx->rsvd_page_tables.count - ++ op_ctx->rsvd_page_tables.ptr; ++ ++ if (remaining_pt_count) { ++ kmem_cache_free_bulk(pt_cache, remaining_pt_count, ++ op_ctx->rsvd_page_tables.pages + ++ op_ctx->rsvd_page_tables.ptr); ++ } ++ ++ kfree(op_ctx->rsvd_page_tables.pages); ++ ++ if (op_ctx->map.vm_bo) ++ panthor_vm_bo_put(op_ctx->map.vm_bo); ++ ++ for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) ++ kfree(op_ctx->preallocated_vmas[i]); ++ ++ list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) { ++ list_del(&vma->node); ++ panthor_vm_bo_put(vma->base.vm_bo); ++ kfree(vma); ++ } ++} ++ ++static struct panthor_vma * ++panthor_vm_op_ctx_get_vma(struct panthor_vm_op_ctx *op_ctx) ++{ ++ for (u32 i = 0; i < ARRAY_SIZE(op_ctx->preallocated_vmas); i++) { ++ struct panthor_vma *vma = op_ctx->preallocated_vmas[i]; ++ ++ if (vma) { ++ op_ctx->preallocated_vmas[i] = NULL; ++ return vma; ++ } ++ } ++ ++ return NULL; ++} ++ ++static int ++panthor_vm_op_ctx_prealloc_vmas(struct panthor_vm_op_ctx *op_ctx) ++{ ++ u32 vma_count; ++ ++ switch (op_ctx->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) { ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: ++ /* One VMA for the new mapping, and two more VMAs for the remap case ++ * which might contain both a prev and next VA. ++ */ ++ vma_count = 3; ++ break; ++ ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP: ++ /* Partial unmaps might trigger a remap with either a prev or a next VA, ++ * but not both. ++ */ ++ vma_count = 1; ++ break; ++ ++ default: ++ return 0; ++ } ++ ++ for (u32 i = 0; i < vma_count; i++) { ++ struct panthor_vma *vma = kzalloc(sizeof(*vma), GFP_KERNEL); ++ ++ if (!vma) ++ return -ENOMEM; ++ ++ op_ctx->preallocated_vmas[i] = vma; ++ } ++ ++ return 0; ++} ++ ++#define PANTHOR_VM_BIND_OP_MAP_FLAGS \ ++ (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \ ++ DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) ++ ++static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, ++ struct panthor_vm *vm, ++ struct panthor_gem_object *bo, ++ u64 offset, ++ u64 size, u64 va, ++ u32 flags) ++{ ++ struct drm_gpuvm_bo *preallocated_vm_bo; ++ struct sg_table *sgt = NULL; ++ u64 pt_count; ++ int ret; ++ ++ if (!bo) ++ return -EINVAL; ++ ++ if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) || ++ (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP) ++ return -EINVAL; ++ ++ /* Make sure the VA and size are aligned and in-bounds. */ ++ if (size > bo->base.base.size || offset > bo->base.base.size - size) ++ return -EINVAL; ++ ++ /* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */ ++ if (bo->exclusive_vm_root_gem && ++ bo->exclusive_vm_root_gem != panthor_vm_root_gem(vm)) ++ return -EINVAL; ++ ++ memset(op_ctx, 0, sizeof(*op_ctx)); ++ INIT_LIST_HEAD(&op_ctx->returned_vmas); ++ op_ctx->flags = flags; ++ op_ctx->va.range = size; ++ op_ctx->va.addr = va; ++ ++ ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx); ++ if (ret) ++ goto err_cleanup; ++ ++ if (!bo->base.base.import_attach) { ++ /* Pre-reserve the BO pages, so the map operation doesn't have to ++ * allocate. ++ */ ++ ret = drm_gem_shmem_pin(&bo->base); ++ if (ret) ++ goto err_cleanup; ++ } ++ ++ sgt = drm_gem_shmem_get_pages_sgt(&bo->base); ++ if (IS_ERR(sgt)) { ++ if (!bo->base.base.import_attach) ++ drm_gem_shmem_unpin(&bo->base); ++ ++ ret = PTR_ERR(sgt); ++ goto err_cleanup; ++ } ++ ++ op_ctx->map.sgt = sgt; ++ ++ preallocated_vm_bo = drm_gpuvm_bo_create(&vm->base, &bo->base.base); ++ if (!preallocated_vm_bo) { ++ if (!bo->base.base.import_attach) ++ drm_gem_shmem_unpin(&bo->base); ++ ++ ret = -ENOMEM; ++ goto err_cleanup; ++ } ++ ++ mutex_lock(&bo->gpuva_list_lock); ++ op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo); ++ mutex_unlock(&bo->gpuva_list_lock); ++ ++ /* If the a vm_bo for this combination exists, it already ++ * retains a pin ref, and we can release the one we took earlier. ++ * ++ * If our pre-allocated vm_bo is picked, it now retains the pin ref, ++ * which will be released in panthor_vm_bo_put(). ++ */ ++ if (preallocated_vm_bo != op_ctx->map.vm_bo && ++ !bo->base.base.import_attach) ++ drm_gem_shmem_unpin(&bo->base); ++ ++ op_ctx->map.bo_offset = offset; ++ ++ /* L1, L2 and L3 page tables. ++ * We could optimize L3 allocation by iterating over the sgt and merging ++ * 2M contiguous blocks, but it's simpler to over-provision and return ++ * the pages if they're not used. ++ */ ++ pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) + ++ ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) + ++ ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21); ++ ++ op_ctx->rsvd_page_tables.pages = kcalloc(pt_count, ++ sizeof(*op_ctx->rsvd_page_tables.pages), ++ GFP_KERNEL); ++ if (!op_ctx->rsvd_page_tables.pages) ++ goto err_cleanup; ++ ++ ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, ++ op_ctx->rsvd_page_tables.pages); ++ op_ctx->rsvd_page_tables.count = ret; ++ if (ret != pt_count) { ++ ret = -ENOMEM; ++ goto err_cleanup; ++ } ++ ++ /* Insert BO into the extobj list last, when we know nothing can fail. */ ++ dma_resv_lock(panthor_vm_resv(vm), NULL); ++ drm_gpuvm_bo_extobj_add(op_ctx->map.vm_bo); ++ dma_resv_unlock(panthor_vm_resv(vm)); ++ ++ return 0; ++ ++err_cleanup: ++ panthor_vm_cleanup_op_ctx(op_ctx, vm); ++ return ret; ++} ++ ++static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx, ++ struct panthor_vm *vm, ++ u64 va, u64 size) ++{ ++ u32 pt_count = 0; ++ int ret; ++ ++ memset(op_ctx, 0, sizeof(*op_ctx)); ++ INIT_LIST_HEAD(&op_ctx->returned_vmas); ++ op_ctx->va.range = size; ++ op_ctx->va.addr = va; ++ op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP; ++ ++ /* Pre-allocate L3 page tables to account for the split-2M-block ++ * situation on unmap. ++ */ ++ if (va != ALIGN(va, SZ_2M)) ++ pt_count++; ++ ++ if (va + size != ALIGN(va + size, SZ_2M) && ++ ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M)) ++ pt_count++; ++ ++ ret = panthor_vm_op_ctx_prealloc_vmas(op_ctx); ++ if (ret) ++ goto err_cleanup; ++ ++ if (pt_count) { ++ op_ctx->rsvd_page_tables.pages = kcalloc(pt_count, ++ sizeof(*op_ctx->rsvd_page_tables.pages), ++ GFP_KERNEL); ++ if (!op_ctx->rsvd_page_tables.pages) ++ goto err_cleanup; ++ ++ ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, ++ op_ctx->rsvd_page_tables.pages); ++ if (ret != pt_count) { ++ ret = -ENOMEM; ++ goto err_cleanup; ++ } ++ op_ctx->rsvd_page_tables.count = pt_count; ++ } ++ ++ return 0; ++ ++err_cleanup: ++ panthor_vm_cleanup_op_ctx(op_ctx, vm); ++ return ret; ++} ++ ++static void panthor_vm_prepare_sync_only_op_ctx(struct panthor_vm_op_ctx *op_ctx, ++ struct panthor_vm *vm) ++{ ++ memset(op_ctx, 0, sizeof(*op_ctx)); ++ INIT_LIST_HEAD(&op_ctx->returned_vmas); ++ op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY; ++} ++ ++/** ++ * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address ++ * @vm: VM to look into. ++ * @va: Virtual address to search for. ++ * @bo_offset: Offset of the GEM object mapped at this virtual address. ++ * Only valid on success. ++ * ++ * The object returned by this function might no longer be mapped when the ++ * function returns. It's the caller responsibility to ensure there's no ++ * concurrent map/unmap operations making the returned value invalid, or ++ * make sure it doesn't matter if the object is no longer mapped. ++ * ++ * Return: A valid pointer on success, an ERR_PTR() otherwise. ++ */ ++struct panthor_gem_object * ++panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset) ++{ ++ struct panthor_gem_object *bo = ERR_PTR(-ENOENT); ++ struct drm_gpuva *gpuva; ++ struct panthor_vma *vma; ++ ++ /* Take the VM lock to prevent concurrent map/unmap operations. */ ++ mutex_lock(&vm->op_lock); ++ gpuva = drm_gpuva_find_first(&vm->base, va, 1); ++ vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL; ++ if (vma && vma->base.gem.obj) { ++ drm_gem_object_get(vma->base.gem.obj); ++ bo = to_panthor_bo(vma->base.gem.obj); ++ *bo_offset = vma->base.gem.offset + (va - vma->base.va.addr); ++ } ++ mutex_unlock(&vm->op_lock); ++ ++ return bo; ++} ++ ++#define PANTHOR_VM_MIN_KERNEL_VA_SIZE SZ_256M ++ ++static u64 ++panthor_vm_create_get_user_va_range(const struct drm_panthor_vm_create *args, ++ u64 full_va_range) ++{ ++ u64 user_va_range; ++ ++ /* Make sure we have a minimum amount of VA space for kernel objects. */ ++ if (full_va_range < PANTHOR_VM_MIN_KERNEL_VA_SIZE) ++ return 0; ++ ++ if (args->user_va_range) { ++ /* Use the user provided value if != 0. */ ++ user_va_range = args->user_va_range; ++ } else if (TASK_SIZE_OF(current) < full_va_range) { ++ /* If the task VM size is smaller than the GPU VA range, pick this ++ * as our default user VA range, so userspace can CPU/GPU map buffers ++ * at the same address. ++ */ ++ user_va_range = TASK_SIZE_OF(current); ++ } else { ++ /* If the GPU VA range is smaller than the task VM size, we ++ * just have to live with the fact we won't be able to map ++ * all buffers at the same GPU/CPU address. ++ * ++ * If the GPU VA range is bigger than 4G (more than 32-bit of ++ * VA), we split the range in two, and assign half of it to ++ * the user and the other half to the kernel, if it's not, we ++ * keep the kernel VA space as small as possible. ++ */ ++ user_va_range = full_va_range > SZ_4G ? ++ full_va_range / 2 : ++ full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE; ++ } ++ ++ if (full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE < user_va_range) ++ user_va_range = full_va_range - PANTHOR_VM_MIN_KERNEL_VA_SIZE; ++ ++ return user_va_range; ++} ++ ++#define PANTHOR_VM_CREATE_FLAGS 0 ++ ++static int ++panthor_vm_create_check_args(const struct panthor_device *ptdev, ++ const struct drm_panthor_vm_create *args, ++ u64 *kernel_va_start, u64 *kernel_va_range) ++{ ++ u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features); ++ u64 full_va_range = 1ull << va_bits; ++ u64 user_va_range; ++ ++ if (args->flags & ~PANTHOR_VM_CREATE_FLAGS) ++ return -EINVAL; ++ ++ user_va_range = panthor_vm_create_get_user_va_range(args, full_va_range); ++ if (!user_va_range || (args->user_va_range && args->user_va_range > user_va_range)) ++ return -EINVAL; ++ ++ /* Pick a kernel VA range that's a power of two, to have a clear split. */ ++ *kernel_va_range = rounddown_pow_of_two(full_va_range - user_va_range); ++ *kernel_va_start = full_va_range - *kernel_va_range; ++ return 0; ++} ++ ++/* ++ * Only 32 VMs per open file. If that becomes a limiting factor, we can ++ * increase this number. ++ */ ++#define PANTHOR_MAX_VMS_PER_FILE 32 ++ ++/** ++ * panthor_vm_pool_create_vm() - Create a VM ++ * @pool: The VM to create this VM on. ++ * @kernel_va_start: Start of the region reserved for kernel objects. ++ * @kernel_va_range: Size of the region reserved for kernel objects. ++ * ++ * Return: a positive VM ID on success, a negative error code otherwise. ++ */ ++int panthor_vm_pool_create_vm(struct panthor_device *ptdev, ++ struct panthor_vm_pool *pool, ++ struct drm_panthor_vm_create *args) ++{ ++ u64 kernel_va_start, kernel_va_range; ++ struct panthor_vm *vm; ++ int ret; ++ u32 id; ++ ++ ret = panthor_vm_create_check_args(ptdev, args, &kernel_va_start, &kernel_va_range); ++ if (ret) ++ return ret; ++ ++ vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range, ++ kernel_va_start, kernel_va_range); ++ if (IS_ERR(vm)) ++ return PTR_ERR(vm); ++ ++ ret = xa_alloc(&pool->xa, &id, vm, ++ XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL); ++ ++ if (ret) { ++ panthor_vm_put(vm); ++ return ret; ++ } ++ ++ args->user_va_range = kernel_va_start; ++ return id; ++} ++ ++static void panthor_vm_destroy(struct panthor_vm *vm) ++{ ++ if (!vm) ++ return; ++ ++ vm->destroyed = true; ++ ++ mutex_lock(&vm->heaps.lock); ++ panthor_heap_pool_destroy(vm->heaps.pool); ++ vm->heaps.pool = NULL; ++ mutex_unlock(&vm->heaps.lock); ++ ++ drm_WARN_ON(&vm->ptdev->base, ++ panthor_vm_unmap_range(vm, vm->base.mm_start, vm->base.mm_range)); ++ panthor_vm_put(vm); ++} ++ ++/** ++ * panthor_vm_pool_destroy_vm() - Destroy a VM. ++ * @pool: VM pool. ++ * @handle: VM handle. ++ * ++ * This function doesn't free the VM object or its resources, it just kills ++ * all mappings, and makes sure nothing can be mapped after that point. ++ * ++ * If there was any active jobs at the time this function is called, these ++ * jobs should experience page faults and be killed as a result. ++ * ++ * The VM resources are freed when the last reference on the VM object is ++ * dropped. ++ */ ++int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle) ++{ ++ struct panthor_vm *vm; ++ ++ vm = xa_erase(&pool->xa, handle); ++ ++ panthor_vm_destroy(vm); ++ ++ return vm ? 0 : -EINVAL; ++} ++ ++/** ++ * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle ++ * @pool: VM pool to check. ++ * @handle: Handle of the VM to retrieve. ++ * ++ * Return: A valid pointer if the VM exists, NULL otherwise. ++ */ ++struct panthor_vm * ++panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle) ++{ ++ struct panthor_vm *vm; ++ ++ vm = panthor_vm_get(xa_load(&pool->xa, handle)); ++ ++ return vm; ++} ++ ++/** ++ * panthor_vm_pool_destroy() - Destroy a VM pool. ++ * @pfile: File. ++ * ++ * Destroy all VMs in the pool, and release the pool resources. ++ * ++ * Note that VMs can outlive the pool they were created from if other ++ * objects hold a reference to there VMs. ++ */ ++void panthor_vm_pool_destroy(struct panthor_file *pfile) ++{ ++ struct panthor_vm *vm; ++ unsigned long i; ++ ++ if (!pfile->vms) ++ return; ++ ++ xa_for_each(&pfile->vms->xa, i, vm) ++ panthor_vm_destroy(vm); ++ ++ xa_destroy(&pfile->vms->xa); ++ kfree(pfile->vms); ++} ++ ++/** ++ * panthor_vm_pool_create() - Create a VM pool ++ * @pfile: File. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_pool_create(struct panthor_file *pfile) ++{ ++ pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL); ++ if (!pfile->vms) ++ return -ENOMEM; ++ ++ xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1); ++ return 0; ++} ++ ++/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */ ++static void mmu_tlb_flush_all(void *cookie) ++{ ++} ++ ++static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie) ++{ ++} ++ ++static const struct iommu_flush_ops mmu_tlb_ops = { ++ .tlb_flush_all = mmu_tlb_flush_all, ++ .tlb_flush_walk = mmu_tlb_flush_walk, ++}; ++ ++static const char *access_type_name(struct panthor_device *ptdev, ++ u32 fault_status) ++{ ++ switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) { ++ case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC: ++ return "ATOMIC"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_READ: ++ return "READ"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_WRITE: ++ return "WRITE"; ++ case AS_FAULTSTATUS_ACCESS_TYPE_EX: ++ return "EXECUTE"; ++ default: ++ drm_WARN_ON(&ptdev->base, 1); ++ return NULL; ++ } ++} ++ ++static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status) ++{ ++ bool has_unhandled_faults = false; ++ ++ status = panthor_mmu_fault_mask(ptdev, status); ++ while (status) { ++ u32 as = ffs(status | (status >> 16)) - 1; ++ u32 mask = panthor_mmu_as_fault_mask(ptdev, as); ++ u32 new_int_mask; ++ u64 addr; ++ u32 fault_status; ++ u32 exception_type; ++ u32 access_type; ++ u32 source_id; ++ ++ fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as)); ++ addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as)); ++ addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32; ++ ++ /* decode the fault status */ ++ exception_type = fault_status & 0xFF; ++ access_type = (fault_status >> 8) & 0x3; ++ source_id = (fault_status >> 16); ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ ++ ptdev->mmu->as.faulty_mask |= mask; ++ new_int_mask = ++ panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask); ++ ++ /* terminal fault, print info about the fault */ ++ drm_err(&ptdev->base, ++ "Unhandled Page fault in AS%d at VA 0x%016llX\n" ++ "raw fault status: 0x%X\n" ++ "decoded fault status: %s\n" ++ "exception type 0x%X: %s\n" ++ "access type 0x%X: %s\n" ++ "source id 0x%X\n", ++ as, addr, ++ fault_status, ++ (fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"), ++ exception_type, panthor_exception_name(ptdev, exception_type), ++ access_type, access_type_name(ptdev, fault_status), ++ source_id); ++ ++ /* Ignore MMU interrupts on this AS until it's been ++ * re-enabled. ++ */ ++ ptdev->mmu->irq.mask = new_int_mask; ++ gpu_write(ptdev, MMU_INT_MASK, new_int_mask); ++ ++ if (ptdev->mmu->as.slots[as].vm) ++ ptdev->mmu->as.slots[as].vm->unhandled_fault = true; ++ ++ /* Disable the MMU to kill jobs on this AS. */ ++ panthor_mmu_as_disable(ptdev, as); ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ status &= ~mask; ++ has_unhandled_faults = true; ++ } ++ ++ if (has_unhandled_faults) ++ panthor_sched_report_mmu_fault(ptdev); ++} ++PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler); ++ ++/** ++ * panthor_mmu_suspend() - Suspend the MMU logic ++ * @ptdev: Device. ++ * ++ * All we do here is de-assign the AS slots on all active VMs, so things ++ * get flushed to the main memory, and no further access to these VMs are ++ * possible. ++ * ++ * We also suspend the MMU IRQ. ++ */ ++void panthor_mmu_suspend(struct panthor_device *ptdev) ++{ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) { ++ struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm; ++ ++ if (vm) { ++ drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i)); ++ panthor_vm_release_as_locked(vm); ++ } ++ } ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ panthor_mmu_irq_suspend(&ptdev->mmu->irq); ++} ++ ++/** ++ * panthor_mmu_resume() - Resume the MMU logic ++ * @ptdev: Device. ++ * ++ * Resume the IRQ. ++ * ++ * We don't re-enable previously active VMs. We assume other parts of the ++ * driver will call panthor_vm_active() on the VMs they intend to use. ++ */ ++void panthor_mmu_resume(struct panthor_device *ptdev) ++{ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ ptdev->mmu->as.alloc_mask = 0; ++ ptdev->mmu->as.faulty_mask = 0; ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0)); ++} ++ ++/** ++ * panthor_mmu_pre_reset() - Prepare for a reset ++ * @ptdev: Device. ++ * ++ * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we ++ * don't get asked to do a VM operation while the GPU is down. ++ * ++ * We don't cleanly shutdown the AS slots here, because the reset might ++ * come from an AS_ACTIVE_BIT stuck situation. ++ */ ++void panthor_mmu_pre_reset(struct panthor_device *ptdev) ++{ ++ struct panthor_vm *vm; ++ ++ panthor_mmu_irq_suspend(&ptdev->mmu->irq); ++ ++ mutex_lock(&ptdev->mmu->vm.lock); ++ ptdev->mmu->vm.reset_in_progress = true; ++ list_for_each_entry(vm, &ptdev->mmu->vm.list, node) ++ panthor_vm_stop(vm); ++ mutex_unlock(&ptdev->mmu->vm.lock); ++} ++ ++/** ++ * panthor_mmu_post_reset() - Restore things after a reset ++ * @ptdev: Device. ++ * ++ * Put the MMU logic back in action after a reset. That implies resuming the ++ * IRQ and re-enabling the VM_BIND queues. ++ */ ++void panthor_mmu_post_reset(struct panthor_device *ptdev) ++{ ++ struct panthor_vm *vm; ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ ++ /* Now that the reset is effective, we can assume that none of the ++ * AS slots are setup, and clear the faulty flags too. ++ */ ++ ptdev->mmu->as.alloc_mask = 0; ++ ptdev->mmu->as.faulty_mask = 0; ++ ++ for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) { ++ struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm; ++ ++ if (vm) ++ panthor_vm_release_as_locked(vm); ++ } ++ ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0)); ++ ++ /* Restart the VM_BIND queues. */ ++ mutex_lock(&ptdev->mmu->vm.lock); ++ list_for_each_entry(vm, &ptdev->mmu->vm.list, node) { ++ panthor_vm_start(vm); ++ } ++ ptdev->mmu->vm.reset_in_progress = false; ++ mutex_unlock(&ptdev->mmu->vm.lock); ++} ++ ++static void panthor_vm_free(struct drm_gpuvm *gpuvm) ++{ ++ struct panthor_vm *vm = container_of(gpuvm, struct panthor_vm, base); ++ struct panthor_device *ptdev = vm->ptdev; ++ ++ mutex_lock(&vm->heaps.lock); ++ if (drm_WARN_ON(&ptdev->base, vm->heaps.pool)) ++ panthor_heap_pool_destroy(vm->heaps.pool); ++ mutex_unlock(&vm->heaps.lock); ++ mutex_destroy(&vm->heaps.lock); ++ ++ mutex_lock(&ptdev->mmu->vm.lock); ++ list_del(&vm->node); ++ /* Restore the scheduler state so we can call drm_sched_entity_destroy() ++ * and drm_sched_fini(). If get there, that means we have no job left ++ * and no new jobs can be queued, so we can start the scheduler without ++ * risking interfering with the reset. ++ */ ++ if (ptdev->mmu->vm.reset_in_progress) ++ panthor_vm_start(vm); ++ mutex_unlock(&ptdev->mmu->vm.lock); ++ ++ drm_sched_entity_destroy(&vm->entity); ++ drm_sched_fini(&vm->sched); ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ if (vm->as.id >= 0) { ++ int cookie; ++ ++ if (drm_dev_enter(&ptdev->base, &cookie)) { ++ panthor_mmu_as_disable(ptdev, vm->as.id); ++ drm_dev_exit(cookie); ++ } ++ ++ ptdev->mmu->as.slots[vm->as.id].vm = NULL; ++ clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask); ++ list_del(&vm->as.lru_node); ++ } ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++ ++ free_io_pgtable_ops(vm->pgtbl_ops); ++ ++ drm_mm_takedown(&vm->mm); ++ kfree(vm); ++} ++ ++/** ++ * panthor_vm_put() - Release a reference on a VM ++ * @vm: VM to release the reference on. Can be NULL. ++ */ ++void panthor_vm_put(struct panthor_vm *vm) ++{ ++ drm_gpuvm_put(vm ? &vm->base : NULL); ++} ++ ++/** ++ * panthor_vm_get() - Get a VM reference ++ * @vm: VM to get the reference on. Can be NULL. ++ * ++ * Return: @vm value. ++ */ ++struct panthor_vm *panthor_vm_get(struct panthor_vm *vm) ++{ ++ if (vm) ++ drm_gpuvm_get(&vm->base); ++ ++ return vm; ++} ++ ++/** ++ * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM ++ * @vm: VM to query the heap pool on. ++ * @create: True if the heap pool should be created when it doesn't exist. ++ * ++ * Heap pools are per-VM. This function allows one to retrieve the heap pool ++ * attached to a VM. ++ * ++ * If no heap pool exists yet, and @create is true, we create one. ++ * ++ * The returned panthor_heap_pool should be released with panthor_heap_pool_put(). ++ * ++ * Return: A valid pointer on success, an ERR_PTR() otherwise. ++ */ ++struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create) ++{ ++ struct panthor_heap_pool *pool; ++ ++ mutex_lock(&vm->heaps.lock); ++ if (!vm->heaps.pool && create) { ++ if (vm->destroyed) ++ pool = ERR_PTR(-EINVAL); ++ else ++ pool = panthor_heap_pool_create(vm->ptdev, vm); ++ ++ if (!IS_ERR(pool)) ++ vm->heaps.pool = panthor_heap_pool_get(pool); ++ } else { ++ pool = panthor_heap_pool_get(vm->heaps.pool); ++ } ++ mutex_unlock(&vm->heaps.lock); ++ ++ return pool; ++} ++ ++static u64 mair_to_memattr(u64 mair) ++{ ++ u64 memattr = 0; ++ u32 i; ++ ++ for (i = 0; i < 8; i++) { ++ u8 in_attr = mair >> (8 * i), out_attr; ++ u8 outer = in_attr >> 4, inner = in_attr & 0xf; ++ ++ /* For caching to be enabled, inner and outer caching policy ++ * have to be both write-back, if one of them is write-through ++ * or non-cacheable, we just choose non-cacheable. Device ++ * memory is also translated to non-cacheable. ++ */ ++ if (!(outer & 3) || !(outer & 4) || !(inner & 4)) { ++ out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC | ++ AS_MEMATTR_AARCH64_SH_MIDGARD_INNER | ++ AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false); ++ } else { ++ /* Use SH_CPU_INNER mode so SH_IS, which is used when ++ * IOMMU_CACHE is set, actually maps to the standard ++ * definition of inner-shareable and not Mali's ++ * internal-shareable mode. ++ */ ++ out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB | ++ AS_MEMATTR_AARCH64_SH_CPU_INNER | ++ AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2); ++ } ++ ++ memattr |= (u64)out_attr << (8 * i); ++ } ++ ++ return memattr; ++} ++ ++static void panthor_vma_link(struct panthor_vm *vm, ++ struct panthor_vma *vma, ++ struct drm_gpuvm_bo *vm_bo) ++{ ++ struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj); ++ ++ mutex_lock(&bo->gpuva_list_lock); ++ drm_gpuva_link(&vma->base, vm_bo); ++ drm_WARN_ON(&vm->ptdev->base, drm_gpuvm_bo_put(vm_bo)); ++ mutex_unlock(&bo->gpuva_list_lock); ++} ++ ++static void panthor_vma_unlink(struct panthor_vm *vm, ++ struct panthor_vma *vma) ++{ ++ struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj); ++ struct drm_gpuvm_bo *vm_bo = drm_gpuvm_bo_get(vma->base.vm_bo); ++ ++ mutex_lock(&bo->gpuva_list_lock); ++ drm_gpuva_unlink(&vma->base); ++ mutex_unlock(&bo->gpuva_list_lock); ++ ++ /* drm_gpuva_unlink() release the vm_bo, but we manually retained it ++ * when entering this function, so we can implement deferred VMA ++ * destruction. Re-assign it here. ++ */ ++ vma->base.vm_bo = vm_bo; ++ list_add_tail(&vma->node, &vm->op_ctx->returned_vmas); ++} ++ ++static void panthor_vma_init(struct panthor_vma *vma, u32 flags) ++{ ++ INIT_LIST_HEAD(&vma->node); ++ vma->flags = flags; ++} ++ ++#define PANTHOR_VM_MAP_FLAGS \ ++ (DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \ ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \ ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED) ++ ++static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv) ++{ ++ struct panthor_vm *vm = priv; ++ struct panthor_vm_op_ctx *op_ctx = vm->op_ctx; ++ struct panthor_vma *vma = panthor_vm_op_ctx_get_vma(op_ctx); ++ int ret; ++ ++ if (!vma) ++ return -EINVAL; ++ ++ panthor_vma_init(vma, op_ctx->flags & PANTHOR_VM_MAP_FLAGS); ++ ++ ret = panthor_vm_map_pages(vm, op->map.va.addr, flags_to_prot(vma->flags), ++ op_ctx->map.sgt, op->map.gem.offset, ++ op->map.va.range); ++ if (ret) ++ return ret; ++ ++ /* Ref owned by the mapping now, clear the obj field so we don't release the ++ * pinning/obj ref behind GPUVA's back. ++ */ ++ drm_gpuva_map(&vm->base, &vma->base, &op->map); ++ panthor_vma_link(vm, vma, op_ctx->map.vm_bo); ++ op_ctx->map.vm_bo = NULL; ++ return 0; ++} ++ ++static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op, ++ void *priv) ++{ ++ struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base); ++ struct panthor_vm *vm = priv; ++ struct panthor_vm_op_ctx *op_ctx = vm->op_ctx; ++ struct panthor_vma *prev_vma = NULL, *next_vma = NULL; ++ u64 unmap_start, unmap_range; ++ int ret; ++ ++ drm_gpuva_op_remap_to_unmap_range(&op->remap, &unmap_start, &unmap_range); ++ ret = panthor_vm_unmap_pages(vm, unmap_start, unmap_range); ++ if (ret) ++ return ret; ++ ++ if (op->remap.prev) { ++ prev_vma = panthor_vm_op_ctx_get_vma(op_ctx); ++ panthor_vma_init(prev_vma, unmap_vma->flags); ++ } ++ ++ if (op->remap.next) { ++ next_vma = panthor_vm_op_ctx_get_vma(op_ctx); ++ panthor_vma_init(next_vma, unmap_vma->flags); ++ } ++ ++ drm_gpuva_remap(prev_vma ? &prev_vma->base : NULL, ++ next_vma ? &next_vma->base : NULL, ++ &op->remap); ++ ++ if (prev_vma) { ++ /* panthor_vma_link() transfers the vm_bo ownership to ++ * the VMA object. Since the vm_bo we're passing is still ++ * owned by the old mapping which will be released when this ++ * mapping is destroyed, we need to grab a ref here. ++ */ ++ panthor_vma_link(vm, prev_vma, ++ drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo)); ++ } ++ ++ if (next_vma) { ++ panthor_vma_link(vm, next_vma, ++ drm_gpuvm_bo_get(op->remap.unmap->va->vm_bo)); ++ } ++ ++ panthor_vma_unlink(vm, unmap_vma); ++ return 0; ++} ++ ++static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op, ++ void *priv) ++{ ++ struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base); ++ struct panthor_vm *vm = priv; ++ int ret; ++ ++ ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr, ++ unmap_vma->base.va.range); ++ if (drm_WARN_ON(&vm->ptdev->base, ret)) ++ return ret; ++ ++ drm_gpuva_unmap(&op->unmap); ++ panthor_vma_unlink(vm, unmap_vma); ++ return 0; ++} ++ ++static const struct drm_gpuvm_ops panthor_gpuvm_ops = { ++ .vm_free = panthor_vm_free, ++ .sm_step_map = panthor_gpuva_sm_step_map, ++ .sm_step_remap = panthor_gpuva_sm_step_remap, ++ .sm_step_unmap = panthor_gpuva_sm_step_unmap, ++}; ++ ++/** ++ * panthor_vm_resv() - Get the dma_resv object attached to a VM. ++ * @vm: VM to get the dma_resv of. ++ * ++ * Return: A dma_resv object. ++ */ ++struct dma_resv *panthor_vm_resv(struct panthor_vm *vm) ++{ ++ return drm_gpuvm_resv(&vm->base); ++} ++ ++struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm) ++{ ++ if (!vm) ++ return NULL; ++ ++ return vm->base.r_obj; ++} ++ ++static int ++panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op, ++ bool flag_vm_unusable_on_failure) ++{ ++ u32 op_type = op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK; ++ int ret; ++ ++ if (op_type == DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY) ++ return 0; ++ ++ mutex_lock(&vm->op_lock); ++ vm->op_ctx = op; ++ switch (op_type) { ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: ++ if (vm->unusable) { ++ ret = -EINVAL; ++ break; ++ } ++ ++ ret = drm_gpuvm_sm_map(&vm->base, vm, op->va.addr, op->va.range, ++ op->map.vm_bo->obj, op->map.bo_offset); ++ break; ++ ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP: ++ ret = drm_gpuvm_sm_unmap(&vm->base, vm, op->va.addr, op->va.range); ++ break; ++ ++ default: ++ ret = -EINVAL; ++ break; ++ } ++ ++ if (ret && flag_vm_unusable_on_failure) ++ vm->unusable = true; ++ ++ vm->op_ctx = NULL; ++ mutex_unlock(&vm->op_lock); ++ ++ return ret; ++} ++ ++static struct dma_fence * ++panthor_vm_bind_run_job(struct drm_sched_job *sched_job) ++{ ++ struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base); ++ bool cookie; ++ int ret; ++ ++ /* Not only we report an error whose result is propagated to the ++ * drm_sched finished fence, but we also flag the VM as unusable, because ++ * a failure in the async VM_BIND results in an inconsistent state. VM needs ++ * to be destroyed and recreated. ++ */ ++ cookie = dma_fence_begin_signalling(); ++ ret = panthor_vm_exec_op(job->vm, &job->ctx, true); ++ dma_fence_end_signalling(cookie); ++ ++ return ret ? ERR_PTR(ret) : NULL; ++} ++ ++static void panthor_vm_bind_job_release(struct kref *kref) ++{ ++ struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount); ++ ++ if (job->base.s_fence) ++ drm_sched_job_cleanup(&job->base); ++ ++ panthor_vm_cleanup_op_ctx(&job->ctx, job->vm); ++ panthor_vm_put(job->vm); ++ kfree(job); ++} ++ ++/** ++ * panthor_vm_bind_job_put() - Release a VM_BIND job reference ++ * @sched_job: Job to release the reference on. ++ */ ++void panthor_vm_bind_job_put(struct drm_sched_job *sched_job) ++{ ++ struct panthor_vm_bind_job *job = ++ container_of(sched_job, struct panthor_vm_bind_job, base); ++ ++ if (sched_job) ++ kref_put(&job->refcount, panthor_vm_bind_job_release); ++} ++ ++static void ++panthor_vm_bind_free_job(struct drm_sched_job *sched_job) ++{ ++ struct panthor_vm_bind_job *job = ++ container_of(sched_job, struct panthor_vm_bind_job, base); ++ ++ drm_sched_job_cleanup(sched_job); ++ ++ /* Do the heavy cleanups asynchronously, so we're out of the ++ * dma-signaling path and can acquire dma-resv locks safely. ++ */ ++ queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work); ++} ++ ++static enum drm_gpu_sched_stat ++panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job) ++{ ++ WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!"); ++ return DRM_GPU_SCHED_STAT_NOMINAL; ++} ++ ++static const struct drm_sched_backend_ops panthor_vm_bind_ops = { ++ .run_job = panthor_vm_bind_run_job, ++ .free_job = panthor_vm_bind_free_job, ++ .timedout_job = panthor_vm_bind_timedout_job, ++}; ++ ++/** ++ * panthor_vm_create() - Create a VM ++ * @ptdev: Device. ++ * @for_mcu: True if this is the FW MCU VM. ++ * @kernel_va_start: Start of the range reserved for kernel BO mapping. ++ * @kernel_va_size: Size of the range reserved for kernel BO mapping. ++ * @auto_kernel_va_start: Start of the auto-VA kernel range. ++ * @auto_kernel_va_size: Size of the auto-VA kernel range. ++ * ++ * Return: A valid pointer on success, an ERR_PTR() otherwise. ++ */ ++struct panthor_vm * ++panthor_vm_create(struct panthor_device *ptdev, bool for_mcu, ++ u64 kernel_va_start, u64 kernel_va_size, ++ u64 auto_kernel_va_start, u64 auto_kernel_va_size) ++{ ++ u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features); ++ u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features); ++ u64 full_va_range = 1ull << va_bits; ++ struct drm_gem_object *dummy_gem; ++ struct drm_gpu_scheduler *sched; ++ struct io_pgtable_cfg pgtbl_cfg; ++ u64 mair, min_va, va_range; ++ struct panthor_vm *vm; ++ int ret; ++ ++ vm = kzalloc(sizeof(*vm), GFP_KERNEL); ++ if (!vm) ++ return ERR_PTR(-ENOMEM); ++ ++ /* We allocate a dummy GEM for the VM. */ ++ dummy_gem = drm_gpuvm_resv_object_alloc(&ptdev->base); ++ if (!dummy_gem) { ++ ret = -ENOMEM; ++ goto err_free_vm; ++ } ++ ++ mutex_init(&vm->heaps.lock); ++ vm->for_mcu = for_mcu; ++ vm->ptdev = ptdev; ++ mutex_init(&vm->op_lock); ++ ++ if (for_mcu) { ++ /* CSF MCU is a cortex M7, and can only address 4G */ ++ min_va = 0; ++ va_range = SZ_4G; ++ } else { ++ min_va = 0; ++ va_range = full_va_range; ++ } ++ ++ mutex_init(&vm->mm_lock); ++ drm_mm_init(&vm->mm, kernel_va_start, kernel_va_size); ++ vm->kernel_auto_va.start = auto_kernel_va_start; ++ vm->kernel_auto_va.end = vm->kernel_auto_va.start + auto_kernel_va_size - 1; ++ ++ INIT_LIST_HEAD(&vm->node); ++ INIT_LIST_HEAD(&vm->as.lru_node); ++ vm->as.id = -1; ++ refcount_set(&vm->as.active_cnt, 0); ++ ++ pgtbl_cfg = (struct io_pgtable_cfg) { ++ .pgsize_bitmap = SZ_4K | SZ_2M, ++ .ias = va_bits, ++ .oas = pa_bits, ++ .coherent_walk = ptdev->coherent, ++ .tlb = &mmu_tlb_ops, ++ .iommu_dev = ptdev->base.dev, ++ .alloc = alloc_pt, ++ .free = free_pt, ++ }; ++ ++ vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm); ++ if (!vm->pgtbl_ops) { ++ ret = -EINVAL; ++ goto err_mm_takedown; ++ } ++ ++ /* Bind operations are synchronous for now, no timeout needed. */ ++ ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq, ++ 1, 1, 0, ++ MAX_SCHEDULE_TIMEOUT, NULL, NULL, ++ "panthor-vm-bind", ptdev->base.dev); ++ if (ret) ++ goto err_free_io_pgtable; ++ ++ sched = &vm->sched; ++ ret = drm_sched_entity_init(&vm->entity, 0, &sched, 1, NULL); ++ if (ret) ++ goto err_sched_fini; ++ ++ mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair; ++ vm->memattr = mair_to_memattr(mair); ++ ++ mutex_lock(&ptdev->mmu->vm.lock); ++ list_add_tail(&vm->node, &ptdev->mmu->vm.list); ++ ++ /* If a reset is in progress, stop the scheduler. */ ++ if (ptdev->mmu->vm.reset_in_progress) ++ panthor_vm_stop(vm); ++ mutex_unlock(&ptdev->mmu->vm.lock); ++ ++ /* We intentionally leave the reserved range to zero, because we want kernel VMAs ++ * to be handled the same way user VMAs are. ++ */ ++ drm_gpuvm_init(&vm->base, for_mcu ? "panthor-MCU-VM" : "panthor-GPU-VM", ++ DRM_GPUVM_RESV_PROTECTED, &ptdev->base, dummy_gem, ++ min_va, va_range, 0, 0, &panthor_gpuvm_ops); ++ drm_gem_object_put(dummy_gem); ++ return vm; ++ ++err_sched_fini: ++ drm_sched_fini(&vm->sched); ++ ++err_free_io_pgtable: ++ free_io_pgtable_ops(vm->pgtbl_ops); ++ ++err_mm_takedown: ++ drm_mm_takedown(&vm->mm); ++ drm_gem_object_put(dummy_gem); ++ ++err_free_vm: ++ kfree(vm); ++ return ERR_PTR(ret); ++} ++ ++static int ++panthor_vm_bind_prepare_op_ctx(struct drm_file *file, ++ struct panthor_vm *vm, ++ const struct drm_panthor_vm_bind_op *op, ++ struct panthor_vm_op_ctx *op_ctx) ++{ ++ struct drm_gem_object *gem; ++ int ret; ++ ++ /* Aligned on page size. */ ++ if ((op->va | op->size) & ~PAGE_MASK) ++ return -EINVAL; ++ ++ switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) { ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP: ++ gem = drm_gem_object_lookup(file, op->bo_handle); ++ ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm, ++ gem ? to_panthor_bo(gem) : NULL, ++ op->bo_offset, ++ op->size, ++ op->va, ++ op->flags); ++ drm_gem_object_put(gem); ++ return ret; ++ ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP: ++ if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) ++ return -EINVAL; ++ ++ if (op->bo_handle || op->bo_offset) ++ return -EINVAL; ++ ++ return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size); ++ ++ case DRM_PANTHOR_VM_BIND_OP_TYPE_SYNC_ONLY: ++ if (op->flags & ~DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) ++ return -EINVAL; ++ ++ if (op->bo_handle || op->bo_offset) ++ return -EINVAL; ++ ++ if (op->va || op->size) ++ return -EINVAL; ++ ++ if (!op->syncs.count) ++ return -EINVAL; ++ ++ panthor_vm_prepare_sync_only_op_ctx(op_ctx, vm); ++ return 0; ++ ++ default: ++ return -EINVAL; ++ } ++} ++ ++static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work) ++{ ++ struct panthor_vm_bind_job *job = ++ container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work); ++ ++ panthor_vm_bind_job_put(&job->base); ++} ++ ++/** ++ * panthor_vm_bind_job_create() - Create a VM_BIND job ++ * @file: File. ++ * @vm: VM targeted by the VM_BIND job. ++ * @op: VM operation data. ++ * ++ * Return: A valid pointer on success, an ERR_PTR() otherwise. ++ */ ++struct drm_sched_job * ++panthor_vm_bind_job_create(struct drm_file *file, ++ struct panthor_vm *vm, ++ const struct drm_panthor_vm_bind_op *op) ++{ ++ struct panthor_vm_bind_job *job; ++ int ret; ++ ++ if (!vm) ++ return ERR_PTR(-EINVAL); ++ ++ if (vm->destroyed || vm->unusable) ++ return ERR_PTR(-EINVAL); ++ ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (!job) ++ return ERR_PTR(-ENOMEM); ++ ++ ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx); ++ if (ret) { ++ kfree(job); ++ return ERR_PTR(ret); ++ } ++ ++ INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work); ++ kref_init(&job->refcount); ++ job->vm = panthor_vm_get(vm); ++ ++ ret = drm_sched_job_init(&job->base, &vm->entity, 1, vm); ++ if (ret) ++ goto err_put_job; ++ ++ return &job->base; ++ ++err_put_job: ++ panthor_vm_bind_job_put(&job->base); ++ return ERR_PTR(ret); ++} ++ ++/** ++ * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs ++ * @exec: The locking/preparation context. ++ * @sched_job: The job to prepare resvs on. ++ * ++ * Locks and prepare the VM resv. ++ * ++ * If this is a map operation, locks and prepares the GEM resv. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec, ++ struct drm_sched_job *sched_job) ++{ ++ struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base); ++ int ret; ++ ++ /* Acquire the VM lock an reserve a slot for this VM bind job. */ ++ ret = drm_gpuvm_prepare_vm(&job->vm->base, exec, 1); ++ if (ret) ++ return ret; ++ ++ if (job->ctx.map.vm_bo) { ++ /* Lock/prepare the GEM being mapped. */ ++ ret = drm_exec_prepare_obj(exec, job->ctx.map.vm_bo->obj, 1); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job ++ * @exec: drm_exec context. ++ * @sched_job: Job to update the resvs on. ++ */ ++void panthor_vm_bind_job_update_resvs(struct drm_exec *exec, ++ struct drm_sched_job *sched_job) ++{ ++ struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base); ++ ++ /* Explicit sync => we just register our job finished fence as bookkeep. */ ++ drm_gpuvm_resv_add_fence(&job->vm->base, exec, ++ &sched_job->s_fence->finished, ++ DMA_RESV_USAGE_BOOKKEEP, ++ DMA_RESV_USAGE_BOOKKEEP); ++} ++ ++void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec, ++ struct dma_fence *fence, ++ enum dma_resv_usage private_usage, ++ enum dma_resv_usage extobj_usage) ++{ ++ drm_gpuvm_resv_add_fence(&vm->base, exec, fence, private_usage, extobj_usage); ++} ++ ++/** ++ * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously. ++ * @file: File. ++ * @vm: VM targeted by the VM operation. ++ * @op: Data describing the VM operation. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_bind_exec_sync_op(struct drm_file *file, ++ struct panthor_vm *vm, ++ struct drm_panthor_vm_bind_op *op) ++{ ++ struct panthor_vm_op_ctx op_ctx; ++ int ret; ++ ++ /* No sync objects allowed on synchronous operations. */ ++ if (op->syncs.count) ++ return -EINVAL; ++ ++ if (!op->size) ++ return 0; ++ ++ ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx); ++ if (ret) ++ return ret; ++ ++ ret = panthor_vm_exec_op(vm, &op_ctx, false); ++ panthor_vm_cleanup_op_ctx(&op_ctx, vm); ++ ++ return ret; ++} ++ ++/** ++ * panthor_vm_map_bo_range() - Map a GEM object range to a VM ++ * @vm: VM to map the GEM to. ++ * @bo: GEM object to map. ++ * @offset: Offset in the GEM object. ++ * @size: Size to map. ++ * @va: Virtual address to map the object to. ++ * @flags: Combination of drm_panthor_vm_bind_op_flags flags. ++ * Only map-related flags are valid. ++ * ++ * Internal use only. For userspace requests, use ++ * panthor_vm_bind_exec_sync_op() instead. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo, ++ u64 offset, u64 size, u64 va, u32 flags) ++{ ++ struct panthor_vm_op_ctx op_ctx; ++ int ret; ++ ++ ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags); ++ if (ret) ++ return ret; ++ ++ ret = panthor_vm_exec_op(vm, &op_ctx, false); ++ panthor_vm_cleanup_op_ctx(&op_ctx, vm); ++ ++ return ret; ++} ++ ++/** ++ * panthor_vm_unmap_range() - Unmap a portion of the VA space ++ * @vm: VM to unmap the region from. ++ * @va: Virtual address to unmap. Must be 4k aligned. ++ * @size: Size of the region to unmap. Must be 4k aligned. ++ * ++ * Internal use only. For userspace requests, use ++ * panthor_vm_bind_exec_sync_op() instead. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size) ++{ ++ struct panthor_vm_op_ctx op_ctx; ++ int ret; ++ ++ ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size); ++ if (ret) ++ return ret; ++ ++ ret = panthor_vm_exec_op(vm, &op_ctx, false); ++ panthor_vm_cleanup_op_ctx(&op_ctx, vm); ++ ++ return ret; ++} ++ ++/** ++ * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs. ++ * @exec: Locking/preparation context. ++ * @vm: VM targeted by the GPU job. ++ * @slot_count: Number of slots to reserve. ++ * ++ * GPU jobs assume all BOs bound to the VM at the time the job is submitted ++ * are available when the job is executed. In order to guarantee that, we ++ * need to reserve a slot on all BOs mapped to a VM and update this slot with ++ * the job fence after its submission. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm, ++ u32 slot_count) ++{ ++ int ret; ++ ++ /* Acquire the VM lock and reserve a slot for this GPU job. */ ++ ret = drm_gpuvm_prepare_vm(&vm->base, exec, slot_count); ++ if (ret) ++ return ret; ++ ++ return drm_gpuvm_prepare_objects(&vm->base, exec, slot_count); ++} ++ ++/** ++ * panthor_mmu_unplug() - Unplug the MMU logic ++ * @ptdev: Device. ++ * ++ * No access to the MMU regs should be done after this function is called. ++ * We suspend the IRQ and disable all VMs to guarantee that. ++ */ ++void panthor_mmu_unplug(struct panthor_device *ptdev) ++{ ++ panthor_mmu_irq_suspend(&ptdev->mmu->irq); ++ ++ mutex_lock(&ptdev->mmu->as.slots_lock); ++ for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) { ++ struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm; ++ ++ if (vm) { ++ drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i)); ++ panthor_vm_release_as_locked(vm); ++ } ++ } ++ mutex_unlock(&ptdev->mmu->as.slots_lock); ++} ++ ++static void panthor_mmu_release_wq(struct drm_device *ddev, void *res) ++{ ++ destroy_workqueue(res); ++} ++ ++/** ++ * panthor_mmu_init() - Initialize the MMU logic. ++ * @ptdev: Device. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_mmu_init(struct panthor_device *ptdev) ++{ ++ u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features); ++ struct panthor_mmu *mmu; ++ int ret, irq; ++ ++ mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL); ++ if (!mmu) ++ return -ENOMEM; ++ ++ INIT_LIST_HEAD(&mmu->as.lru_list); ++ ++ ret = drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock); ++ if (ret) ++ return ret; ++ ++ INIT_LIST_HEAD(&mmu->vm.list); ++ ret = drmm_mutex_init(&ptdev->base, &mmu->vm.lock); ++ if (ret) ++ return ret; ++ ++ ptdev->mmu = mmu; ++ ++ irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu"); ++ if (irq <= 0) ++ return -ENODEV; ++ ++ ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq, ++ panthor_mmu_fault_mask(ptdev, ~0)); ++ if (ret) ++ return ret; ++ ++ mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0); ++ if (!mmu->vm.wq) ++ return -ENOMEM; ++ ++ /* On 32-bit kernels, the VA space is limited by the io_pgtable_ops abstraction, ++ * which passes iova as an unsigned long. Patch the mmu_features to reflect this ++ * limitation. ++ */ ++ if (sizeof(unsigned long) * 8 < va_bits) { ++ ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0); ++ ptdev->gpu_info.mmu_features |= sizeof(unsigned long) * 8; ++ } ++ ++ return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq); ++} ++ ++#ifdef CONFIG_DEBUG_FS ++static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m) ++{ ++ int ret; ++ ++ mutex_lock(&vm->op_lock); ++ ret = drm_debugfs_gpuva_info(m, &vm->base); ++ mutex_unlock(&vm->op_lock); ++ ++ return ret; ++} ++ ++static int show_each_vm(struct seq_file *m, void *arg) ++{ ++ struct drm_info_node *node = (struct drm_info_node *)m->private; ++ struct drm_device *ddev = node->minor->dev; ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data; ++ struct panthor_vm *vm; ++ int ret = 0; ++ ++ mutex_lock(&ptdev->mmu->vm.lock); ++ list_for_each_entry(vm, &ptdev->mmu->vm.list, node) { ++ ret = show(vm, m); ++ if (ret < 0) ++ break; ++ ++ seq_puts(m, "\n"); ++ } ++ mutex_unlock(&ptdev->mmu->vm.lock); ++ ++ return ret; ++} ++ ++static struct drm_info_list panthor_mmu_debugfs_list[] = { ++ DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas), ++}; ++ ++/** ++ * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries ++ * @minor: Minor. ++ */ ++void panthor_mmu_debugfs_init(struct drm_minor *minor) ++{ ++ drm_debugfs_create_files(panthor_mmu_debugfs_list, ++ ARRAY_SIZE(panthor_mmu_debugfs_list), ++ minor->debugfs_root, minor); ++} ++#endif /* CONFIG_DEBUG_FS */ ++ ++/** ++ * panthor_mmu_pt_cache_init() - Initialize the page table cache. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_mmu_pt_cache_init(void) ++{ ++ pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL); ++ if (!pt_cache) ++ return -ENOMEM; ++ ++ return 0; ++} ++ ++/** ++ * panthor_mmu_pt_cache_fini() - Destroy the page table cache. ++ */ ++void panthor_mmu_pt_cache_fini(void) ++{ ++ kmem_cache_destroy(pt_cache); ++} +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h +new file mode 100644 +index 00000000000000..f3c1ed19f973f6 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_mmu.h +@@ -0,0 +1,102 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2019 Linaro, Ltd, Rob Herring */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_MMU_H__ ++#define __PANTHOR_MMU_H__ ++ ++#include ++ ++struct drm_exec; ++struct drm_sched_job; ++struct panthor_gem_object; ++struct panthor_heap_pool; ++struct panthor_vm; ++struct panthor_vma; ++struct panthor_mmu; ++ ++int panthor_mmu_init(struct panthor_device *ptdev); ++void panthor_mmu_unplug(struct panthor_device *ptdev); ++void panthor_mmu_pre_reset(struct panthor_device *ptdev); ++void panthor_mmu_post_reset(struct panthor_device *ptdev); ++void panthor_mmu_suspend(struct panthor_device *ptdev); ++void panthor_mmu_resume(struct panthor_device *ptdev); ++ ++int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo, ++ u64 offset, u64 size, u64 va, u32 flags); ++int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, u64 size); ++struct panthor_gem_object * ++panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset); ++ ++int panthor_vm_active(struct panthor_vm *vm); ++void panthor_vm_idle(struct panthor_vm *vm); ++int panthor_vm_as(struct panthor_vm *vm); ++ ++struct panthor_heap_pool * ++panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create); ++ ++struct panthor_vm *panthor_vm_get(struct panthor_vm *vm); ++void panthor_vm_put(struct panthor_vm *vm); ++struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu, ++ u64 kernel_va_start, u64 kernel_va_size, ++ u64 kernel_auto_va_start, ++ u64 kernel_auto_va_size); ++ ++int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, ++ struct panthor_vm *vm, ++ u32 slot_count); ++int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm, ++ struct drm_sched_job *job); ++void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm, ++ struct drm_sched_job *job); ++ ++struct dma_resv *panthor_vm_resv(struct panthor_vm *vm); ++struct drm_gem_object *panthor_vm_root_gem(struct panthor_vm *vm); ++ ++void panthor_vm_pool_destroy(struct panthor_file *pfile); ++int panthor_vm_pool_create(struct panthor_file *pfile); ++int panthor_vm_pool_create_vm(struct panthor_device *ptdev, ++ struct panthor_vm_pool *pool, ++ struct drm_panthor_vm_create *args); ++int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle); ++struct panthor_vm *panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle); ++ ++bool panthor_vm_has_unhandled_faults(struct panthor_vm *vm); ++bool panthor_vm_is_unusable(struct panthor_vm *vm); ++ ++/* ++ * PANTHOR_VM_KERNEL_AUTO_VA: Use this magic address when you want the GEM ++ * logic to auto-allocate the virtual address in the reserved kernel VA range. ++ */ ++#define PANTHOR_VM_KERNEL_AUTO_VA ~0ull ++ ++int panthor_vm_alloc_va(struct panthor_vm *vm, u64 va, u64 size, ++ struct drm_mm_node *va_node); ++void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *va_node); ++ ++int panthor_vm_bind_exec_sync_op(struct drm_file *file, ++ struct panthor_vm *vm, ++ struct drm_panthor_vm_bind_op *op); ++ ++struct drm_sched_job * ++panthor_vm_bind_job_create(struct drm_file *file, ++ struct panthor_vm *vm, ++ const struct drm_panthor_vm_bind_op *op); ++void panthor_vm_bind_job_put(struct drm_sched_job *job); ++int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec, ++ struct drm_sched_job *job); ++void panthor_vm_bind_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *job); ++ ++void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec, ++ struct dma_fence *fence, ++ enum dma_resv_usage private_usage, ++ enum dma_resv_usage extobj_usage); ++ ++int panthor_mmu_pt_cache_init(void); ++void panthor_mmu_pt_cache_fini(void); ++ ++#ifdef CONFIG_DEBUG_FS ++void panthor_mmu_debugfs_init(struct drm_minor *minor); ++#endif ++ ++#endif + +From f95aab2f53a50b5a85b7417cf55e3686c7e42e6c Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:22 +0100 +Subject: [PATCH 087/150] drm/panthor: Add the FW logical block + +Contains everything that's FW related, that includes the code dealing +with the microcontroller unit (MCU) that's running the FW, and anything +related to allocating memory shared between the FW and the CPU. + +A few global FW events are processed in the IRQ handler, the rest is +forwarded to the scheduler, since scheduling is the primary reason for +the FW existence, and also the main source of FW <-> kernel +interactions. + +v6: +- Add Maxime's and Heiko's acks +- Keep header inclusion alphabetically ordered + +v5: +- Fix typo in GLB_PERFCNT_SAMPLE definition +- Fix unbalanced panthor_vm_idle/active() calls +- Fallback to a slow reset when the fast reset fails +- Add extra information when reporting a FW boot failure + +v4: +- Add a MODULE_FIRMWARE() entry for gen 10.8 +- Fix a wrong return ERR_PTR() in panthor_fw_load_section_entry() +- Fix typos +- Add Steve's R-b + +v3: +- Make the FW path more future-proof (Liviu) +- Use one waitqueue for all FW events +- Simplify propagation of FW events to the scheduler logic +- Drop the panthor_fw_mem abstraction and use panthor_kernel_bo instead +- Account for the panthor_vm changes +- Replace magic number with 0x7fffffff with ~0 to better signify that + it's the maximum permitted value. +- More accurate rounding when computing the firmware timeout. +- Add a 'sub iterator' helper function. This also adds a check that a + firmware entry doesn't overflow the firmware image. +- Drop __packed from FW structures, natural alignment is good enough. +- Other minor code improvements. + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-9-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 1362 ++++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_fw.h | 503 ++++++++++ + 2 files changed, 1865 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_fw.c + create mode 100644 drivers/gpu/drm/panthor/panthor_fw.h + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +new file mode 100644 +index 00000000000000..33c87a59834ecf +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -0,0 +1,1362 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifdef CONFIG_ARM_ARCH_TIMER ++#include ++#endif ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++ ++#include "panthor_device.h" ++#include "panthor_fw.h" ++#include "panthor_gem.h" ++#include "panthor_gpu.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++#include "panthor_sched.h" ++ ++#define CSF_FW_NAME "mali_csffw.bin" ++ ++#define PING_INTERVAL_MS 12000 ++#define PROGRESS_TIMEOUT_CYCLES (5ull * 500 * 1024 * 1024) ++#define PROGRESS_TIMEOUT_SCALE_SHIFT 10 ++#define IDLE_HYSTERESIS_US 800 ++#define PWROFF_HYSTERESIS_US 10000 ++ ++/** ++ * struct panthor_fw_binary_hdr - Firmware binary header. ++ */ ++struct panthor_fw_binary_hdr { ++ /** @magic: Magic value to check binary validity. */ ++ u32 magic; ++#define CSF_FW_BINARY_HEADER_MAGIC 0xc3f13a6e ++ ++ /** @minor: Minor FW version. */ ++ u8 minor; ++ ++ /** @major: Major FW version. */ ++ u8 major; ++#define CSF_FW_BINARY_HEADER_MAJOR_MAX 0 ++ ++ /** @padding1: MBZ. */ ++ u16 padding1; ++ ++ /** @version_hash: FW version hash. */ ++ u32 version_hash; ++ ++ /** @padding2: MBZ. */ ++ u32 padding2; ++ ++ /** @size: FW binary size. */ ++ u32 size; ++}; ++ ++/** ++ * enum panthor_fw_binary_entry_type - Firmware binary entry type ++ */ ++enum panthor_fw_binary_entry_type { ++ /** @CSF_FW_BINARY_ENTRY_TYPE_IFACE: Host <-> FW interface. */ ++ CSF_FW_BINARY_ENTRY_TYPE_IFACE = 0, ++ ++ /** @CSF_FW_BINARY_ENTRY_TYPE_CONFIG: FW config. */ ++ CSF_FW_BINARY_ENTRY_TYPE_CONFIG = 1, ++ ++ /** @CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST: Unit-tests. */ ++ CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST = 2, ++ ++ /** @CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER: Trace buffer interface. */ ++ CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER = 3, ++ ++ /** @CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: Timeline metadata interface. */ ++ CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA = 4, ++}; ++ ++#define CSF_FW_BINARY_ENTRY_TYPE(ehdr) ((ehdr) & 0xff) ++#define CSF_FW_BINARY_ENTRY_SIZE(ehdr) (((ehdr) >> 8) & 0xff) ++#define CSF_FW_BINARY_ENTRY_UPDATE BIT(30) ++#define CSF_FW_BINARY_ENTRY_OPTIONAL BIT(31) ++ ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_RD BIT(0) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_WR BIT(1) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_EX BIT(2) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_NONE (0 << 3) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED (1 << 3) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_UNCACHED_COHERENT (2 << 3) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED_COHERENT (3 << 3) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK GENMASK(4, 3) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_PROT BIT(5) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED BIT(30) ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO BIT(31) ++ ++#define CSF_FW_BINARY_IFACE_ENTRY_RD_SUPPORTED_FLAGS \ ++ (CSF_FW_BINARY_IFACE_ENTRY_RD_RD | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_WR | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_EX | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_PROT | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED | \ ++ CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO) ++ ++/** ++ * struct panthor_fw_binary_section_entry_hdr - Describes a section of FW binary ++ */ ++struct panthor_fw_binary_section_entry_hdr { ++ /** @flags: Section flags. */ ++ u32 flags; ++ ++ /** @va: MCU virtual range to map this binary section to. */ ++ struct { ++ /** @start: Start address. */ ++ u32 start; ++ ++ /** @end: End address. */ ++ u32 end; ++ } va; ++ ++ /** @data: Data to initialize the FW section with. */ ++ struct { ++ /** @start: Start offset in the FW binary. */ ++ u32 start; ++ ++ /** @end: End offset in the FW binary. */ ++ u32 end; ++ } data; ++}; ++ ++/** ++ * struct panthor_fw_binary_iter - Firmware binary iterator ++ * ++ * Used to parse a firmware binary. ++ */ ++struct panthor_fw_binary_iter { ++ /** @data: FW binary data. */ ++ const void *data; ++ ++ /** @size: FW binary size. */ ++ size_t size; ++ ++ /** @offset: Iterator offset. */ ++ size_t offset; ++}; ++ ++/** ++ * struct panthor_fw_section - FW section ++ */ ++struct panthor_fw_section { ++ /** @node: Used to keep track of FW sections. */ ++ struct list_head node; ++ ++ /** @flags: Section flags, as encoded in the FW binary. */ ++ u32 flags; ++ ++ /** @mem: Section memory. */ ++ struct panthor_kernel_bo *mem; ++ ++ /** ++ * @name: Name of the section, as specified in the binary. ++ * ++ * Can be NULL. ++ */ ++ const char *name; ++ ++ /** ++ * @data: Initial data copied to the FW memory. ++ * ++ * We keep data around so we can reload sections after a reset. ++ */ ++ struct { ++ /** @buf: Buffed used to store init data. */ ++ const void *buf; ++ ++ /** @size: Size of @buf in bytes. */ ++ size_t size; ++ } data; ++}; ++ ++#define CSF_MCU_SHARED_REGION_START 0x04000000ULL ++#define CSF_MCU_SHARED_REGION_SIZE 0x04000000ULL ++ ++#define MIN_CS_PER_CSG 8 ++#define MIN_CSGS 3 ++#define MAX_CSG_PRIO 0xf ++ ++#define CSF_IFACE_VERSION(major, minor, patch) \ ++ (((major) << 24) | ((minor) << 16) | (patch)) ++#define CSF_IFACE_VERSION_MAJOR(v) ((v) >> 24) ++#define CSF_IFACE_VERSION_MINOR(v) (((v) >> 16) & 0xff) ++#define CSF_IFACE_VERSION_PATCH(v) ((v) & 0xffff) ++ ++#define CSF_GROUP_CONTROL_OFFSET 0x1000 ++#define CSF_STREAM_CONTROL_OFFSET 0x40 ++#define CSF_UNPRESERVED_REG_COUNT 4 ++ ++/** ++ * struct panthor_fw_iface - FW interfaces ++ */ ++struct panthor_fw_iface { ++ /** @global: Global interface. */ ++ struct panthor_fw_global_iface global; ++ ++ /** @groups: Group slot interfaces. */ ++ struct panthor_fw_csg_iface groups[MAX_CSGS]; ++ ++ /** @streams: Command stream slot interfaces. */ ++ struct panthor_fw_cs_iface streams[MAX_CSGS][MAX_CS_PER_CSG]; ++}; ++ ++/** ++ * struct panthor_fw - Firmware management ++ */ ++struct panthor_fw { ++ /** @vm: MCU VM. */ ++ struct panthor_vm *vm; ++ ++ /** @sections: List of FW sections. */ ++ struct list_head sections; ++ ++ /** @shared_section: The section containing the FW interfaces. */ ++ struct panthor_fw_section *shared_section; ++ ++ /** @iface: FW interfaces. */ ++ struct panthor_fw_iface iface; ++ ++ /** @watchdog: Collection of fields relating to the FW watchdog. */ ++ struct { ++ /** @ping_work: Delayed work used to ping the FW. */ ++ struct delayed_work ping_work; ++ } watchdog; ++ ++ /** ++ * @req_waitqueue: FW request waitqueue. ++ * ++ * Everytime a request is sent to a command stream group or the global ++ * interface, the caller will first busy wait for the request to be ++ * acknowledged, and then fallback to a sleeping wait. ++ * ++ * This wait queue is here to support the sleeping wait flavor. ++ */ ++ wait_queue_head_t req_waitqueue; ++ ++ /** @booted: True is the FW is booted */ ++ bool booted; ++ ++ /** ++ * @fast_reset: True if the post_reset logic can proceed with a fast reset. ++ * ++ * A fast reset is just a reset where the driver doesn't reload the FW sections. ++ * ++ * Any time the firmware is properly suspended, a fast reset can take place. ++ * On the other hand, if the halt operation failed, the driver will reload ++ * all sections to make sure we start from a fresh state. ++ */ ++ bool fast_reset; ++ ++ /** @irq: Job irq data. */ ++ struct panthor_irq irq; ++}; ++ ++struct panthor_vm *panthor_fw_vm(struct panthor_device *ptdev) ++{ ++ return ptdev->fw->vm; ++} ++ ++/** ++ * panthor_fw_get_glb_iface() - Get the global interface ++ * @ptdev: Device. ++ * ++ * Return: The global interface. ++ */ ++struct panthor_fw_global_iface * ++panthor_fw_get_glb_iface(struct panthor_device *ptdev) ++{ ++ return &ptdev->fw->iface.global; ++} ++ ++/** ++ * panthor_fw_get_csg_iface() - Get a command stream group slot interface ++ * @ptdev: Device. ++ * @csg_slot: Index of the command stream group slot. ++ * ++ * Return: The command stream group slot interface. ++ */ ++struct panthor_fw_csg_iface * ++panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot) ++{ ++ if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS)) ++ return NULL; ++ ++ return &ptdev->fw->iface.groups[csg_slot]; ++} ++ ++/** ++ * panthor_fw_get_cs_iface() - Get a command stream slot interface ++ * @ptdev: Device. ++ * @csg_slot: Index of the command stream group slot. ++ * @cs_slot: Index of the command stream slot. ++ * ++ * Return: The command stream slot interface. ++ */ ++struct panthor_fw_cs_iface * ++panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot) ++{ ++ if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS || cs_slot > MAX_CS_PER_CSG)) ++ return NULL; ++ ++ return &ptdev->fw->iface.streams[csg_slot][cs_slot]; ++} ++ ++/** ++ * panthor_fw_conv_timeout() - Convert a timeout into a cycle-count ++ * @ptdev: Device. ++ * @timeout_us: Timeout expressed in micro-seconds. ++ * ++ * The FW has two timer sources: the GPU counter or arch-timer. We need ++ * to express timeouts in term of number of cycles and specify which ++ * timer source should be used. ++ * ++ * Return: A value suitable for timeout fields in the global interface. ++ */ ++static u32 panthor_fw_conv_timeout(struct panthor_device *ptdev, u32 timeout_us) ++{ ++ bool use_cycle_counter = false; ++ u32 timer_rate = 0; ++ u64 mod_cycles; ++ ++#ifdef CONFIG_ARM_ARCH_TIMER ++ timer_rate = arch_timer_get_cntfrq(); ++#endif ++ ++ if (!timer_rate) { ++ use_cycle_counter = true; ++ timer_rate = clk_get_rate(ptdev->clks.core); ++ } ++ ++ if (drm_WARN_ON(&ptdev->base, !timer_rate)) { ++ /* We couldn't get a valid clock rate, let's just pick the ++ * maximum value so the FW still handles the core ++ * power on/off requests. ++ */ ++ return GLB_TIMER_VAL(~0) | ++ GLB_TIMER_SOURCE_GPU_COUNTER; ++ } ++ ++ mod_cycles = DIV_ROUND_UP_ULL((u64)timeout_us * timer_rate, ++ 1000000ull << 10); ++ if (drm_WARN_ON(&ptdev->base, mod_cycles > GLB_TIMER_VAL(~0))) ++ mod_cycles = GLB_TIMER_VAL(~0); ++ ++ return GLB_TIMER_VAL(mod_cycles) | ++ (use_cycle_counter ? GLB_TIMER_SOURCE_GPU_COUNTER : 0); ++} ++ ++static int panthor_fw_binary_iter_read(struct panthor_device *ptdev, ++ struct panthor_fw_binary_iter *iter, ++ void *out, size_t size) ++{ ++ size_t new_offset = iter->offset + size; ++ ++ if (new_offset > iter->size || new_offset < iter->offset) { ++ drm_err(&ptdev->base, "Firmware too small\n"); ++ return -EINVAL; ++ } ++ ++ memcpy(out, iter->data + iter->offset, size); ++ iter->offset = new_offset; ++ return 0; ++} ++ ++static int panthor_fw_binary_sub_iter_init(struct panthor_device *ptdev, ++ struct panthor_fw_binary_iter *iter, ++ struct panthor_fw_binary_iter *sub_iter, ++ size_t size) ++{ ++ size_t new_offset = iter->offset + size; ++ ++ if (new_offset > iter->size || new_offset < iter->offset) { ++ drm_err(&ptdev->base, "Firmware entry too long\n"); ++ return -EINVAL; ++ } ++ ++ sub_iter->offset = 0; ++ sub_iter->data = iter->data + iter->offset; ++ sub_iter->size = size; ++ iter->offset = new_offset; ++ return 0; ++} ++ ++static void panthor_fw_init_section_mem(struct panthor_device *ptdev, ++ struct panthor_fw_section *section) ++{ ++ bool was_mapped = !!section->mem->kmap; ++ int ret; ++ ++ if (!section->data.size && ++ !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO)) ++ return; ++ ++ ret = panthor_kernel_bo_vmap(section->mem); ++ if (drm_WARN_ON(&ptdev->base, ret)) ++ return; ++ ++ memcpy(section->mem->kmap, section->data.buf, section->data.size); ++ if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_ZERO) { ++ memset(section->mem->kmap + section->data.size, 0, ++ panthor_kernel_bo_size(section->mem) - section->data.size); ++ } ++ ++ if (!was_mapped) ++ panthor_kernel_bo_vunmap(section->mem); ++} ++ ++/** ++ * panthor_fw_alloc_queue_iface_mem() - Allocate a ring-buffer interfaces. ++ * @ptdev: Device. ++ * @input: Pointer holding the input interface on success. ++ * Should be ignored on failure. ++ * @output: Pointer holding the output interface on success. ++ * Should be ignored on failure. ++ * @input_fw_va: Pointer holding the input interface FW VA on success. ++ * Should be ignored on failure. ++ * @output_fw_va: Pointer holding the output interface FW VA on success. ++ * Should be ignored on failure. ++ * ++ * Allocates panthor_fw_ringbuf_{input,out}_iface interfaces. The input ++ * interface is at offset 0, and the output interface at offset 4096. ++ * ++ * Return: A valid pointer in case of success, an ERR_PTR() otherwise. ++ */ ++struct panthor_kernel_bo * ++panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev, ++ struct panthor_fw_ringbuf_input_iface **input, ++ const struct panthor_fw_ringbuf_output_iface **output, ++ u32 *input_fw_va, u32 *output_fw_va) ++{ ++ struct panthor_kernel_bo *mem; ++ int ret; ++ ++ mem = panthor_kernel_bo_create(ptdev, ptdev->fw->vm, SZ_8K, ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++ if (IS_ERR(mem)) ++ return mem; ++ ++ ret = panthor_kernel_bo_vmap(mem); ++ if (ret) { ++ panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), mem); ++ return ERR_PTR(ret); ++ } ++ ++ memset(mem->kmap, 0, panthor_kernel_bo_size(mem)); ++ *input = mem->kmap; ++ *output = mem->kmap + SZ_4K; ++ *input_fw_va = panthor_kernel_bo_gpuva(mem); ++ *output_fw_va = *input_fw_va + SZ_4K; ++ ++ return mem; ++} ++ ++/** ++ * panthor_fw_alloc_suspend_buf_mem() - Allocate a suspend buffer for a command stream group. ++ * @ptdev: Device. ++ * @size: Size of the suspend buffer. ++ * ++ * Return: A valid pointer in case of success, an ERR_PTR() otherwise. ++ */ ++struct panthor_kernel_bo * ++panthor_fw_alloc_suspend_buf_mem(struct panthor_device *ptdev, size_t size) ++{ ++ return panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev), size, ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++} ++ ++static int panthor_fw_load_section_entry(struct panthor_device *ptdev, ++ const struct firmware *fw, ++ struct panthor_fw_binary_iter *iter, ++ u32 ehdr) ++{ ++ struct panthor_fw_binary_section_entry_hdr hdr; ++ struct panthor_fw_section *section; ++ u32 section_size; ++ u32 name_len; ++ int ret; ++ ++ ret = panthor_fw_binary_iter_read(ptdev, iter, &hdr, sizeof(hdr)); ++ if (ret) ++ return ret; ++ ++ if (hdr.data.end < hdr.data.start) { ++ drm_err(&ptdev->base, "Firmware corrupted, data.end < data.start (0x%x < 0x%x)\n", ++ hdr.data.end, hdr.data.start); ++ return -EINVAL; ++ } ++ ++ if (hdr.va.end < hdr.va.start) { ++ drm_err(&ptdev->base, "Firmware corrupted, hdr.va.end < hdr.va.start (0x%x < 0x%x)\n", ++ hdr.va.end, hdr.va.start); ++ return -EINVAL; ++ } ++ ++ if (hdr.data.end > fw->size) { ++ drm_err(&ptdev->base, "Firmware corrupted, file truncated? data_end=0x%x > fw size=0x%zx\n", ++ hdr.data.end, fw->size); ++ return -EINVAL; ++ } ++ ++ if ((hdr.va.start & ~PAGE_MASK) != 0 || ++ (hdr.va.end & ~PAGE_MASK) != 0) { ++ drm_err(&ptdev->base, "Firmware corrupted, virtual addresses not page aligned: 0x%x-0x%x\n", ++ hdr.va.start, hdr.va.end); ++ return -EINVAL; ++ } ++ ++ if (hdr.flags & ~CSF_FW_BINARY_IFACE_ENTRY_RD_SUPPORTED_FLAGS) { ++ drm_err(&ptdev->base, "Firmware contains interface with unsupported flags (0x%x)\n", ++ hdr.flags); ++ return -EINVAL; ++ } ++ ++ if (hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_PROT) { ++ drm_warn(&ptdev->base, ++ "Firmware protected mode entry not be supported, ignoring"); ++ return 0; ++ } ++ ++ if (hdr.va.start == CSF_MCU_SHARED_REGION_START && ++ !(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED)) { ++ drm_err(&ptdev->base, ++ "Interface at 0x%llx must be shared", CSF_MCU_SHARED_REGION_START); ++ return -EINVAL; ++ } ++ ++ name_len = iter->size - iter->offset; ++ ++ section = drmm_kzalloc(&ptdev->base, sizeof(*section), GFP_KERNEL); ++ if (!section) ++ return -ENOMEM; ++ ++ list_add_tail(§ion->node, &ptdev->fw->sections); ++ section->flags = hdr.flags; ++ section->data.size = hdr.data.end - hdr.data.start; ++ ++ if (section->data.size > 0) { ++ void *data = drmm_kmalloc(&ptdev->base, section->data.size, GFP_KERNEL); ++ ++ if (!data) ++ return -ENOMEM; ++ ++ memcpy(data, fw->data + hdr.data.start, section->data.size); ++ section->data.buf = data; ++ } ++ ++ if (name_len > 0) { ++ char *name = drmm_kmalloc(&ptdev->base, name_len + 1, GFP_KERNEL); ++ ++ if (!name) ++ return -ENOMEM; ++ ++ memcpy(name, iter->data + iter->offset, name_len); ++ name[name_len] = '\0'; ++ section->name = name; ++ } ++ ++ section_size = hdr.va.end - hdr.va.start; ++ if (section_size) { ++ u32 cache_mode = hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_MASK; ++ struct panthor_gem_object *bo; ++ u32 vm_map_flags = 0; ++ struct sg_table *sgt; ++ u64 va = hdr.va.start; ++ ++ if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_WR)) ++ vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_READONLY; ++ ++ if (!(hdr.flags & CSF_FW_BINARY_IFACE_ENTRY_RD_EX)) ++ vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC; ++ ++ /* TODO: CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_*_COHERENT are mapped to ++ * non-cacheable for now. We might want to introduce a new ++ * IOMMU_xxx flag (or abuse IOMMU_MMIO, which maps to device ++ * memory and is currently not used by our driver) for ++ * AS_MEMATTR_AARCH64_SHARED memory, so we can take benefit ++ * of IO-coherent systems. ++ */ ++ if (cache_mode != CSF_FW_BINARY_IFACE_ENTRY_RD_CACHE_MODE_CACHED) ++ vm_map_flags |= DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED; ++ ++ section->mem = panthor_kernel_bo_create(ptdev, panthor_fw_vm(ptdev), ++ section_size, ++ DRM_PANTHOR_BO_NO_MMAP, ++ vm_map_flags, va); ++ if (IS_ERR(section->mem)) ++ return PTR_ERR(section->mem); ++ ++ if (drm_WARN_ON(&ptdev->base, section->mem->va_node.start != hdr.va.start)) ++ return -EINVAL; ++ ++ if (section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_SHARED) { ++ ret = panthor_kernel_bo_vmap(section->mem); ++ if (ret) ++ return ret; ++ } ++ ++ panthor_fw_init_section_mem(ptdev, section); ++ ++ bo = to_panthor_bo(section->mem->obj); ++ sgt = drm_gem_shmem_get_pages_sgt(&bo->base); ++ if (IS_ERR(sgt)) ++ return PTR_ERR(sgt); ++ ++ dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE); ++ } ++ ++ if (hdr.va.start == CSF_MCU_SHARED_REGION_START) ++ ptdev->fw->shared_section = section; ++ ++ return 0; ++} ++ ++static void ++panthor_reload_fw_sections(struct panthor_device *ptdev, bool full_reload) ++{ ++ struct panthor_fw_section *section; ++ ++ list_for_each_entry(section, &ptdev->fw->sections, node) { ++ struct sg_table *sgt; ++ ++ if (!full_reload && !(section->flags & CSF_FW_BINARY_IFACE_ENTRY_RD_WR)) ++ continue; ++ ++ panthor_fw_init_section_mem(ptdev, section); ++ sgt = drm_gem_shmem_get_pages_sgt(&to_panthor_bo(section->mem->obj)->base); ++ if (!drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(sgt))) ++ dma_sync_sgtable_for_device(ptdev->base.dev, sgt, DMA_TO_DEVICE); ++ } ++} ++ ++static int panthor_fw_load_entry(struct panthor_device *ptdev, ++ const struct firmware *fw, ++ struct panthor_fw_binary_iter *iter) ++{ ++ struct panthor_fw_binary_iter eiter; ++ u32 ehdr; ++ int ret; ++ ++ ret = panthor_fw_binary_iter_read(ptdev, iter, &ehdr, sizeof(ehdr)); ++ if (ret) ++ return ret; ++ ++ if ((iter->offset % sizeof(u32)) || ++ (CSF_FW_BINARY_ENTRY_SIZE(ehdr) % sizeof(u32))) { ++ drm_err(&ptdev->base, "Firmware entry isn't 32 bit aligned, offset=0x%x size=0x%x\n", ++ (u32)(iter->offset - sizeof(u32)), CSF_FW_BINARY_ENTRY_SIZE(ehdr)); ++ return -EINVAL; ++ } ++ ++ if (panthor_fw_binary_sub_iter_init(ptdev, iter, &eiter, ++ CSF_FW_BINARY_ENTRY_SIZE(ehdr) - sizeof(ehdr))) ++ return -EINVAL; ++ ++ switch (CSF_FW_BINARY_ENTRY_TYPE(ehdr)) { ++ case CSF_FW_BINARY_ENTRY_TYPE_IFACE: ++ return panthor_fw_load_section_entry(ptdev, fw, &eiter, ehdr); ++ ++ /* FIXME: handle those entry types? */ ++ case CSF_FW_BINARY_ENTRY_TYPE_CONFIG: ++ case CSF_FW_BINARY_ENTRY_TYPE_FUTF_TEST: ++ case CSF_FW_BINARY_ENTRY_TYPE_TRACE_BUFFER: ++ case CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: ++ return 0; ++ default: ++ break; ++ } ++ ++ if (ehdr & CSF_FW_BINARY_ENTRY_OPTIONAL) ++ return 0; ++ ++ drm_err(&ptdev->base, ++ "Unsupported non-optional entry type %u in firmware\n", ++ CSF_FW_BINARY_ENTRY_TYPE(ehdr)); ++ return -EINVAL; ++} ++ ++static int panthor_fw_load(struct panthor_device *ptdev) ++{ ++ const struct firmware *fw = NULL; ++ struct panthor_fw_binary_iter iter = {}; ++ struct panthor_fw_binary_hdr hdr; ++ char fw_path[128]; ++ int ret; ++ ++ snprintf(fw_path, sizeof(fw_path), "arm/mali/arch%d.%d/%s", ++ (u32)GPU_ARCH_MAJOR(ptdev->gpu_info.gpu_id), ++ (u32)GPU_ARCH_MINOR(ptdev->gpu_info.gpu_id), ++ CSF_FW_NAME); ++ ++ ret = request_firmware(&fw, fw_path, ptdev->base.dev); ++ if (ret) { ++ drm_err(&ptdev->base, "Failed to load firmware image '%s'\n", ++ CSF_FW_NAME); ++ return ret; ++ } ++ ++ iter.data = fw->data; ++ iter.size = fw->size; ++ ret = panthor_fw_binary_iter_read(ptdev, &iter, &hdr, sizeof(hdr)); ++ if (ret) ++ goto out; ++ ++ if (hdr.magic != CSF_FW_BINARY_HEADER_MAGIC) { ++ ret = -EINVAL; ++ drm_err(&ptdev->base, "Invalid firmware magic\n"); ++ goto out; ++ } ++ ++ if (hdr.major != CSF_FW_BINARY_HEADER_MAJOR_MAX) { ++ ret = -EINVAL; ++ drm_err(&ptdev->base, "Unsupported firmware binary header version %d.%d (expected %d.x)\n", ++ hdr.major, hdr.minor, CSF_FW_BINARY_HEADER_MAJOR_MAX); ++ goto out; ++ } ++ ++ if (hdr.size > iter.size) { ++ drm_err(&ptdev->base, "Firmware image is truncated\n"); ++ goto out; ++ } ++ ++ iter.size = hdr.size; ++ ++ while (iter.offset < hdr.size) { ++ ret = panthor_fw_load_entry(ptdev, fw, &iter); ++ if (ret) ++ goto out; ++ } ++ ++ if (!ptdev->fw->shared_section) { ++ drm_err(&ptdev->base, "Shared interface region not found\n"); ++ ret = -EINVAL; ++ goto out; ++ } ++ ++out: ++ release_firmware(fw); ++ return ret; ++} ++ ++/** ++ * iface_fw_to_cpu_addr() - Turn an MCU address into a CPU address ++ * @ptdev: Device. ++ * @mcu_va: MCU address. ++ * ++ * Return: NULL if the address is not part of the shared section, non-NULL otherwise. ++ */ ++static void *iface_fw_to_cpu_addr(struct panthor_device *ptdev, u32 mcu_va) ++{ ++ u64 shared_mem_start = panthor_kernel_bo_gpuva(ptdev->fw->shared_section->mem); ++ u64 shared_mem_end = shared_mem_start + ++ panthor_kernel_bo_size(ptdev->fw->shared_section->mem); ++ if (mcu_va < shared_mem_start || mcu_va >= shared_mem_end) ++ return NULL; ++ ++ return ptdev->fw->shared_section->mem->kmap + (mcu_va - shared_mem_start); ++} ++ ++static int panthor_init_cs_iface(struct panthor_device *ptdev, ++ unsigned int csg_idx, unsigned int cs_idx) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_idx); ++ struct panthor_fw_cs_iface *cs_iface = &ptdev->fw->iface.streams[csg_idx][cs_idx]; ++ u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); ++ u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + ++ (csg_idx * glb_iface->control->group_stride) + ++ CSF_STREAM_CONTROL_OFFSET + ++ (cs_idx * csg_iface->control->stream_stride); ++ struct panthor_fw_cs_iface *first_cs_iface = ++ panthor_fw_get_cs_iface(ptdev, 0, 0); ++ ++ if (iface_offset + sizeof(*cs_iface) >= shared_section_sz) ++ return -EINVAL; ++ ++ spin_lock_init(&cs_iface->lock); ++ cs_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; ++ cs_iface->input = iface_fw_to_cpu_addr(ptdev, cs_iface->control->input_va); ++ cs_iface->output = iface_fw_to_cpu_addr(ptdev, cs_iface->control->output_va); ++ ++ if (!cs_iface->input || !cs_iface->output) { ++ drm_err(&ptdev->base, "Invalid stream control interface input/output VA"); ++ return -EINVAL; ++ } ++ ++ if (cs_iface != first_cs_iface) { ++ if (cs_iface->control->features != first_cs_iface->control->features) { ++ drm_err(&ptdev->base, "Expecting identical CS slots"); ++ return -EINVAL; ++ } ++ } else { ++ u32 reg_count = CS_FEATURES_WORK_REGS(cs_iface->control->features); ++ ++ ptdev->csif_info.cs_reg_count = reg_count; ++ ptdev->csif_info.unpreserved_cs_reg_count = CSF_UNPRESERVED_REG_COUNT; ++ } ++ ++ return 0; ++} ++ ++static bool compare_csg(const struct panthor_fw_csg_control_iface *a, ++ const struct panthor_fw_csg_control_iface *b) ++{ ++ if (a->features != b->features) ++ return false; ++ if (a->suspend_size != b->suspend_size) ++ return false; ++ if (a->protm_suspend_size != b->protm_suspend_size) ++ return false; ++ if (a->stream_num != b->stream_num) ++ return false; ++ return true; ++} ++ ++static int panthor_init_csg_iface(struct panthor_device *ptdev, ++ unsigned int csg_idx) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ struct panthor_fw_csg_iface *csg_iface = &ptdev->fw->iface.groups[csg_idx]; ++ u64 shared_section_sz = panthor_kernel_bo_size(ptdev->fw->shared_section->mem); ++ u32 iface_offset = CSF_GROUP_CONTROL_OFFSET + (csg_idx * glb_iface->control->group_stride); ++ unsigned int i; ++ ++ if (iface_offset + sizeof(*csg_iface) >= shared_section_sz) ++ return -EINVAL; ++ ++ spin_lock_init(&csg_iface->lock); ++ csg_iface->control = ptdev->fw->shared_section->mem->kmap + iface_offset; ++ csg_iface->input = iface_fw_to_cpu_addr(ptdev, csg_iface->control->input_va); ++ csg_iface->output = iface_fw_to_cpu_addr(ptdev, csg_iface->control->output_va); ++ ++ if (csg_iface->control->stream_num < MIN_CS_PER_CSG || ++ csg_iface->control->stream_num > MAX_CS_PER_CSG) ++ return -EINVAL; ++ ++ if (!csg_iface->input || !csg_iface->output) { ++ drm_err(&ptdev->base, "Invalid group control interface input/output VA"); ++ return -EINVAL; ++ } ++ ++ if (csg_idx > 0) { ++ struct panthor_fw_csg_iface *first_csg_iface = ++ panthor_fw_get_csg_iface(ptdev, 0); ++ ++ if (!compare_csg(first_csg_iface->control, csg_iface->control)) { ++ drm_err(&ptdev->base, "Expecting identical CSG slots"); ++ return -EINVAL; ++ } ++ } ++ ++ for (i = 0; i < csg_iface->control->stream_num; i++) { ++ int ret = panthor_init_cs_iface(ptdev, csg_idx, i); ++ ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static u32 panthor_get_instr_features(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ ++ if (glb_iface->control->version < CSF_IFACE_VERSION(1, 1, 0)) ++ return 0; ++ ++ return glb_iface->control->instr_features; ++} ++ ++static int panthor_fw_init_ifaces(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = &ptdev->fw->iface.global; ++ unsigned int i; ++ ++ if (!ptdev->fw->shared_section->mem->kmap) ++ return -EINVAL; ++ ++ spin_lock_init(&glb_iface->lock); ++ glb_iface->control = ptdev->fw->shared_section->mem->kmap; ++ ++ if (!glb_iface->control->version) { ++ drm_err(&ptdev->base, "Firmware version is 0. Firmware may have failed to boot"); ++ return -EINVAL; ++ } ++ ++ glb_iface->input = iface_fw_to_cpu_addr(ptdev, glb_iface->control->input_va); ++ glb_iface->output = iface_fw_to_cpu_addr(ptdev, glb_iface->control->output_va); ++ if (!glb_iface->input || !glb_iface->output) { ++ drm_err(&ptdev->base, "Invalid global control interface input/output VA"); ++ return -EINVAL; ++ } ++ ++ if (glb_iface->control->group_num > MAX_CSGS || ++ glb_iface->control->group_num < MIN_CSGS) { ++ drm_err(&ptdev->base, "Invalid number of control groups"); ++ return -EINVAL; ++ } ++ ++ for (i = 0; i < glb_iface->control->group_num; i++) { ++ int ret = panthor_init_csg_iface(ptdev, i); ++ ++ if (ret) ++ return ret; ++ } ++ ++ drm_info(&ptdev->base, "CSF FW v%d.%d.%d, Features %#x Instrumentation features %#x", ++ CSF_IFACE_VERSION_MAJOR(glb_iface->control->version), ++ CSF_IFACE_VERSION_MINOR(glb_iface->control->version), ++ CSF_IFACE_VERSION_PATCH(glb_iface->control->version), ++ glb_iface->control->features, ++ panthor_get_instr_features(ptdev)); ++ return 0; ++} ++ ++static void panthor_fw_init_global_iface(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ ++ /* Enable all cores. */ ++ glb_iface->input->core_en_mask = ptdev->gpu_info.shader_present; ++ ++ /* Setup timers. */ ++ glb_iface->input->poweroff_timer = panthor_fw_conv_timeout(ptdev, PWROFF_HYSTERESIS_US); ++ glb_iface->input->progress_timer = PROGRESS_TIMEOUT_CYCLES >> PROGRESS_TIMEOUT_SCALE_SHIFT; ++ glb_iface->input->idle_timer = panthor_fw_conv_timeout(ptdev, IDLE_HYSTERESIS_US); ++ ++ /* Enable interrupts we care about. */ ++ glb_iface->input->ack_irq_mask = GLB_CFG_ALLOC_EN | ++ GLB_PING | ++ GLB_CFG_PROGRESS_TIMER | ++ GLB_CFG_POWEROFF_TIMER | ++ GLB_IDLE_EN | ++ GLB_IDLE; ++ ++ panthor_fw_update_reqs(glb_iface, req, GLB_IDLE_EN, GLB_IDLE_EN); ++ panthor_fw_toggle_reqs(glb_iface, req, ack, ++ GLB_CFG_ALLOC_EN | ++ GLB_CFG_POWEROFF_TIMER | ++ GLB_CFG_PROGRESS_TIMER); ++ ++ gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); ++ ++ /* Kick the watchdog. */ ++ mod_delayed_work(ptdev->reset.wq, &ptdev->fw->watchdog.ping_work, ++ msecs_to_jiffies(PING_INTERVAL_MS)); ++} ++ ++static void panthor_job_irq_handler(struct panthor_device *ptdev, u32 status) ++{ ++ if (!ptdev->fw->booted && (status & JOB_INT_GLOBAL_IF)) ++ ptdev->fw->booted = true; ++ ++ wake_up_all(&ptdev->fw->req_waitqueue); ++ ++ /* If the FW is not booted, don't process IRQs, just flag the FW as booted. */ ++ if (!ptdev->fw->booted) ++ return; ++ ++ panthor_sched_report_fw_events(ptdev, status); ++} ++PANTHOR_IRQ_HANDLER(job, JOB, panthor_job_irq_handler); ++ ++static int panthor_fw_start(struct panthor_device *ptdev) ++{ ++ bool timedout = false; ++ ++ ptdev->fw->booted = false; ++ panthor_job_irq_resume(&ptdev->fw->irq, ~0); ++ gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_AUTO); ++ ++ if (!wait_event_timeout(ptdev->fw->req_waitqueue, ++ ptdev->fw->booted, ++ msecs_to_jiffies(1000))) { ++ if (!ptdev->fw->booted && ++ !(gpu_read(ptdev, JOB_INT_STAT) & JOB_INT_GLOBAL_IF)) ++ timedout = true; ++ } ++ ++ if (timedout) { ++ static const char * const status_str[] = { ++ [MCU_STATUS_DISABLED] = "disabled", ++ [MCU_STATUS_ENABLED] = "enabled", ++ [MCU_STATUS_HALT] = "halt", ++ [MCU_STATUS_FATAL] = "fatal", ++ }; ++ u32 status = gpu_read(ptdev, MCU_STATUS); ++ ++ drm_err(&ptdev->base, "Failed to boot MCU (status=%s)", ++ status < ARRAY_SIZE(status_str) ? status_str[status] : "unknown"); ++ return -ETIMEDOUT; ++ } ++ ++ return 0; ++} ++ ++static void panthor_fw_stop(struct panthor_device *ptdev) ++{ ++ u32 status; ++ ++ gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE); ++ if (readl_poll_timeout(ptdev->iomem + MCU_STATUS, status, ++ status == MCU_STATUS_DISABLED, 10, 100000)) ++ drm_err(&ptdev->base, "Failed to stop MCU"); ++} ++ ++/** ++ * panthor_fw_pre_reset() - Call before a reset. ++ * @ptdev: Device. ++ * @on_hang: true if the reset was triggered on a GPU hang. ++ * ++ * If the reset is not triggered on a hang, we try to gracefully halt the ++ * MCU, so we can do a fast-reset when panthor_fw_post_reset() is called. ++ */ ++void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang) ++{ ++ /* Make sure we won't be woken up by a ping. */ ++ cancel_delayed_work_sync(&ptdev->fw->watchdog.ping_work); ++ ++ ptdev->fw->fast_reset = false; ++ ++ if (!on_hang) { ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ u32 status; ++ ++ panthor_fw_update_reqs(glb_iface, req, GLB_HALT, GLB_HALT); ++ gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); ++ if (!readl_poll_timeout(ptdev->iomem + MCU_STATUS, status, ++ status == MCU_STATUS_HALT, 10, 100000) && ++ glb_iface->output->halt_status == PANTHOR_FW_HALT_OK) { ++ ptdev->fw->fast_reset = true; ++ } else { ++ drm_warn(&ptdev->base, "Failed to cleanly suspend MCU"); ++ } ++ ++ /* The FW detects 0 -> 1 transitions. Make sure we reset ++ * the HALT bit before the FW is rebooted. ++ */ ++ panthor_fw_update_reqs(glb_iface, req, 0, GLB_HALT); ++ } ++ ++ panthor_job_irq_suspend(&ptdev->fw->irq); ++} ++ ++/** ++ * panthor_fw_post_reset() - Call after a reset. ++ * @ptdev: Device. ++ * ++ * Start the FW. If this is not a fast reset, all FW sections are reloaded to ++ * make sure we can recover from a memory corruption. ++ */ ++int panthor_fw_post_reset(struct panthor_device *ptdev) ++{ ++ int ret; ++ ++ /* Make the MCU VM active. */ ++ ret = panthor_vm_active(ptdev->fw->vm); ++ if (ret) ++ return ret; ++ ++ /* If this is a fast reset, try to start the MCU without reloading ++ * the FW sections. If it fails, go for a full reset. ++ */ ++ if (ptdev->fw->fast_reset) { ++ ret = panthor_fw_start(ptdev); ++ if (!ret) ++ goto out; ++ ++ /* Force a disable, so we get a fresh boot on the next ++ * panthor_fw_start() call. ++ */ ++ gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE); ++ drm_err(&ptdev->base, "FW fast reset failed, trying a slow reset"); ++ } ++ ++ /* Reload all sections, including RO ones. We're not supposed ++ * to end up here anyway, let's just assume the overhead of ++ * reloading everything is acceptable. ++ */ ++ panthor_reload_fw_sections(ptdev, true); ++ ++ ret = panthor_fw_start(ptdev); ++ if (ret) { ++ drm_err(&ptdev->base, "FW slow reset failed"); ++ return ret; ++ } ++ ++out: ++ /* We must re-initialize the global interface even on fast-reset. */ ++ panthor_fw_init_global_iface(ptdev); ++ return 0; ++} ++ ++/** ++ * panthor_fw_unplug() - Called when the device is unplugged. ++ * @ptdev: Device. ++ * ++ * This function must make sure all pending operations are flushed before ++ * will release device resources, thus preventing any interaction with ++ * the HW. ++ * ++ * If there is still FW-related work running after this function returns, ++ * they must use drm_dev_{enter,exit}() and skip any HW access when ++ * drm_dev_enter() returns false. ++ */ ++void panthor_fw_unplug(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_section *section; ++ ++ cancel_delayed_work_sync(&ptdev->fw->watchdog.ping_work); ++ ++ /* Make sure the IRQ handler can be called after that point. */ ++ if (ptdev->fw->irq.irq) ++ panthor_job_irq_suspend(&ptdev->fw->irq); ++ ++ panthor_fw_stop(ptdev); ++ ++ list_for_each_entry(section, &ptdev->fw->sections, node) ++ panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), section->mem); ++ ++ /* We intentionally don't call panthor_vm_idle() and let ++ * panthor_mmu_unplug() release the AS we acquired with ++ * panthor_vm_active() so we don't have to track the VM active/idle ++ * state to keep the active_refcnt balanced. ++ */ ++ panthor_vm_put(ptdev->fw->vm); ++ ++ panthor_gpu_power_off(ptdev, L2, ptdev->gpu_info.l2_present, 20000); ++} ++ ++/** ++ * panthor_fw_wait_acks() - Wait for requests to be acknowledged by the FW. ++ * @req_ptr: Pointer to the req register. ++ * @ack_ptr: Pointer to the ack register. ++ * @wq: Wait queue to use for the sleeping wait. ++ * @req_mask: Mask of requests to wait for. ++ * @acked: Pointer to field that's updated with the acked requests. ++ * If the function returns 0, *acked == req_mask. ++ * @timeout_ms: Timeout expressed in milliseconds. ++ * ++ * Return: 0 on success, -ETIMEDOUT otherwise. ++ */ ++static int panthor_fw_wait_acks(const u32 *req_ptr, const u32 *ack_ptr, ++ wait_queue_head_t *wq, ++ u32 req_mask, u32 *acked, ++ u32 timeout_ms) ++{ ++ u32 ack, req = READ_ONCE(*req_ptr) & req_mask; ++ int ret; ++ ++ /* Busy wait for a few µsecs before falling back to a sleeping wait. */ ++ *acked = req_mask; ++ ret = read_poll_timeout_atomic(READ_ONCE, ack, ++ (ack & req_mask) == req, ++ 0, 10, 0, ++ *ack_ptr); ++ if (!ret) ++ return 0; ++ ++ if (wait_event_timeout(*wq, (READ_ONCE(*ack_ptr) & req_mask) == req, ++ msecs_to_jiffies(timeout_ms))) ++ return 0; ++ ++ /* Check one last time, in case we were not woken up for some reason. */ ++ ack = READ_ONCE(*ack_ptr); ++ if ((ack & req_mask) == req) ++ return 0; ++ ++ *acked = ~(req ^ ack) & req_mask; ++ return -ETIMEDOUT; ++} ++ ++/** ++ * panthor_fw_glb_wait_acks() - Wait for global requests to be acknowledged. ++ * @ptdev: Device. ++ * @req_mask: Mask of requests to wait for. ++ * @acked: Pointer to field that's updated with the acked requests. ++ * If the function returns 0, *acked == req_mask. ++ * @timeout_ms: Timeout expressed in milliseconds. ++ * ++ * Return: 0 on success, -ETIMEDOUT otherwise. ++ */ ++int panthor_fw_glb_wait_acks(struct panthor_device *ptdev, ++ u32 req_mask, u32 *acked, ++ u32 timeout_ms) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ ++ /* GLB_HALT doesn't get acked through the FW interface. */ ++ if (drm_WARN_ON(&ptdev->base, req_mask & (~GLB_REQ_MASK | GLB_HALT))) ++ return -EINVAL; ++ ++ return panthor_fw_wait_acks(&glb_iface->input->req, ++ &glb_iface->output->ack, ++ &ptdev->fw->req_waitqueue, ++ req_mask, acked, timeout_ms); ++} ++ ++/** ++ * panthor_fw_csg_wait_acks() - Wait for command stream group requests to be acknowledged. ++ * @ptdev: Device. ++ * @csg_slot: CSG slot ID. ++ * @req_mask: Mask of requests to wait for. ++ * @acked: Pointer to field that's updated with the acked requests. ++ * If the function returns 0, *acked == req_mask. ++ * @timeout_ms: Timeout expressed in milliseconds. ++ * ++ * Return: 0 on success, -ETIMEDOUT otherwise. ++ */ ++int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_slot, ++ u32 req_mask, u32 *acked, u32 timeout_ms) ++{ ++ struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, csg_slot); ++ int ret; ++ ++ if (drm_WARN_ON(&ptdev->base, req_mask & ~CSG_REQ_MASK)) ++ return -EINVAL; ++ ++ ret = panthor_fw_wait_acks(&csg_iface->input->req, ++ &csg_iface->output->ack, ++ &ptdev->fw->req_waitqueue, ++ req_mask, acked, timeout_ms); ++ ++ /* ++ * Check that all bits in the state field were updated, if any mismatch ++ * then clear all bits in the state field. This allows code to do ++ * (acked & CSG_STATE_MASK) and get the right value. ++ */ ++ ++ if ((*acked & CSG_STATE_MASK) != CSG_STATE_MASK) ++ *acked &= ~CSG_STATE_MASK; ++ ++ return ret; ++} ++ ++/** ++ * panthor_fw_ring_csg_doorbells() - Ring command stream group doorbells. ++ * @ptdev: Device. ++ * @csg_mask: Bitmask encoding the command stream group doorbells to ring. ++ * ++ * This function is toggling bits in the doorbell_req and ringing the ++ * global doorbell. It doesn't require a user doorbell to be attached to ++ * the group. ++ */ ++void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_mask) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ ++ panthor_fw_toggle_reqs(glb_iface, doorbell_req, doorbell_ack, csg_mask); ++ gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); ++} ++ ++static void panthor_fw_ping_work(struct work_struct *work) ++{ ++ struct panthor_fw *fw = container_of(work, struct panthor_fw, watchdog.ping_work.work); ++ struct panthor_device *ptdev = fw->irq.ptdev; ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ u32 acked; ++ int ret; ++ ++ if (panthor_device_reset_is_pending(ptdev)) ++ return; ++ ++ panthor_fw_toggle_reqs(glb_iface, req, ack, GLB_PING); ++ gpu_write(ptdev, CSF_DOORBELL(CSF_GLB_DOORBELL_ID), 1); ++ ++ ret = panthor_fw_glb_wait_acks(ptdev, GLB_PING, &acked, 100); ++ if (ret) { ++ panthor_device_schedule_reset(ptdev); ++ drm_err(&ptdev->base, "FW ping timeout, scheduling a reset"); ++ } else { ++ mod_delayed_work(ptdev->reset.wq, &fw->watchdog.ping_work, ++ msecs_to_jiffies(PING_INTERVAL_MS)); ++ } ++} ++ ++/** ++ * panthor_fw_init() - Initialize FW related data. ++ * @ptdev: Device. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++int panthor_fw_init(struct panthor_device *ptdev) ++{ ++ struct panthor_fw *fw; ++ int ret, irq; ++ ++ fw = drmm_kzalloc(&ptdev->base, sizeof(*fw), GFP_KERNEL); ++ if (!fw) ++ return -ENOMEM; ++ ++ ptdev->fw = fw; ++ init_waitqueue_head(&fw->req_waitqueue); ++ INIT_LIST_HEAD(&fw->sections); ++ INIT_DELAYED_WORK(&fw->watchdog.ping_work, panthor_fw_ping_work); ++ ++ irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "job"); ++ if (irq <= 0) ++ return -ENODEV; ++ ++ ret = panthor_request_job_irq(ptdev, &fw->irq, irq, 0); ++ if (ret) { ++ drm_err(&ptdev->base, "failed to request job irq"); ++ return ret; ++ } ++ ++ ret = panthor_gpu_l2_power_on(ptdev); ++ if (ret) ++ return ret; ++ ++ fw->vm = panthor_vm_create(ptdev, true, ++ 0, SZ_4G, ++ CSF_MCU_SHARED_REGION_START, ++ CSF_MCU_SHARED_REGION_SIZE); ++ if (IS_ERR(fw->vm)) { ++ ret = PTR_ERR(fw->vm); ++ fw->vm = NULL; ++ goto err_unplug_fw; ++ } ++ ++ ret = panthor_fw_load(ptdev); ++ if (ret) ++ goto err_unplug_fw; ++ ++ ret = panthor_vm_active(fw->vm); ++ if (ret) ++ goto err_unplug_fw; ++ ++ ret = panthor_fw_start(ptdev); ++ if (ret) ++ goto err_unplug_fw; ++ ++ ret = panthor_fw_init_ifaces(ptdev); ++ if (ret) ++ goto err_unplug_fw; ++ ++ panthor_fw_init_global_iface(ptdev); ++ return 0; ++ ++err_unplug_fw: ++ panthor_fw_unplug(ptdev); ++ return ret; ++} ++ ++MODULE_FIRMWARE("arm/mali/arch10.8/mali_csffw.bin"); +diff --git a/drivers/gpu/drm/panthor/panthor_fw.h b/drivers/gpu/drm/panthor/panthor_fw.h +new file mode 100644 +index 00000000000000..22448abde99232 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_fw.h +@@ -0,0 +1,503 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_MCU_H__ ++#define __PANTHOR_MCU_H__ ++ ++#include ++ ++struct panthor_device; ++struct panthor_kernel_bo; ++ ++#define MAX_CSGS 31 ++#define MAX_CS_PER_CSG 32 ++ ++struct panthor_fw_ringbuf_input_iface { ++ u64 insert; ++ u64 extract; ++}; ++ ++struct panthor_fw_ringbuf_output_iface { ++ u64 extract; ++ u32 active; ++}; ++ ++struct panthor_fw_cs_control_iface { ++#define CS_FEATURES_WORK_REGS(x) (((x) & GENMASK(7, 0)) + 1) ++#define CS_FEATURES_SCOREBOARDS(x) (((x) & GENMASK(15, 8)) >> 8) ++#define CS_FEATURES_COMPUTE BIT(16) ++#define CS_FEATURES_FRAGMENT BIT(17) ++#define CS_FEATURES_TILER BIT(18) ++ u32 features; ++ u32 input_va; ++ u32 output_va; ++}; ++ ++struct panthor_fw_cs_input_iface { ++#define CS_STATE_MASK GENMASK(2, 0) ++#define CS_STATE_STOP 0 ++#define CS_STATE_START 1 ++#define CS_EXTRACT_EVENT BIT(4) ++#define CS_IDLE_SYNC_WAIT BIT(8) ++#define CS_IDLE_PROTM_PENDING BIT(9) ++#define CS_IDLE_EMPTY BIT(10) ++#define CS_IDLE_RESOURCE_REQ BIT(11) ++#define CS_TILER_OOM BIT(26) ++#define CS_PROTM_PENDING BIT(27) ++#define CS_FATAL BIT(30) ++#define CS_FAULT BIT(31) ++#define CS_REQ_MASK (CS_STATE_MASK | \ ++ CS_EXTRACT_EVENT | \ ++ CS_IDLE_SYNC_WAIT | \ ++ CS_IDLE_PROTM_PENDING | \ ++ CS_IDLE_EMPTY | \ ++ CS_IDLE_RESOURCE_REQ) ++#define CS_EVT_MASK (CS_TILER_OOM | \ ++ CS_PROTM_PENDING | \ ++ CS_FATAL | \ ++ CS_FAULT) ++ u32 req; ++ ++#define CS_CONFIG_PRIORITY(x) ((x) & GENMASK(3, 0)) ++#define CS_CONFIG_DOORBELL(x) (((x) << 8) & GENMASK(15, 8)) ++ u32 config; ++ u32 reserved1; ++ u32 ack_irq_mask; ++ u64 ringbuf_base; ++ u32 ringbuf_size; ++ u32 reserved2; ++ u64 heap_start; ++ u64 heap_end; ++ u64 ringbuf_input; ++ u64 ringbuf_output; ++ u32 instr_config; ++ u32 instrbuf_size; ++ u64 instrbuf_base; ++ u64 instrbuf_offset_ptr; ++}; ++ ++struct panthor_fw_cs_output_iface { ++ u32 ack; ++ u32 reserved1[15]; ++ u64 status_cmd_ptr; ++ ++#define CS_STATUS_WAIT_SB_MASK GENMASK(15, 0) ++#define CS_STATUS_WAIT_SB_SRC_MASK GENMASK(19, 16) ++#define CS_STATUS_WAIT_SB_SRC_NONE (0 << 16) ++#define CS_STATUS_WAIT_SB_SRC_WAIT (8 << 16) ++#define CS_STATUS_WAIT_SYNC_COND_LE (0 << 24) ++#define CS_STATUS_WAIT_SYNC_COND_GT (1 << 24) ++#define CS_STATUS_WAIT_SYNC_COND_MASK GENMASK(27, 24) ++#define CS_STATUS_WAIT_PROGRESS BIT(28) ++#define CS_STATUS_WAIT_PROTM BIT(29) ++#define CS_STATUS_WAIT_SYNC_64B BIT(30) ++#define CS_STATUS_WAIT_SYNC BIT(31) ++ u32 status_wait; ++ u32 status_req_resource; ++ u64 status_wait_sync_ptr; ++ u32 status_wait_sync_value; ++ u32 status_scoreboards; ++ ++#define CS_STATUS_BLOCKED_REASON_UNBLOCKED 0 ++#define CS_STATUS_BLOCKED_REASON_SB_WAIT 1 ++#define CS_STATUS_BLOCKED_REASON_PROGRESS_WAIT 2 ++#define CS_STATUS_BLOCKED_REASON_SYNC_WAIT 3 ++#define CS_STATUS_BLOCKED_REASON_DEFERRED 5 ++#define CS_STATUS_BLOCKED_REASON_RES 6 ++#define CS_STATUS_BLOCKED_REASON_FLUSH 7 ++#define CS_STATUS_BLOCKED_REASON_MASK GENMASK(3, 0) ++ u32 status_blocked_reason; ++ u32 status_wait_sync_value_hi; ++ u32 reserved2[6]; ++ ++#define CS_EXCEPTION_TYPE(x) ((x) & GENMASK(7, 0)) ++#define CS_EXCEPTION_DATA(x) (((x) >> 8) & GENMASK(23, 0)) ++ u32 fault; ++ u32 fatal; ++ u64 fault_info; ++ u64 fatal_info; ++ u32 reserved3[10]; ++ u32 heap_vt_start; ++ u32 heap_vt_end; ++ u32 reserved4; ++ u32 heap_frag_end; ++ u64 heap_address; ++}; ++ ++struct panthor_fw_csg_control_iface { ++ u32 features; ++ u32 input_va; ++ u32 output_va; ++ u32 suspend_size; ++ u32 protm_suspend_size; ++ u32 stream_num; ++ u32 stream_stride; ++}; ++ ++struct panthor_fw_csg_input_iface { ++#define CSG_STATE_MASK GENMASK(2, 0) ++#define CSG_STATE_TERMINATE 0 ++#define CSG_STATE_START 1 ++#define CSG_STATE_SUSPEND 2 ++#define CSG_STATE_RESUME 3 ++#define CSG_ENDPOINT_CONFIG BIT(4) ++#define CSG_STATUS_UPDATE BIT(5) ++#define CSG_SYNC_UPDATE BIT(28) ++#define CSG_IDLE BIT(29) ++#define CSG_DOORBELL BIT(30) ++#define CSG_PROGRESS_TIMER_EVENT BIT(31) ++#define CSG_REQ_MASK (CSG_STATE_MASK | \ ++ CSG_ENDPOINT_CONFIG | \ ++ CSG_STATUS_UPDATE) ++#define CSG_EVT_MASK (CSG_SYNC_UPDATE | \ ++ CSG_IDLE | \ ++ CSG_PROGRESS_TIMER_EVENT) ++ u32 req; ++ u32 ack_irq_mask; ++ ++ u32 doorbell_req; ++ u32 cs_irq_ack; ++ u32 reserved1[4]; ++ u64 allow_compute; ++ u64 allow_fragment; ++ u32 allow_other; ++ ++#define CSG_EP_REQ_COMPUTE(x) ((x) & GENMASK(7, 0)) ++#define CSG_EP_REQ_FRAGMENT(x) (((x) << 8) & GENMASK(15, 8)) ++#define CSG_EP_REQ_TILER(x) (((x) << 16) & GENMASK(19, 16)) ++#define CSG_EP_REQ_EXCL_COMPUTE BIT(20) ++#define CSG_EP_REQ_EXCL_FRAGMENT BIT(21) ++#define CSG_EP_REQ_PRIORITY(x) (((x) << 28) & GENMASK(31, 28)) ++#define CSG_EP_REQ_PRIORITY_MASK GENMASK(31, 28) ++ u32 endpoint_req; ++ u32 reserved2[2]; ++ u64 suspend_buf; ++ u64 protm_suspend_buf; ++ u32 config; ++ u32 iter_trace_config; ++}; ++ ++struct panthor_fw_csg_output_iface { ++ u32 ack; ++ u32 reserved1; ++ u32 doorbell_ack; ++ u32 cs_irq_req; ++ u32 status_endpoint_current; ++ u32 status_endpoint_req; ++ ++#define CSG_STATUS_STATE_IS_IDLE BIT(0) ++ u32 status_state; ++ u32 resource_dep; ++}; ++ ++struct panthor_fw_global_control_iface { ++ u32 version; ++ u32 features; ++ u32 input_va; ++ u32 output_va; ++ u32 group_num; ++ u32 group_stride; ++ u32 perfcnt_size; ++ u32 instr_features; ++}; ++ ++struct panthor_fw_global_input_iface { ++#define GLB_HALT BIT(0) ++#define GLB_CFG_PROGRESS_TIMER BIT(1) ++#define GLB_CFG_ALLOC_EN BIT(2) ++#define GLB_CFG_POWEROFF_TIMER BIT(3) ++#define GLB_PROTM_ENTER BIT(4) ++#define GLB_PERFCNT_EN BIT(5) ++#define GLB_PERFCNT_SAMPLE BIT(6) ++#define GLB_COUNTER_EN BIT(7) ++#define GLB_PING BIT(8) ++#define GLB_FWCFG_UPDATE BIT(9) ++#define GLB_IDLE_EN BIT(10) ++#define GLB_SLEEP BIT(12) ++#define GLB_INACTIVE_COMPUTE BIT(20) ++#define GLB_INACTIVE_FRAGMENT BIT(21) ++#define GLB_INACTIVE_TILER BIT(22) ++#define GLB_PROTM_EXIT BIT(23) ++#define GLB_PERFCNT_THRESHOLD BIT(24) ++#define GLB_PERFCNT_OVERFLOW BIT(25) ++#define GLB_IDLE BIT(26) ++#define GLB_DBG_CSF BIT(30) ++#define GLB_DBG_HOST BIT(31) ++#define GLB_REQ_MASK GENMASK(10, 0) ++#define GLB_EVT_MASK GENMASK(26, 20) ++ u32 req; ++ u32 ack_irq_mask; ++ u32 doorbell_req; ++ u32 reserved1; ++ u32 progress_timer; ++ ++#define GLB_TIMER_VAL(x) ((x) & GENMASK(30, 0)) ++#define GLB_TIMER_SOURCE_GPU_COUNTER BIT(31) ++ u32 poweroff_timer; ++ u64 core_en_mask; ++ u32 reserved2; ++ u32 perfcnt_as; ++ u64 perfcnt_base; ++ u32 perfcnt_extract; ++ u32 reserved3[3]; ++ u32 perfcnt_config; ++ u32 perfcnt_csg_select; ++ u32 perfcnt_fw_enable; ++ u32 perfcnt_csg_enable; ++ u32 perfcnt_csf_enable; ++ u32 perfcnt_shader_enable; ++ u32 perfcnt_tiler_enable; ++ u32 perfcnt_mmu_l2_enable; ++ u32 reserved4[8]; ++ u32 idle_timer; ++}; ++ ++enum panthor_fw_halt_status { ++ PANTHOR_FW_HALT_OK = 0, ++ PANTHOR_FW_HALT_ON_PANIC = 0x4e, ++ PANTHOR_FW_HALT_ON_WATCHDOG_EXPIRATION = 0x4f, ++}; ++ ++struct panthor_fw_global_output_iface { ++ u32 ack; ++ u32 reserved1; ++ u32 doorbell_ack; ++ u32 reserved2; ++ u32 halt_status; ++ u32 perfcnt_status; ++ u32 perfcnt_insert; ++}; ++ ++/** ++ * struct panthor_fw_cs_iface - Firmware command stream slot interface ++ */ ++struct panthor_fw_cs_iface { ++ /** ++ * @lock: Lock protecting access to the panthor_fw_cs_input_iface::req ++ * field. ++ * ++ * Needed so we can update the req field concurrently from the interrupt ++ * handler and the scheduler logic. ++ * ++ * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW ++ * interface sections are mapped uncached/write-combined right now, and ++ * using cmpxchg() on such mappings leads to SError faults. Revisit when ++ * we have 'SHARED' GPU mappings hooked up. ++ */ ++ spinlock_t lock; ++ ++ /** ++ * @control: Command stream slot control interface. ++ * ++ * Used to expose command stream slot properties. ++ * ++ * This interface is read-only. ++ */ ++ struct panthor_fw_cs_control_iface *control; ++ ++ /** ++ * @input: Command stream slot input interface. ++ * ++ * Used for host updates/events. ++ */ ++ struct panthor_fw_cs_input_iface *input; ++ ++ /** ++ * @output: Command stream slot output interface. ++ * ++ * Used for FW updates/events. ++ * ++ * This interface is read-only. ++ */ ++ const struct panthor_fw_cs_output_iface *output; ++}; ++ ++/** ++ * struct panthor_fw_csg_iface - Firmware command stream group slot interface ++ */ ++struct panthor_fw_csg_iface { ++ /** ++ * @lock: Lock protecting access to the panthor_fw_csg_input_iface::req ++ * field. ++ * ++ * Needed so we can update the req field concurrently from the interrupt ++ * handler and the scheduler logic. ++ * ++ * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW ++ * interface sections are mapped uncached/write-combined right now, and ++ * using cmpxchg() on such mappings leads to SError faults. Revisit when ++ * we have 'SHARED' GPU mappings hooked up. ++ */ ++ spinlock_t lock; ++ ++ /** ++ * @control: Command stream group slot control interface. ++ * ++ * Used to expose command stream group slot properties. ++ * ++ * This interface is read-only. ++ */ ++ const struct panthor_fw_csg_control_iface *control; ++ ++ /** ++ * @input: Command stream slot input interface. ++ * ++ * Used for host updates/events. ++ */ ++ struct panthor_fw_csg_input_iface *input; ++ ++ /** ++ * @output: Command stream group slot output interface. ++ * ++ * Used for FW updates/events. ++ * ++ * This interface is read-only. ++ */ ++ const struct panthor_fw_csg_output_iface *output; ++}; ++ ++/** ++ * struct panthor_fw_global_iface - Firmware global interface ++ */ ++struct panthor_fw_global_iface { ++ /** ++ * @lock: Lock protecting access to the panthor_fw_global_input_iface::req ++ * field. ++ * ++ * Needed so we can update the req field concurrently from the interrupt ++ * handler and the scheduler/FW management logic. ++ * ++ * TODO: Ideally we'd want to use a cmpxchg() to update the req, but FW ++ * interface sections are mapped uncached/write-combined right now, and ++ * using cmpxchg() on such mappings leads to SError faults. Revisit when ++ * we have 'SHARED' GPU mappings hooked up. ++ */ ++ spinlock_t lock; ++ ++ /** ++ * @control: Command stream group slot control interface. ++ * ++ * Used to expose global FW properties. ++ * ++ * This interface is read-only. ++ */ ++ const struct panthor_fw_global_control_iface *control; ++ ++ /** ++ * @input: Global input interface. ++ * ++ * Used for host updates/events. ++ */ ++ struct panthor_fw_global_input_iface *input; ++ ++ /** ++ * @output: Global output interface. ++ * ++ * Used for FW updates/events. ++ * ++ * This interface is read-only. ++ */ ++ const struct panthor_fw_global_output_iface *output; ++}; ++ ++/** ++ * panthor_fw_toggle_reqs() - Toggle acknowledge bits to send an event to the FW ++ * @__iface: The interface to operate on. ++ * @__in_reg: Name of the register to update in the input section of the interface. ++ * @__out_reg: Name of the register to take as a reference in the output section of the ++ * interface. ++ * @__mask: Mask to apply to the update. ++ * ++ * The Host -> FW event/message passing was designed to be lockless, with each side of ++ * the channel having its writeable section. Events are signaled as a difference between ++ * the host and FW side in the req/ack registers (when a bit differs, there's an event ++ * pending, when they are the same, nothing needs attention). ++ * ++ * This helper allows one to update the req register based on the current value of the ++ * ack register managed by the FW. Toggling a specific bit will flag an event. In order ++ * for events to be re-evaluated, the interface doorbell needs to be rung. ++ * ++ * Concurrent accesses to the same req register is covered. ++ * ++ * Anything requiring atomic updates to multiple registers requires a dedicated lock. ++ */ ++#define panthor_fw_toggle_reqs(__iface, __in_reg, __out_reg, __mask) \ ++ do { \ ++ u32 __cur_val, __new_val, __out_val; \ ++ spin_lock(&(__iface)->lock); \ ++ __cur_val = READ_ONCE((__iface)->input->__in_reg); \ ++ __out_val = READ_ONCE((__iface)->output->__out_reg); \ ++ __new_val = ((__out_val ^ (__mask)) & (__mask)) | (__cur_val & ~(__mask)); \ ++ WRITE_ONCE((__iface)->input->__in_reg, __new_val); \ ++ spin_unlock(&(__iface)->lock); \ ++ } while (0) ++ ++/** ++ * panthor_fw_update_reqs() - Update bits to reflect a configuration change ++ * @__iface: The interface to operate on. ++ * @__in_reg: Name of the register to update in the input section of the interface. ++ * @__val: Value to set. ++ * @__mask: Mask to apply to the update. ++ * ++ * Some configuration get passed through req registers that are also used to ++ * send events to the FW. Those req registers being updated from the interrupt ++ * handler, they require special helpers to update the configuration part as well. ++ * ++ * Concurrent accesses to the same req register is covered. ++ * ++ * Anything requiring atomic updates to multiple registers requires a dedicated lock. ++ */ ++#define panthor_fw_update_reqs(__iface, __in_reg, __val, __mask) \ ++ do { \ ++ u32 __cur_val, __new_val; \ ++ spin_lock(&(__iface)->lock); \ ++ __cur_val = READ_ONCE((__iface)->input->__in_reg); \ ++ __new_val = (__cur_val & ~(__mask)) | ((__val) & (__mask)); \ ++ WRITE_ONCE((__iface)->input->__in_reg, __new_val); \ ++ spin_unlock(&(__iface)->lock); \ ++ } while (0) ++ ++struct panthor_fw_global_iface * ++panthor_fw_get_glb_iface(struct panthor_device *ptdev); ++ ++struct panthor_fw_csg_iface * ++panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot); ++ ++struct panthor_fw_cs_iface * ++panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot); ++ ++int panthor_fw_csg_wait_acks(struct panthor_device *ptdev, u32 csg_id, u32 req_mask, ++ u32 *acked, u32 timeout_ms); ++ ++int panthor_fw_glb_wait_acks(struct panthor_device *ptdev, u32 req_mask, u32 *acked, ++ u32 timeout_ms); ++ ++void panthor_fw_ring_csg_doorbells(struct panthor_device *ptdev, u32 csg_slot); ++ ++struct panthor_kernel_bo * ++panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev, ++ struct panthor_fw_ringbuf_input_iface **input, ++ const struct panthor_fw_ringbuf_output_iface **output, ++ u32 *input_fw_va, u32 *output_fw_va); ++struct panthor_kernel_bo * ++panthor_fw_alloc_suspend_buf_mem(struct panthor_device *ptdev, size_t size); ++ ++struct panthor_vm *panthor_fw_vm(struct panthor_device *ptdev); ++ ++void panthor_fw_pre_reset(struct panthor_device *ptdev, bool on_hang); ++int panthor_fw_post_reset(struct panthor_device *ptdev); ++ ++static inline void panthor_fw_suspend(struct panthor_device *ptdev) ++{ ++ panthor_fw_pre_reset(ptdev, false); ++} ++ ++static inline int panthor_fw_resume(struct panthor_device *ptdev) ++{ ++ return panthor_fw_post_reset(ptdev); ++} ++ ++int panthor_fw_init(struct panthor_device *ptdev); ++void panthor_fw_unplug(struct panthor_device *ptdev); ++ ++#endif + +From 0dfeb7c785f8a5b12b935186a25675eb0ed66da0 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:23 +0100 +Subject: [PATCH 088/150] drm/panthor: Add the heap logical block + +Tiler heap growing requires some kernel driver involvement: when the +tiler runs out of heap memory, it will raise an exception which is +either directly handled by the firmware if some free heap chunks are +available in the heap context, or passed back to the kernel otherwise. +The heap helpers will be used by the scheduler logic to allocate more +heap chunks to a heap context, when such a situation happens. + +Heap context creation is explicitly requested by userspace (using +the TILER_HEAP_CREATE ioctl), and the returned context is attached to a +queue through some command stream instruction. + +All the kernel does is keep the list of heap chunks allocated to a +context, so they can be freed when TILER_HEAP_DESTROY is called, or +extended when the FW requests a new chunk. + +v6: +- Add Maxime's and Heiko's acks + +v5: +- Fix FIXME comment +- Add Steve's R-b + +v4: +- Rework locking to allow concurrent calls to panthor_heap_grow() +- Add a helper to return a heap chunk if we couldn't pass it to the + FW because the group was scheduled out + +v3: +- Add a FIXME for the heap OOM deadlock +- Use the panthor_kernel_bo abstraction for the heap context and heap + chunks +- Drop the panthor_heap_gpu_ctx struct as it is opaque to the driver +- Ensure that the heap context is aligned to the GPU cache line size +- Minor code tidy ups + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-10-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_heap.c | 597 +++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_heap.h | 39 ++ + 2 files changed, 636 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_heap.c + create mode 100644 drivers/gpu/drm/panthor/panthor_heap.h + +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +new file mode 100644 +index 00000000000000..143fa35f2e746c +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -0,0 +1,597 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2023 Collabora ltd. */ ++ ++#include ++#include ++ ++#include ++ ++#include "panthor_device.h" ++#include "panthor_gem.h" ++#include "panthor_heap.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++ ++/* ++ * The GPU heap context is an opaque structure used by the GPU to track the ++ * heap allocations. The driver should only touch it to initialize it (zero all ++ * fields). Because the CPU and GPU can both access this structure it is ++ * required to be GPU cache line aligned. ++ */ ++#define HEAP_CONTEXT_SIZE 32 ++ ++/** ++ * struct panthor_heap_chunk_header - Heap chunk header ++ */ ++struct panthor_heap_chunk_header { ++ /** ++ * @next: Next heap chunk in the list. ++ * ++ * This is a GPU VA. ++ */ ++ u64 next; ++ ++ /** @unknown: MBZ. */ ++ u32 unknown[14]; ++}; ++ ++/** ++ * struct panthor_heap_chunk - Structure used to keep track of allocated heap chunks. ++ */ ++struct panthor_heap_chunk { ++ /** @node: Used to insert the heap chunk in panthor_heap::chunks. */ ++ struct list_head node; ++ ++ /** @bo: Buffer object backing the heap chunk. */ ++ struct panthor_kernel_bo *bo; ++}; ++ ++/** ++ * struct panthor_heap - Structure used to manage tiler heap contexts. ++ */ ++struct panthor_heap { ++ /** @chunks: List containing all heap chunks allocated so far. */ ++ struct list_head chunks; ++ ++ /** @lock: Lock protecting insertion in the chunks list. */ ++ struct mutex lock; ++ ++ /** @chunk_size: Size of each chunk. */ ++ u32 chunk_size; ++ ++ /** @max_chunks: Maximum number of chunks. */ ++ u32 max_chunks; ++ ++ /** ++ * @target_in_flight: Number of in-flight render passes after which ++ * we'd let the FW wait for fragment job to finish instead of allocating new chunks. ++ */ ++ u32 target_in_flight; ++ ++ /** @chunk_count: Number of heap chunks currently allocated. */ ++ u32 chunk_count; ++}; ++ ++#define MAX_HEAPS_PER_POOL 128 ++ ++/** ++ * struct panthor_heap_pool - Pool of heap contexts ++ * ++ * The pool is attached to a panthor_file and can't be shared across processes. ++ */ ++struct panthor_heap_pool { ++ /** @refcount: Reference count. */ ++ struct kref refcount; ++ ++ /** @ptdev: Device. */ ++ struct panthor_device *ptdev; ++ ++ /** @vm: VM this pool is bound to. */ ++ struct panthor_vm *vm; ++ ++ /** @lock: Lock protecting access to @xa. */ ++ struct rw_semaphore lock; ++ ++ /** @xa: Array storing panthor_heap objects. */ ++ struct xarray xa; ++ ++ /** @gpu_contexts: Buffer object containing the GPU heap contexts. */ ++ struct panthor_kernel_bo *gpu_contexts; ++}; ++ ++static int panthor_heap_ctx_stride(struct panthor_device *ptdev) ++{ ++ u32 l2_features = ptdev->gpu_info.l2_features; ++ u32 gpu_cache_line_size = GPU_L2_FEATURES_LINE_SIZE(l2_features); ++ ++ return ALIGN(HEAP_CONTEXT_SIZE, gpu_cache_line_size); ++} ++ ++static int panthor_get_heap_ctx_offset(struct panthor_heap_pool *pool, int id) ++{ ++ return panthor_heap_ctx_stride(pool->ptdev) * id; ++} ++ ++static void *panthor_get_heap_ctx(struct panthor_heap_pool *pool, int id) ++{ ++ return pool->gpu_contexts->kmap + ++ panthor_get_heap_ctx_offset(pool, id); ++} ++ ++static void panthor_free_heap_chunk(struct panthor_vm *vm, ++ struct panthor_heap *heap, ++ struct panthor_heap_chunk *chunk) ++{ ++ mutex_lock(&heap->lock); ++ list_del(&chunk->node); ++ heap->chunk_count--; ++ mutex_unlock(&heap->lock); ++ ++ panthor_kernel_bo_destroy(vm, chunk->bo); ++ kfree(chunk); ++} ++ ++static int panthor_alloc_heap_chunk(struct panthor_device *ptdev, ++ struct panthor_vm *vm, ++ struct panthor_heap *heap, ++ bool initial_chunk) ++{ ++ struct panthor_heap_chunk *chunk; ++ struct panthor_heap_chunk_header *hdr; ++ int ret; ++ ++ chunk = kmalloc(sizeof(*chunk), GFP_KERNEL); ++ if (!chunk) ++ return -ENOMEM; ++ ++ chunk->bo = panthor_kernel_bo_create(ptdev, vm, heap->chunk_size, ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++ if (IS_ERR(chunk->bo)) { ++ ret = PTR_ERR(chunk->bo); ++ goto err_free_chunk; ++ } ++ ++ ret = panthor_kernel_bo_vmap(chunk->bo); ++ if (ret) ++ goto err_destroy_bo; ++ ++ hdr = chunk->bo->kmap; ++ memset(hdr, 0, sizeof(*hdr)); ++ ++ if (initial_chunk && !list_empty(&heap->chunks)) { ++ struct panthor_heap_chunk *prev_chunk; ++ u64 prev_gpuva; ++ ++ prev_chunk = list_first_entry(&heap->chunks, ++ struct panthor_heap_chunk, ++ node); ++ ++ prev_gpuva = panthor_kernel_bo_gpuva(prev_chunk->bo); ++ hdr->next = (prev_gpuva & GENMASK_ULL(63, 12)) | ++ (heap->chunk_size >> 12); ++ } ++ ++ panthor_kernel_bo_vunmap(chunk->bo); ++ ++ mutex_lock(&heap->lock); ++ list_add(&chunk->node, &heap->chunks); ++ heap->chunk_count++; ++ mutex_unlock(&heap->lock); ++ ++ return 0; ++ ++err_destroy_bo: ++ panthor_kernel_bo_destroy(vm, chunk->bo); ++ ++err_free_chunk: ++ kfree(chunk); ++ ++ return ret; ++} ++ ++static void panthor_free_heap_chunks(struct panthor_vm *vm, ++ struct panthor_heap *heap) ++{ ++ struct panthor_heap_chunk *chunk, *tmp; ++ ++ list_for_each_entry_safe(chunk, tmp, &heap->chunks, node) ++ panthor_free_heap_chunk(vm, heap, chunk); ++} ++ ++static int panthor_alloc_heap_chunks(struct panthor_device *ptdev, ++ struct panthor_vm *vm, ++ struct panthor_heap *heap, ++ u32 chunk_count) ++{ ++ int ret; ++ u32 i; ++ ++ for (i = 0; i < chunk_count; i++) { ++ ret = panthor_alloc_heap_chunk(ptdev, vm, heap, true); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++static int ++panthor_heap_destroy_locked(struct panthor_heap_pool *pool, u32 handle) ++{ ++ struct panthor_heap *heap; ++ ++ heap = xa_erase(&pool->xa, handle); ++ if (!heap) ++ return -EINVAL; ++ ++ panthor_free_heap_chunks(pool->vm, heap); ++ mutex_destroy(&heap->lock); ++ kfree(heap); ++ return 0; ++} ++ ++/** ++ * panthor_heap_destroy() - Destroy a heap context ++ * @pool: Pool this context belongs to. ++ * @handle: Handle returned by panthor_heap_create(). ++ */ ++int panthor_heap_destroy(struct panthor_heap_pool *pool, u32 handle) ++{ ++ int ret; ++ ++ down_write(&pool->lock); ++ ret = panthor_heap_destroy_locked(pool, handle); ++ up_write(&pool->lock); ++ ++ return ret; ++} ++ ++/** ++ * panthor_heap_create() - Create a heap context ++ * @pool: Pool to instantiate the heap context from. ++ * @initial_chunk_count: Number of chunk allocated at initialization time. ++ * Must be at least 1. ++ * @chunk_size: The size of each chunk. Must be a power of two between 256k ++ * and 2M. ++ * @max_chunks: Maximum number of chunks that can be allocated. ++ * @target_in_flight: Maximum number of in-flight render passes. ++ * @heap_ctx_gpu_va: Pointer holding the GPU address of the allocated heap ++ * context. ++ * @first_chunk_gpu_va: Pointer holding the GPU address of the first chunk ++ * assigned to the heap context. ++ * ++ * Return: a positive handle on success, a negative error otherwise. ++ */ ++int panthor_heap_create(struct panthor_heap_pool *pool, ++ u32 initial_chunk_count, ++ u32 chunk_size, ++ u32 max_chunks, ++ u32 target_in_flight, ++ u64 *heap_ctx_gpu_va, ++ u64 *first_chunk_gpu_va) ++{ ++ struct panthor_heap *heap; ++ struct panthor_heap_chunk *first_chunk; ++ struct panthor_vm *vm; ++ int ret = 0; ++ u32 id; ++ ++ if (initial_chunk_count == 0) ++ return -EINVAL; ++ ++ if (hweight32(chunk_size) != 1 || ++ chunk_size < SZ_256K || chunk_size > SZ_2M) ++ return -EINVAL; ++ ++ down_read(&pool->lock); ++ vm = panthor_vm_get(pool->vm); ++ up_read(&pool->lock); ++ ++ /* The pool has been destroyed, we can't create a new heap. */ ++ if (!vm) ++ return -EINVAL; ++ ++ heap = kzalloc(sizeof(*heap), GFP_KERNEL); ++ if (!heap) { ++ ret = -ENOMEM; ++ goto err_put_vm; ++ } ++ ++ mutex_init(&heap->lock); ++ INIT_LIST_HEAD(&heap->chunks); ++ heap->chunk_size = chunk_size; ++ heap->max_chunks = max_chunks; ++ heap->target_in_flight = target_in_flight; ++ ++ ret = panthor_alloc_heap_chunks(pool->ptdev, vm, heap, ++ initial_chunk_count); ++ if (ret) ++ goto err_free_heap; ++ ++ first_chunk = list_first_entry(&heap->chunks, ++ struct panthor_heap_chunk, ++ node); ++ *first_chunk_gpu_va = panthor_kernel_bo_gpuva(first_chunk->bo); ++ ++ down_write(&pool->lock); ++ /* The pool has been destroyed, we can't create a new heap. */ ++ if (!pool->vm) { ++ ret = -EINVAL; ++ } else { ++ ret = xa_alloc(&pool->xa, &id, heap, XA_LIMIT(1, MAX_HEAPS_PER_POOL), GFP_KERNEL); ++ if (!ret) { ++ void *gpu_ctx = panthor_get_heap_ctx(pool, id); ++ ++ memset(gpu_ctx, 0, panthor_heap_ctx_stride(pool->ptdev)); ++ *heap_ctx_gpu_va = panthor_kernel_bo_gpuva(pool->gpu_contexts) + ++ panthor_get_heap_ctx_offset(pool, id); ++ } ++ } ++ up_write(&pool->lock); ++ ++ if (ret) ++ goto err_free_heap; ++ ++ panthor_vm_put(vm); ++ return id; ++ ++err_free_heap: ++ panthor_free_heap_chunks(pool->vm, heap); ++ mutex_destroy(&heap->lock); ++ kfree(heap); ++ ++err_put_vm: ++ panthor_vm_put(vm); ++ return ret; ++} ++ ++/** ++ * panthor_heap_return_chunk() - Return an unused heap chunk ++ * @pool: The pool this heap belongs to. ++ * @heap_gpu_va: The GPU address of the heap context. ++ * @chunk_gpu_va: The chunk VA to return. ++ * ++ * This function is used when a chunk allocated with panthor_heap_grow() ++ * couldn't be linked to the heap context through the FW interface because ++ * the group requesting the allocation was scheduled out in the meantime. ++ */ ++int panthor_heap_return_chunk(struct panthor_heap_pool *pool, ++ u64 heap_gpu_va, ++ u64 chunk_gpu_va) ++{ ++ u64 offset = heap_gpu_va - panthor_kernel_bo_gpuva(pool->gpu_contexts); ++ u32 heap_id = (u32)offset / panthor_heap_ctx_stride(pool->ptdev); ++ struct panthor_heap_chunk *chunk, *tmp, *removed = NULL; ++ struct panthor_heap *heap; ++ int ret; ++ ++ if (offset > U32_MAX || heap_id >= MAX_HEAPS_PER_POOL) ++ return -EINVAL; ++ ++ down_read(&pool->lock); ++ heap = xa_load(&pool->xa, heap_id); ++ if (!heap) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ chunk_gpu_va &= GENMASK_ULL(63, 12); ++ ++ mutex_lock(&heap->lock); ++ list_for_each_entry_safe(chunk, tmp, &heap->chunks, node) { ++ if (panthor_kernel_bo_gpuva(chunk->bo) == chunk_gpu_va) { ++ removed = chunk; ++ list_del(&chunk->node); ++ heap->chunk_count--; ++ break; ++ } ++ } ++ mutex_unlock(&heap->lock); ++ ++ if (removed) { ++ panthor_kernel_bo_destroy(pool->vm, chunk->bo); ++ kfree(chunk); ++ ret = 0; ++ } else { ++ ret = -EINVAL; ++ } ++ ++out_unlock: ++ up_read(&pool->lock); ++ return ret; ++} ++ ++/** ++ * panthor_heap_grow() - Make a heap context grow. ++ * @pool: The pool this heap belongs to. ++ * @heap_gpu_va: The GPU address of the heap context. ++ * @renderpasses_in_flight: Number of render passes currently in-flight. ++ * @pending_frag_count: Number of fragment jobs waiting for execution/completion. ++ * @new_chunk_gpu_va: Pointer used to return the chunk VA. ++ */ ++int panthor_heap_grow(struct panthor_heap_pool *pool, ++ u64 heap_gpu_va, ++ u32 renderpasses_in_flight, ++ u32 pending_frag_count, ++ u64 *new_chunk_gpu_va) ++{ ++ u64 offset = heap_gpu_va - panthor_kernel_bo_gpuva(pool->gpu_contexts); ++ u32 heap_id = (u32)offset / panthor_heap_ctx_stride(pool->ptdev); ++ struct panthor_heap_chunk *chunk; ++ struct panthor_heap *heap; ++ int ret; ++ ++ if (offset > U32_MAX || heap_id >= MAX_HEAPS_PER_POOL) ++ return -EINVAL; ++ ++ down_read(&pool->lock); ++ heap = xa_load(&pool->xa, heap_id); ++ if (!heap) { ++ ret = -EINVAL; ++ goto out_unlock; ++ } ++ ++ /* If we reached the target in-flight render passes, or if we ++ * reached the maximum number of chunks, let the FW figure another way to ++ * find some memory (wait for render passes to finish, or call the exception ++ * handler provided by the userspace driver, if any). ++ */ ++ if (renderpasses_in_flight > heap->target_in_flight || ++ (pending_frag_count > 0 && heap->chunk_count >= heap->max_chunks)) { ++ ret = -EBUSY; ++ goto out_unlock; ++ } else if (heap->chunk_count >= heap->max_chunks) { ++ ret = -ENOMEM; ++ goto out_unlock; ++ } ++ ++ /* FIXME: panthor_alloc_heap_chunk() triggers a kernel BO creation, ++ * which goes through the blocking allocation path. Ultimately, we ++ * want a non-blocking allocation, so we can immediately report to the ++ * FW when the system is running out of memory. In that case, the FW ++ * can call a user-provided exception handler, which might try to free ++ * some tiler memory by issuing an intermediate fragment job. If the ++ * exception handler can't do anything, it will flag the queue as ++ * faulty so the job that triggered this tiler chunk allocation and all ++ * further jobs in this queue fail immediately instead of having to ++ * wait for the job timeout. ++ */ ++ ret = panthor_alloc_heap_chunk(pool->ptdev, pool->vm, heap, false); ++ if (ret) ++ goto out_unlock; ++ ++ chunk = list_first_entry(&heap->chunks, ++ struct panthor_heap_chunk, ++ node); ++ *new_chunk_gpu_va = (panthor_kernel_bo_gpuva(chunk->bo) & GENMASK_ULL(63, 12)) | ++ (heap->chunk_size >> 12); ++ ret = 0; ++ ++out_unlock: ++ up_read(&pool->lock); ++ return ret; ++} ++ ++static void panthor_heap_pool_release(struct kref *refcount) ++{ ++ struct panthor_heap_pool *pool = ++ container_of(refcount, struct panthor_heap_pool, refcount); ++ ++ xa_destroy(&pool->xa); ++ kfree(pool); ++} ++ ++/** ++ * panthor_heap_pool_put() - Release a heap pool reference ++ * @pool: Pool to release the reference on. Can be NULL. ++ */ ++void panthor_heap_pool_put(struct panthor_heap_pool *pool) ++{ ++ if (pool) ++ kref_put(&pool->refcount, panthor_heap_pool_release); ++} ++ ++/** ++ * panthor_heap_pool_get() - Get a heap pool reference ++ * @pool: Pool to get the reference on. Can be NULL. ++ * ++ * Return: @pool. ++ */ ++struct panthor_heap_pool * ++panthor_heap_pool_get(struct panthor_heap_pool *pool) ++{ ++ if (pool) ++ kref_get(&pool->refcount); ++ ++ return pool; ++} ++ ++/** ++ * panthor_heap_pool_create() - Create a heap pool ++ * @ptdev: Device. ++ * @vm: The VM this heap pool will be attached to. ++ * ++ * Heap pools might contain up to 128 heap contexts, and are per-VM. ++ * ++ * Return: A valid pointer on success, a negative error code otherwise. ++ */ ++struct panthor_heap_pool * ++panthor_heap_pool_create(struct panthor_device *ptdev, struct panthor_vm *vm) ++{ ++ size_t bosize = ALIGN(MAX_HEAPS_PER_POOL * ++ panthor_heap_ctx_stride(ptdev), ++ 4096); ++ struct panthor_heap_pool *pool; ++ int ret = 0; ++ ++ pool = kzalloc(sizeof(*pool), GFP_KERNEL); ++ if (!pool) ++ return ERR_PTR(-ENOMEM); ++ ++ /* We want a weak ref here: the heap pool belongs to the VM, so we're ++ * sure that, as long as the heap pool exists, the VM exists too. ++ */ ++ pool->vm = vm; ++ pool->ptdev = ptdev; ++ init_rwsem(&pool->lock); ++ xa_init_flags(&pool->xa, XA_FLAGS_ALLOC1); ++ kref_init(&pool->refcount); ++ ++ pool->gpu_contexts = panthor_kernel_bo_create(ptdev, vm, bosize, ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++ if (IS_ERR(pool->gpu_contexts)) { ++ ret = PTR_ERR(pool->gpu_contexts); ++ goto err_destroy_pool; ++ } ++ ++ ret = panthor_kernel_bo_vmap(pool->gpu_contexts); ++ if (ret) ++ goto err_destroy_pool; ++ ++ return pool; ++ ++err_destroy_pool: ++ panthor_heap_pool_destroy(pool); ++ return ERR_PTR(ret); ++} ++ ++/** ++ * panthor_heap_pool_destroy() - Destroy a heap pool. ++ * @pool: Pool to destroy. ++ * ++ * This function destroys all heap contexts and their resources. Thus ++ * preventing any use of the heap context or the chunk attached to them ++ * after that point. ++ * ++ * If the GPU still has access to some heap contexts, a fault should be ++ * triggered, which should flag the command stream groups using these ++ * context as faulty. ++ * ++ * The heap pool object is only released when all references to this pool ++ * are released. ++ */ ++void panthor_heap_pool_destroy(struct panthor_heap_pool *pool) ++{ ++ struct panthor_heap *heap; ++ unsigned long i; ++ ++ if (!pool) ++ return; ++ ++ down_write(&pool->lock); ++ xa_for_each(&pool->xa, i, heap) ++ drm_WARN_ON(&pool->ptdev->base, panthor_heap_destroy_locked(pool, i)); ++ ++ if (!IS_ERR_OR_NULL(pool->gpu_contexts)) ++ panthor_kernel_bo_destroy(pool->vm, pool->gpu_contexts); ++ ++ /* Reflects the fact the pool has been destroyed. */ ++ pool->vm = NULL; ++ up_write(&pool->lock); ++ ++ panthor_heap_pool_put(pool); ++} +diff --git a/drivers/gpu/drm/panthor/panthor_heap.h b/drivers/gpu/drm/panthor/panthor_heap.h +new file mode 100644 +index 00000000000000..25a5f2bba44570 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_heap.h +@@ -0,0 +1,39 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_HEAP_H__ ++#define __PANTHOR_HEAP_H__ ++ ++#include ++ ++struct panthor_device; ++struct panthor_heap_pool; ++struct panthor_vm; ++ ++int panthor_heap_create(struct panthor_heap_pool *pool, ++ u32 initial_chunk_count, ++ u32 chunk_size, ++ u32 max_chunks, ++ u32 target_in_flight, ++ u64 *heap_ctx_gpu_va, ++ u64 *first_chunk_gpu_va); ++int panthor_heap_destroy(struct panthor_heap_pool *pool, u32 handle); ++ ++struct panthor_heap_pool * ++panthor_heap_pool_create(struct panthor_device *ptdev, struct panthor_vm *vm); ++void panthor_heap_pool_destroy(struct panthor_heap_pool *pool); ++ ++struct panthor_heap_pool * ++panthor_heap_pool_get(struct panthor_heap_pool *pool); ++void panthor_heap_pool_put(struct panthor_heap_pool *pool); ++ ++int panthor_heap_grow(struct panthor_heap_pool *pool, ++ u64 heap_gpu_va, ++ u32 renderpasses_in_flight, ++ u32 pending_frag_count, ++ u64 *new_chunk_gpu_va); ++int panthor_heap_return_chunk(struct panthor_heap_pool *pool, ++ u64 heap_gpu_va, ++ u64 chunk_gpu_va); ++ ++#endif + +From 2450156010b67d3eca169b1aeb63649b6215f85f Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:24 +0100 +Subject: [PATCH 089/150] drm/panthor: Add the scheduler logical block + +This is the piece of software interacting with the FW scheduler, and +taking care of some scheduling aspects when the FW comes short of slots +scheduling slots. Indeed, the FW only expose a few slots, and the kernel +has to give all submission contexts, a chance to execute their jobs. + +The kernel-side scheduler is timeslice-based, with a round-robin queue +per priority level. + +Job submission is handled with a 1:1 drm_sched_entity:drm_gpu_scheduler, +allowing us to delegate the dependency tracking to the core. + +All the gory details should be documented inline. + +v6: +- Add Maxime's and Heiko's acks +- Make sure the scheduler is initialized before queueing the tick work + in the MMU fault handler +- Keep header inclusion alphabetically ordered + +v5: +- Fix typos +- Call panthor_kernel_bo_destroy(group->syncobjs) unconditionally +- Don't move the group to the waiting list tail when it was already + waiting for a different syncobj +- Fix fatal_queues flagging in the tiler OOM path +- Don't warn when more than one job timesout on a group +- Add a warning message when we fail to allocate a heap chunk +- Add Steve's R-b + +v4: +- Check drmm_mutex_init() return code +- s/drm_gem_vmap_unlocked/drm_gem_vunmap_unlocked/ in + panthor_queue_put_syncwait_obj() +- Drop unneeded WARN_ON() in cs_slot_sync_queue_state_locked() +- Use atomic_xchg() instead of atomic_fetch_and(0) +- Fix typos +- Let panthor_kernel_bo_destroy() check for IS_ERR_OR_NULL() BOs +- Defer TILER_OOM event handling to a separate workqueue to prevent + deadlocks when the heap chunk allocation is blocked on mem-reclaim. + This is just a temporary solution, until we add support for + non-blocking/failable allocations +- Pass the scheduler workqueue to drm_sched instead of instantiating + a separate one (no longer needed now that heap chunk allocation + happens on a dedicated wq) +- Set WQ_MEM_RECLAIM on the scheduler workqueue, so we can handle + job timeouts when the system is under mem pressure, and hopefully + free up some memory retained by these jobs + +v3: +- Rework the FW event handling logic to avoid races +- Make sure MMU faults kill the group immediately +- Use the panthor_kernel_bo abstraction for group/queue buffers +- Make in_progress an atomic_t, so we can check it without the reset lock + held +- Don't limit the number of groups per context to the FW scheduler + capacity. Fix the limit to 128 for now. +- Add a panthor_job_vm() helper +- Account for panthor_vm changes +- Add our job fence as DMA_RESV_USAGE_WRITE to all external objects + (was previously DMA_RESV_USAGE_BOOKKEEP). I don't get why, given + we're supposed to be fully-explicit, but other drivers do that, so + there must be a good reason +- Account for drm_sched changes +- Provide a panthor_queue_put_syncwait_obj() +- Unconditionally return groups to their idle list in + panthor_sched_suspend() +- Condition of sched_queue_{,delayed_}work fixed to be only when a reset + isn't pending or in progress. +- Several typos in comments fixed. + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-11-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 3502 +++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_sched.h | 50 + + 2 files changed, 3552 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_sched.c + create mode 100644 drivers/gpu/drm/panthor/panthor_sched.h + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +new file mode 100644 +index 00000000000000..5f7803b6fc48c7 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -0,0 +1,3502 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2023 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "panthor_devfreq.h" ++#include "panthor_device.h" ++#include "panthor_fw.h" ++#include "panthor_gem.h" ++#include "panthor_gpu.h" ++#include "panthor_heap.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++#include "panthor_sched.h" ++ ++/** ++ * DOC: Scheduler ++ * ++ * Mali CSF hardware adopts a firmware-assisted scheduling model, where ++ * the firmware takes care of scheduling aspects, to some extent. ++ * ++ * The scheduling happens at the scheduling group level, each group ++ * contains 1 to N queues (N is FW/hardware dependent, and exposed ++ * through the firmware interface). Each queue is assigned a command ++ * stream ring buffer, which serves as a way to get jobs submitted to ++ * the GPU, among other things. ++ * ++ * The firmware can schedule a maximum of M groups (M is FW/hardware ++ * dependent, and exposed through the firmware interface). Passed ++ * this maximum number of groups, the kernel must take care of ++ * rotating the groups passed to the firmware so every group gets ++ * a chance to have his queues scheduled for execution. ++ * ++ * The current implementation only supports with kernel-mode queues. ++ * In other terms, userspace doesn't have access to the ring-buffer. ++ * Instead, userspace passes indirect command stream buffers that are ++ * called from the queue ring-buffer by the kernel using a pre-defined ++ * sequence of command stream instructions to ensure the userspace driver ++ * always gets consistent results (cache maintenance, ++ * synchronization, ...). ++ * ++ * We rely on the drm_gpu_scheduler framework to deal with job ++ * dependencies and submission. As any other driver dealing with a ++ * FW-scheduler, we use the 1:1 entity:scheduler mode, such that each ++ * entity has its own job scheduler. When a job is ready to be executed ++ * (all its dependencies are met), it is pushed to the appropriate ++ * queue ring-buffer, and the group is scheduled for execution if it ++ * wasn't already active. ++ * ++ * Kernel-side group scheduling is timeslice-based. When we have less ++ * groups than there are slots, the periodic tick is disabled and we ++ * just let the FW schedule the active groups. When there are more ++ * groups than slots, we let each group a chance to execute stuff for ++ * a given amount of time, and then re-evaluate and pick new groups ++ * to schedule. The group selection algorithm is based on ++ * priority+round-robin. ++ * ++ * Even though user-mode queues is out of the scope right now, the ++ * current design takes them into account by avoiding any guess on the ++ * group/queue state that would be based on information we wouldn't have ++ * if userspace was in charge of the ring-buffer. That's also one of the ++ * reason we don't do 'cooperative' scheduling (encoding FW group slot ++ * reservation as dma_fence that would be returned from the ++ * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as ++ * a queue of waiters, ordered by job submission order). This approach ++ * would work for kernel-mode queues, but would make user-mode queues a ++ * lot more complicated to retrofit. ++ */ ++ ++#define JOB_TIMEOUT_MS 5000 ++ ++#define MIN_CS_PER_CSG 8 ++ ++#define MIN_CSGS 3 ++#define MAX_CSG_PRIO 0xf ++ ++struct panthor_group; ++ ++/** ++ * struct panthor_csg_slot - Command stream group slot ++ * ++ * This represents a FW slot for a scheduling group. ++ */ ++struct panthor_csg_slot { ++ /** @group: Scheduling group bound to this slot. */ ++ struct panthor_group *group; ++ ++ /** @priority: Group priority. */ ++ u8 priority; ++ ++ /** ++ * @idle: True if the group bound to this slot is idle. ++ * ++ * A group is idle when it has nothing waiting for execution on ++ * all its queues, or when queues are blocked waiting for something ++ * to happen (synchronization object). ++ */ ++ bool idle; ++}; ++ ++/** ++ * enum panthor_csg_priority - Group priority ++ */ ++enum panthor_csg_priority { ++ /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */ ++ PANTHOR_CSG_PRIORITY_LOW = 0, ++ ++ /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */ ++ PANTHOR_CSG_PRIORITY_MEDIUM, ++ ++ /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */ ++ PANTHOR_CSG_PRIORITY_HIGH, ++ ++ /** ++ * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group. ++ * ++ * Real-time priority allows one to preempt scheduling of other ++ * non-real-time groups. When such a group becomes executable, ++ * it will evict the group with the lowest non-rt priority if ++ * there's no free group slot available. ++ * ++ * Currently not exposed to userspace. ++ */ ++ PANTHOR_CSG_PRIORITY_RT, ++ ++ /** @PANTHOR_CSG_PRIORITY_COUNT: Number of priority levels. */ ++ PANTHOR_CSG_PRIORITY_COUNT, ++}; ++ ++/** ++ * struct panthor_scheduler - Object used to manage the scheduler ++ */ ++struct panthor_scheduler { ++ /** @ptdev: Device. */ ++ struct panthor_device *ptdev; ++ ++ /** ++ * @wq: Workqueue used by our internal scheduler logic and ++ * drm_gpu_scheduler. ++ * ++ * Used for the scheduler tick, group update or other kind of FW ++ * event processing that can't be handled in the threaded interrupt ++ * path. Also passed to the drm_gpu_scheduler instances embedded ++ * in panthor_queue. ++ */ ++ struct workqueue_struct *wq; ++ ++ /** ++ * @heap_alloc_wq: Workqueue used to schedule tiler_oom works. ++ * ++ * We have a queue dedicated to heap chunk allocation works to avoid ++ * blocking the rest of the scheduler if the allocation tries to ++ * reclaim memory. ++ */ ++ struct workqueue_struct *heap_alloc_wq; ++ ++ /** @tick_work: Work executed on a scheduling tick. */ ++ struct delayed_work tick_work; ++ ++ /** ++ * @sync_upd_work: Work used to process synchronization object updates. ++ * ++ * We use this work to unblock queues/groups that were waiting on a ++ * synchronization object. ++ */ ++ struct work_struct sync_upd_work; ++ ++ /** ++ * @fw_events_work: Work used to process FW events outside the interrupt path. ++ * ++ * Even if the interrupt is threaded, we need any event processing ++ * that require taking the panthor_scheduler::lock to be processed ++ * outside the interrupt path so we don't block the tick logic when ++ * it calls panthor_fw_{csg,wait}_wait_acks(). Since most of the ++ * event processing requires taking this lock, we just delegate all ++ * FW event processing to the scheduler workqueue. ++ */ ++ struct work_struct fw_events_work; ++ ++ /** ++ * @fw_events: Bitmask encoding pending FW events. ++ */ ++ atomic_t fw_events; ++ ++ /** ++ * @resched_target: When the next tick should occur. ++ * ++ * Expressed in jiffies. ++ */ ++ u64 resched_target; ++ ++ /** ++ * @last_tick: When the last tick occurred. ++ * ++ * Expressed in jiffies. ++ */ ++ u64 last_tick; ++ ++ /** @tick_period: Tick period in jiffies. */ ++ u64 tick_period; ++ ++ /** ++ * @lock: Lock protecting access to all the scheduler fields. ++ * ++ * Should be taken in the tick work, the irq handler, and anywhere the @groups ++ * fields are touched. ++ */ ++ struct mutex lock; ++ ++ /** @groups: Various lists used to classify groups. */ ++ struct { ++ /** ++ * @runnable: Runnable group lists. ++ * ++ * When a group has queues that want to execute something, ++ * its panthor_group::run_node should be inserted here. ++ * ++ * One list per-priority. ++ */ ++ struct list_head runnable[PANTHOR_CSG_PRIORITY_COUNT]; ++ ++ /** ++ * @idle: Idle group lists. ++ * ++ * When all queues of a group are idle (either because they ++ * have nothing to execute, or because they are blocked), the ++ * panthor_group::run_node field should be inserted here. ++ * ++ * One list per-priority. ++ */ ++ struct list_head idle[PANTHOR_CSG_PRIORITY_COUNT]; ++ ++ /** ++ * @waiting: List of groups whose queues are blocked on a ++ * synchronization object. ++ * ++ * Insert panthor_group::wait_node here when a group is waiting ++ * for synchronization objects to be signaled. ++ * ++ * This list is evaluated in the @sync_upd_work work. ++ */ ++ struct list_head waiting; ++ } groups; ++ ++ /** ++ * @csg_slots: FW command stream group slots. ++ */ ++ struct panthor_csg_slot csg_slots[MAX_CSGS]; ++ ++ /** @csg_slot_count: Number of command stream group slots exposed by the FW. */ ++ u32 csg_slot_count; ++ ++ /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */ ++ u32 cs_slot_count; ++ ++ /** @as_slot_count: Number of address space slots supported by the MMU. */ ++ u32 as_slot_count; ++ ++ /** @used_csg_slot_count: Number of command stream group slot currently used. */ ++ u32 used_csg_slot_count; ++ ++ /** @sb_slot_count: Number of scoreboard slots. */ ++ u32 sb_slot_count; ++ ++ /** ++ * @might_have_idle_groups: True if an active group might have become idle. ++ * ++ * This will force a tick, so other runnable groups can be scheduled if one ++ * or more active groups became idle. ++ */ ++ bool might_have_idle_groups; ++ ++ /** @pm: Power management related fields. */ ++ struct { ++ /** @has_ref: True if the scheduler owns a runtime PM reference. */ ++ bool has_ref; ++ } pm; ++ ++ /** @reset: Reset related fields. */ ++ struct { ++ /** @lock: Lock protecting the other reset fields. */ ++ struct mutex lock; ++ ++ /** ++ * @in_progress: True if a reset is in progress. ++ * ++ * Set to true in panthor_sched_pre_reset() and back to false in ++ * panthor_sched_post_reset(). ++ */ ++ atomic_t in_progress; ++ ++ /** ++ * @stopped_groups: List containing all groups that were stopped ++ * before a reset. ++ * ++ * Insert panthor_group::run_node in the pre_reset path. ++ */ ++ struct list_head stopped_groups; ++ } reset; ++}; ++ ++/** ++ * struct panthor_syncobj_32b - 32-bit FW synchronization object ++ */ ++struct panthor_syncobj_32b { ++ /** @seqno: Sequence number. */ ++ u32 seqno; ++ ++ /** ++ * @status: Status. ++ * ++ * Not zero on failure. ++ */ ++ u32 status; ++}; ++ ++/** ++ * struct panthor_syncobj_64b - 64-bit FW synchronization object ++ */ ++struct panthor_syncobj_64b { ++ /** @seqno: Sequence number. */ ++ u64 seqno; ++ ++ /** ++ * @status: Status. ++ * ++ * Not zero on failure. ++ */ ++ u32 status; ++ ++ /** @pad: MBZ. */ ++ u32 pad; ++}; ++ ++/** ++ * struct panthor_queue - Execution queue ++ */ ++struct panthor_queue { ++ /** @scheduler: DRM scheduler used for this queue. */ ++ struct drm_gpu_scheduler scheduler; ++ ++ /** @entity: DRM scheduling entity used for this queue. */ ++ struct drm_sched_entity entity; ++ ++ /** ++ * @remaining_time: Time remaining before the job timeout expires. ++ * ++ * The job timeout is suspended when the queue is not scheduled by the ++ * FW. Every time we suspend the timer, we need to save the remaining ++ * time so we can restore it later on. ++ */ ++ unsigned long remaining_time; ++ ++ /** @timeout_suspended: True if the job timeout was suspended. */ ++ bool timeout_suspended; ++ ++ /** ++ * @doorbell_id: Doorbell assigned to this queue. ++ * ++ * Right now, all groups share the same doorbell, and the doorbell ID ++ * is assigned to group_slot + 1 when the group is assigned a slot. But ++ * we might decide to provide fine grained doorbell assignment at some ++ * point, so don't have to wake up all queues in a group every time one ++ * of them is updated. ++ */ ++ u8 doorbell_id; ++ ++ /** ++ * @priority: Priority of the queue inside the group. ++ * ++ * Must be less than 16 (Only 4 bits available). ++ */ ++ u8 priority; ++#define CSF_MAX_QUEUE_PRIO GENMASK(3, 0) ++ ++ /** @ringbuf: Command stream ring-buffer. */ ++ struct panthor_kernel_bo *ringbuf; ++ ++ /** @iface: Firmware interface. */ ++ struct { ++ /** @mem: FW memory allocated for this interface. */ ++ struct panthor_kernel_bo *mem; ++ ++ /** @input: Input interface. */ ++ struct panthor_fw_ringbuf_input_iface *input; ++ ++ /** @output: Output interface. */ ++ const struct panthor_fw_ringbuf_output_iface *output; ++ ++ /** @input_fw_va: FW virtual address of the input interface buffer. */ ++ u32 input_fw_va; ++ ++ /** @output_fw_va: FW virtual address of the output interface buffer. */ ++ u32 output_fw_va; ++ } iface; ++ ++ /** ++ * @syncwait: Stores information about the synchronization object this ++ * queue is waiting on. ++ */ ++ struct { ++ /** @gpu_va: GPU address of the synchronization object. */ ++ u64 gpu_va; ++ ++ /** @ref: Reference value to compare against. */ ++ u64 ref; ++ ++ /** @gt: True if this is a greater-than test. */ ++ bool gt; ++ ++ /** @sync64: True if this is a 64-bit sync object. */ ++ bool sync64; ++ ++ /** @bo: Buffer object holding the synchronization object. */ ++ struct drm_gem_object *obj; ++ ++ /** @offset: Offset of the synchronization object inside @bo. */ ++ u64 offset; ++ ++ /** ++ * @kmap: Kernel mapping of the buffer object holding the ++ * synchronization object. ++ */ ++ void *kmap; ++ } syncwait; ++ ++ /** @fence_ctx: Fence context fields. */ ++ struct { ++ /** @lock: Used to protect access to all fences allocated by this context. */ ++ spinlock_t lock; ++ ++ /** ++ * @id: Fence context ID. ++ * ++ * Allocated with dma_fence_context_alloc(). ++ */ ++ u64 id; ++ ++ /** @seqno: Sequence number of the last initialized fence. */ ++ atomic64_t seqno; ++ ++ /** ++ * @in_flight_jobs: List containing all in-flight jobs. ++ * ++ * Used to keep track and signal panthor_job::done_fence when the ++ * synchronization object attached to the queue is signaled. ++ */ ++ struct list_head in_flight_jobs; ++ } fence_ctx; ++}; ++ ++/** ++ * enum panthor_group_state - Scheduling group state. ++ */ ++enum panthor_group_state { ++ /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */ ++ PANTHOR_CS_GROUP_CREATED, ++ ++ /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */ ++ PANTHOR_CS_GROUP_ACTIVE, ++ ++ /** ++ * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is ++ * inactive/suspended right now. ++ */ ++ PANTHOR_CS_GROUP_SUSPENDED, ++ ++ /** ++ * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated. ++ * ++ * Can no longer be scheduled. The only allowed action is a destruction. ++ */ ++ PANTHOR_CS_GROUP_TERMINATED, ++}; ++ ++/** ++ * struct panthor_group - Scheduling group object ++ */ ++struct panthor_group { ++ /** @refcount: Reference count */ ++ struct kref refcount; ++ ++ /** @ptdev: Device. */ ++ struct panthor_device *ptdev; ++ ++ /** @vm: VM bound to the group. */ ++ struct panthor_vm *vm; ++ ++ /** @compute_core_mask: Mask of shader cores that can be used for compute jobs. */ ++ u64 compute_core_mask; ++ ++ /** @fragment_core_mask: Mask of shader cores that can be used for fragment jobs. */ ++ u64 fragment_core_mask; ++ ++ /** @tiler_core_mask: Mask of tiler cores that can be used for tiler jobs. */ ++ u64 tiler_core_mask; ++ ++ /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */ ++ u8 max_compute_cores; ++ ++ /** @max_compute_cores: Maximum number of shader cores used for fragment jobs. */ ++ u8 max_fragment_cores; ++ ++ /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */ ++ u8 max_tiler_cores; ++ ++ /** @priority: Group priority (check panthor_csg_priority). */ ++ u8 priority; ++ ++ /** @blocked_queues: Bitmask reflecting the blocked queues. */ ++ u32 blocked_queues; ++ ++ /** @idle_queues: Bitmask reflecting the idle queues. */ ++ u32 idle_queues; ++ ++ /** @fatal_lock: Lock used to protect access to fatal fields. */ ++ spinlock_t fatal_lock; ++ ++ /** @fatal_queues: Bitmask reflecting the queues that hit a fatal exception. */ ++ u32 fatal_queues; ++ ++ /** @tiler_oom: Mask of queues that have a tiler OOM event to process. */ ++ atomic_t tiler_oom; ++ ++ /** @queue_count: Number of queues in this group. */ ++ u32 queue_count; ++ ++ /** @queues: Queues owned by this group. */ ++ struct panthor_queue *queues[MAX_CS_PER_CSG]; ++ ++ /** ++ * @csg_id: ID of the FW group slot. ++ * ++ * -1 when the group is not scheduled/active. ++ */ ++ int csg_id; ++ ++ /** ++ * @destroyed: True when the group has been destroyed. ++ * ++ * If a group is destroyed it becomes useless: no further jobs can be submitted ++ * to its queues. We simply wait for all references to be dropped so we can ++ * release the group object. ++ */ ++ bool destroyed; ++ ++ /** ++ * @timedout: True when a timeout occurred on any of the queues owned by ++ * this group. ++ * ++ * Timeouts can be reported by drm_sched or by the FW. In any case, any ++ * timeout situation is unrecoverable, and the group becomes useless. ++ * We simply wait for all references to be dropped so we can release the ++ * group object. ++ */ ++ bool timedout; ++ ++ /** ++ * @syncobjs: Pool of per-queue synchronization objects. ++ * ++ * One sync object per queue. The position of the sync object is ++ * determined by the queue index. ++ */ ++ struct panthor_kernel_bo *syncobjs; ++ ++ /** @state: Group state. */ ++ enum panthor_group_state state; ++ ++ /** ++ * @suspend_buf: Suspend buffer. ++ * ++ * Stores the state of the group and its queues when a group is suspended. ++ * Used at resume time to restore the group in its previous state. ++ * ++ * The size of the suspend buffer is exposed through the FW interface. ++ */ ++ struct panthor_kernel_bo *suspend_buf; ++ ++ /** ++ * @protm_suspend_buf: Protection mode suspend buffer. ++ * ++ * Stores the state of the group and its queues when a group that's in ++ * protection mode is suspended. ++ * ++ * Used at resume time to restore the group in its previous state. ++ * ++ * The size of the protection mode suspend buffer is exposed through the ++ * FW interface. ++ */ ++ struct panthor_kernel_bo *protm_suspend_buf; ++ ++ /** @sync_upd_work: Work used to check/signal job fences. */ ++ struct work_struct sync_upd_work; ++ ++ /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */ ++ struct work_struct tiler_oom_work; ++ ++ /** @term_work: Work used to finish the group termination procedure. */ ++ struct work_struct term_work; ++ ++ /** ++ * @release_work: Work used to release group resources. ++ * ++ * We need to postpone the group release to avoid a deadlock when ++ * the last ref is released in the tick work. ++ */ ++ struct work_struct release_work; ++ ++ /** ++ * @run_node: Node used to insert the group in the ++ * panthor_group::groups::{runnable,idle} and ++ * panthor_group::reset.stopped_groups lists. ++ */ ++ struct list_head run_node; ++ ++ /** ++ * @wait_node: Node used to insert the group in the ++ * panthor_group::groups::waiting list. ++ */ ++ struct list_head wait_node; ++}; ++ ++/** ++ * group_queue_work() - Queue a group work ++ * @group: Group to queue the work for. ++ * @wname: Work name. ++ * ++ * Grabs a ref and queue a work item to the scheduler workqueue. If ++ * the work was already queued, we release the reference we grabbed. ++ * ++ * Work callbacks must release the reference we grabbed here. ++ */ ++#define group_queue_work(group, wname) \ ++ do { \ ++ group_get(group); \ ++ if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \ ++ group_put(group); \ ++ } while (0) ++ ++/** ++ * sched_queue_work() - Queue a scheduler work. ++ * @sched: Scheduler object. ++ * @wname: Work name. ++ * ++ * Conditionally queues a scheduler work if no reset is pending/in-progress. ++ */ ++#define sched_queue_work(sched, wname) \ ++ do { \ ++ if (!atomic_read(&(sched)->reset.in_progress) && \ ++ !panthor_device_reset_is_pending((sched)->ptdev)) \ ++ queue_work((sched)->wq, &(sched)->wname ## _work); \ ++ } while (0) ++ ++/** ++ * sched_queue_delayed_work() - Queue a scheduler delayed work. ++ * @sched: Scheduler object. ++ * @wname: Work name. ++ * @delay: Work delay in jiffies. ++ * ++ * Conditionally queues a scheduler delayed work if no reset is ++ * pending/in-progress. ++ */ ++#define sched_queue_delayed_work(sched, wname, delay) \ ++ do { \ ++ if (!atomic_read(&sched->reset.in_progress) && \ ++ !panthor_device_reset_is_pending((sched)->ptdev)) \ ++ mod_delayed_work((sched)->wq, &(sched)->wname ## _work, delay); \ ++ } while (0) ++ ++/* ++ * We currently set the maximum of groups per file to an arbitrary low value. ++ * But this can be updated if we need more. ++ */ ++#define MAX_GROUPS_PER_POOL 128 ++ ++/** ++ * struct panthor_group_pool - Group pool ++ * ++ * Each file get assigned a group pool. ++ */ ++struct panthor_group_pool { ++ /** @xa: Xarray used to manage group handles. */ ++ struct xarray xa; ++}; ++ ++/** ++ * struct panthor_job - Used to manage GPU job ++ */ ++struct panthor_job { ++ /** @base: Inherit from drm_sched_job. */ ++ struct drm_sched_job base; ++ ++ /** @refcount: Reference count. */ ++ struct kref refcount; ++ ++ /** @group: Group of the queue this job will be pushed to. */ ++ struct panthor_group *group; ++ ++ /** @queue_idx: Index of the queue inside @group. */ ++ u32 queue_idx; ++ ++ /** @call_info: Information about the userspace command stream call. */ ++ struct { ++ /** @start: GPU address of the userspace command stream. */ ++ u64 start; ++ ++ /** @size: Size of the userspace command stream. */ ++ u32 size; ++ ++ /** ++ * @latest_flush: Flush ID at the time the userspace command ++ * stream was built. ++ * ++ * Needed for the flush reduction mechanism. ++ */ ++ u32 latest_flush; ++ } call_info; ++ ++ /** @ringbuf: Position of this job is in the ring buffer. */ ++ struct { ++ /** @start: Start offset. */ ++ u64 start; ++ ++ /** @end: End offset. */ ++ u64 end; ++ } ringbuf; ++ ++ /** ++ * @node: Used to insert the job in the panthor_queue::fence_ctx::in_flight_jobs ++ * list. ++ */ ++ struct list_head node; ++ ++ /** @done_fence: Fence signaled when the job is finished or cancelled. */ ++ struct dma_fence *done_fence; ++}; ++ ++static void ++panthor_queue_put_syncwait_obj(struct panthor_queue *queue) ++{ ++ if (queue->syncwait.kmap) { ++ struct iosys_map map = IOSYS_MAP_INIT_VADDR(queue->syncwait.kmap); ++ ++ drm_gem_vunmap_unlocked(queue->syncwait.obj, &map); ++ queue->syncwait.kmap = NULL; ++ } ++ ++ drm_gem_object_put(queue->syncwait.obj); ++ queue->syncwait.obj = NULL; ++} ++ ++static void * ++panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_gem_object *bo; ++ struct iosys_map map; ++ int ret; ++ ++ if (queue->syncwait.kmap) ++ return queue->syncwait.kmap + queue->syncwait.offset; ++ ++ bo = panthor_vm_get_bo_for_va(group->vm, ++ queue->syncwait.gpu_va, ++ &queue->syncwait.offset); ++ if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(bo))) ++ goto err_put_syncwait_obj; ++ ++ queue->syncwait.obj = &bo->base.base; ++ ret = drm_gem_vmap_unlocked(queue->syncwait.obj, &map); ++ if (drm_WARN_ON(&ptdev->base, ret)) ++ goto err_put_syncwait_obj; ++ ++ queue->syncwait.kmap = map.vaddr; ++ if (drm_WARN_ON(&ptdev->base, !queue->syncwait.kmap)) ++ goto err_put_syncwait_obj; ++ ++ return queue->syncwait.kmap + queue->syncwait.offset; ++ ++err_put_syncwait_obj: ++ panthor_queue_put_syncwait_obj(queue); ++ return NULL; ++} ++ ++static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue) ++{ ++ if (IS_ERR_OR_NULL(queue)) ++ return; ++ ++ if (queue->entity.fence_context) ++ drm_sched_entity_destroy(&queue->entity); ++ ++ if (queue->scheduler.ops) ++ drm_sched_fini(&queue->scheduler); ++ ++ panthor_queue_put_syncwait_obj(queue); ++ ++ panthor_kernel_bo_destroy(group->vm, queue->ringbuf); ++ panthor_kernel_bo_destroy(panthor_fw_vm(group->ptdev), queue->iface.mem); ++ ++ kfree(queue); ++} ++ ++static void group_release_work(struct work_struct *work) ++{ ++ struct panthor_group *group = container_of(work, ++ struct panthor_group, ++ release_work); ++ struct panthor_device *ptdev = group->ptdev; ++ u32 i; ++ ++ for (i = 0; i < group->queue_count; i++) ++ group_free_queue(group, group->queues[i]); ++ ++ panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->suspend_buf); ++ panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->protm_suspend_buf); ++ panthor_kernel_bo_destroy(group->vm, group->syncobjs); ++ ++ panthor_vm_put(group->vm); ++ kfree(group); ++} ++ ++static void group_release(struct kref *kref) ++{ ++ struct panthor_group *group = container_of(kref, ++ struct panthor_group, ++ refcount); ++ struct panthor_device *ptdev = group->ptdev; ++ ++ drm_WARN_ON(&ptdev->base, group->csg_id >= 0); ++ drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node)); ++ drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node)); ++ ++ queue_work(panthor_cleanup_wq, &group->release_work); ++} ++ ++static void group_put(struct panthor_group *group) ++{ ++ if (group) ++ kref_put(&group->refcount, group_release); ++} ++ ++static struct panthor_group * ++group_get(struct panthor_group *group) ++{ ++ if (group) ++ kref_get(&group->refcount); ++ ++ return group; ++} ++ ++/** ++ * group_bind_locked() - Bind a group to a group slot ++ * @group: Group. ++ * @csg_id: Slot. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++group_bind_locked(struct panthor_group *group, u32 csg_id) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_csg_slot *csg_slot; ++ int ret; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS || ++ ptdev->scheduler->csg_slots[csg_id].group)) ++ return -EINVAL; ++ ++ ret = panthor_vm_active(group->vm); ++ if (ret) ++ return ret; ++ ++ csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ group_get(group); ++ group->csg_id = csg_id; ++ ++ /* Dummy doorbell allocation: doorbell is assigned to the group and ++ * all queues use the same doorbell. ++ * ++ * TODO: Implement LRU-based doorbell assignment, so the most often ++ * updated queues get their own doorbell, thus avoiding useless checks ++ * on queues belonging to the same group that are rarely updated. ++ */ ++ for (u32 i = 0; i < group->queue_count; i++) ++ group->queues[i]->doorbell_id = csg_id + 1; ++ ++ csg_slot->group = group; ++ ++ return 0; ++} ++ ++/** ++ * group_unbind_locked() - Unbind a group from a slot. ++ * @group: Group to unbind. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++group_unbind_locked(struct panthor_group *group) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_csg_slot *slot; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS)) ++ return -EINVAL; ++ ++ if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE)) ++ return -EINVAL; ++ ++ slot = &ptdev->scheduler->csg_slots[group->csg_id]; ++ panthor_vm_idle(group->vm); ++ group->csg_id = -1; ++ ++ /* Tiler OOM events will be re-issued next time the group is scheduled. */ ++ atomic_set(&group->tiler_oom, 0); ++ cancel_work(&group->tiler_oom_work); ++ ++ for (u32 i = 0; i < group->queue_count; i++) ++ group->queues[i]->doorbell_id = -1; ++ ++ slot->group = NULL; ++ ++ group_put(group); ++ return 0; ++} ++ ++/** ++ * cs_slot_prog_locked() - Program a queue slot ++ * @ptdev: Device. ++ * @csg_id: Group slot ID. ++ * @cs_id: Queue slot ID. ++ * ++ * Program a queue slot with the queue information so things can start being ++ * executed on this queue. ++ * ++ * The group slot must have a group bound to it already (group_bind_locked()). ++ */ ++static void ++cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) ++{ ++ struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id]; ++ struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ queue->iface.input->extract = queue->iface.output->extract; ++ drm_WARN_ON(&ptdev->base, queue->iface.input->insert < queue->iface.input->extract); ++ ++ cs_iface->input->ringbuf_base = panthor_kernel_bo_gpuva(queue->ringbuf); ++ cs_iface->input->ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); ++ cs_iface->input->ringbuf_input = queue->iface.input_fw_va; ++ cs_iface->input->ringbuf_output = queue->iface.output_fw_va; ++ cs_iface->input->config = CS_CONFIG_PRIORITY(queue->priority) | ++ CS_CONFIG_DOORBELL(queue->doorbell_id); ++ cs_iface->input->ack_irq_mask = ~0; ++ panthor_fw_update_reqs(cs_iface, req, ++ CS_IDLE_SYNC_WAIT | ++ CS_IDLE_EMPTY | ++ CS_STATE_START | ++ CS_EXTRACT_EVENT, ++ CS_IDLE_SYNC_WAIT | ++ CS_IDLE_EMPTY | ++ CS_STATE_MASK | ++ CS_EXTRACT_EVENT); ++ if (queue->iface.input->insert != queue->iface.input->extract && queue->timeout_suspended) { ++ drm_sched_resume_timeout(&queue->scheduler, queue->remaining_time); ++ queue->timeout_suspended = false; ++ } ++} ++ ++/** ++ * @cs_slot_reset_locked() - Reset a queue slot ++ * @ptdev: Device. ++ * @csg_id: Group slot. ++ * @cs_id: Queue slot. ++ * ++ * Change the queue slot state to STOP and suspend the queue timeout if ++ * the queue is not blocked. ++ * ++ * The group slot must have a group bound to it (group_bind_locked()). ++ */ ++static int ++cs_slot_reset_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) ++{ ++ struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; ++ struct panthor_queue *queue = group->queues[cs_id]; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ panthor_fw_update_reqs(cs_iface, req, ++ CS_STATE_STOP, ++ CS_STATE_MASK); ++ ++ /* If the queue is blocked, we want to keep the timeout running, so ++ * we can detect unbounded waits and kill the group when that happens. ++ */ ++ if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) { ++ queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); ++ queue->timeout_suspended = true; ++ WARN_ON(queue->remaining_time > msecs_to_jiffies(JOB_TIMEOUT_MS)); ++ } ++ ++ return 0; ++} ++ ++/** ++ * csg_slot_sync_priority_locked() - Synchronize the group slot priority ++ * @ptdev: Device. ++ * @csg_id: Group slot ID. ++ * ++ * Group slot priority update happens asynchronously. When we receive a ++ * %CSG_ENDPOINT_CONFIG, we know the update is effective, and can ++ * reflect it to our panthor_csg_slot object. ++ */ ++static void ++csg_slot_sync_priority_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ struct panthor_fw_csg_iface *csg_iface; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ csg_slot->priority = (csg_iface->input->endpoint_req & CSG_EP_REQ_PRIORITY_MASK) >> 28; ++} ++ ++/** ++ * cs_slot_sync_queue_state_locked() - Synchronize the queue slot priority ++ * @ptdev: Device. ++ * @csg_id: Group slot. ++ * @cs_id: Queue slot. ++ * ++ * Queue state is updated on group suspend or STATUS_UPDATE event. ++ */ ++static void ++cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) ++{ ++ struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; ++ struct panthor_queue *queue = group->queues[cs_id]; ++ struct panthor_fw_cs_iface *cs_iface = ++ panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id); ++ ++ u32 status_wait_cond; ++ ++ switch (cs_iface->output->status_blocked_reason) { ++ case CS_STATUS_BLOCKED_REASON_UNBLOCKED: ++ if (queue->iface.input->insert == queue->iface.output->extract && ++ cs_iface->output->status_scoreboards == 0) ++ group->idle_queues |= BIT(cs_id); ++ break; ++ ++ case CS_STATUS_BLOCKED_REASON_SYNC_WAIT: ++ if (list_empty(&group->wait_node)) { ++ list_move_tail(&group->wait_node, ++ &group->ptdev->scheduler->groups.waiting); ++ } ++ group->blocked_queues |= BIT(cs_id); ++ queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr; ++ queue->syncwait.ref = cs_iface->output->status_wait_sync_value; ++ status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK; ++ queue->syncwait.gt = status_wait_cond == CS_STATUS_WAIT_SYNC_COND_GT; ++ if (cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_64B) { ++ u64 sync_val_hi = cs_iface->output->status_wait_sync_value_hi; ++ ++ queue->syncwait.sync64 = true; ++ queue->syncwait.ref |= sync_val_hi << 32; ++ } else { ++ queue->syncwait.sync64 = false; ++ } ++ break; ++ ++ default: ++ /* Other reasons are not blocking. Consider the queue as runnable ++ * in those cases. ++ */ ++ break; ++ } ++} ++ ++static void ++csg_slot_sync_queues_state_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ u32 i; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ group->idle_queues = 0; ++ group->blocked_queues = 0; ++ ++ for (i = 0; i < group->queue_count; i++) { ++ if (group->queues[i]) ++ cs_slot_sync_queue_state_locked(ptdev, csg_id, i); ++ } ++} ++ ++static void ++csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ struct panthor_fw_csg_iface *csg_iface; ++ struct panthor_group *group; ++ enum panthor_group_state new_state, old_state; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ group = csg_slot->group; ++ ++ if (!group) ++ return; ++ ++ old_state = group->state; ++ switch (csg_iface->output->ack & CSG_STATE_MASK) { ++ case CSG_STATE_START: ++ case CSG_STATE_RESUME: ++ new_state = PANTHOR_CS_GROUP_ACTIVE; ++ break; ++ case CSG_STATE_TERMINATE: ++ new_state = PANTHOR_CS_GROUP_TERMINATED; ++ break; ++ case CSG_STATE_SUSPEND: ++ new_state = PANTHOR_CS_GROUP_SUSPENDED; ++ break; ++ } ++ ++ if (old_state == new_state) ++ return; ++ ++ if (new_state == PANTHOR_CS_GROUP_SUSPENDED) ++ csg_slot_sync_queues_state_locked(ptdev, csg_id); ++ ++ if (old_state == PANTHOR_CS_GROUP_ACTIVE) { ++ u32 i; ++ ++ /* Reset the queue slots so we start from a clean ++ * state when starting/resuming a new group on this ++ * CSG slot. No wait needed here, and no ringbell ++ * either, since the CS slot will only be re-used ++ * on the next CSG start operation. ++ */ ++ for (i = 0; i < group->queue_count; i++) { ++ if (group->queues[i]) ++ cs_slot_reset_locked(ptdev, csg_id, i); ++ } ++ } ++ ++ group->state = new_state; ++} ++ ++static int ++csg_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 priority) ++{ ++ struct panthor_fw_csg_iface *csg_iface; ++ struct panthor_csg_slot *csg_slot; ++ struct panthor_group *group; ++ u32 queue_mask = 0, i; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ if (priority > MAX_CSG_PRIO) ++ return -EINVAL; ++ ++ if (drm_WARN_ON(&ptdev->base, csg_id >= MAX_CSGS)) ++ return -EINVAL; ++ ++ csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ group = csg_slot->group; ++ if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE) ++ return 0; ++ ++ csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id); ++ ++ for (i = 0; i < group->queue_count; i++) { ++ if (group->queues[i]) { ++ cs_slot_prog_locked(ptdev, csg_id, i); ++ queue_mask |= BIT(i); ++ } ++ } ++ ++ csg_iface->input->allow_compute = group->compute_core_mask; ++ csg_iface->input->allow_fragment = group->fragment_core_mask; ++ csg_iface->input->allow_other = group->tiler_core_mask; ++ csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) | ++ CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) | ++ CSG_EP_REQ_TILER(group->max_tiler_cores) | ++ CSG_EP_REQ_PRIORITY(priority); ++ csg_iface->input->config = panthor_vm_as(group->vm); ++ ++ if (group->suspend_buf) ++ csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf); ++ else ++ csg_iface->input->suspend_buf = 0; ++ ++ if (group->protm_suspend_buf) { ++ csg_iface->input->protm_suspend_buf = ++ panthor_kernel_bo_gpuva(group->protm_suspend_buf); ++ } else { ++ csg_iface->input->protm_suspend_buf = 0; ++ } ++ ++ csg_iface->input->ack_irq_mask = ~0; ++ panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, queue_mask); ++ return 0; ++} ++ ++static void ++cs_slot_process_fatal_event_locked(struct panthor_device *ptdev, ++ u32 csg_id, u32 cs_id) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ struct panthor_fw_cs_iface *cs_iface; ++ u32 fatal; ++ u64 info; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ fatal = cs_iface->output->fatal; ++ info = cs_iface->output->fatal_info; ++ ++ if (group) ++ group->fatal_queues |= BIT(cs_id); ++ ++ sched_queue_delayed_work(sched, tick, 0); ++ drm_warn(&ptdev->base, ++ "CSG slot %d CS slot: %d\n" ++ "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" ++ "CS_FATAL.EXCEPTION_DATA: 0x%x\n" ++ "CS_FATAL_INFO.EXCEPTION_DATA: 0x%llx\n", ++ csg_id, cs_id, ++ (unsigned int)CS_EXCEPTION_TYPE(fatal), ++ panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fatal)), ++ (unsigned int)CS_EXCEPTION_DATA(fatal), ++ info); ++} ++ ++static void ++cs_slot_process_fault_event_locked(struct panthor_device *ptdev, ++ u32 csg_id, u32 cs_id) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ struct panthor_queue *queue = group && cs_id < group->queue_count ? ++ group->queues[cs_id] : NULL; ++ struct panthor_fw_cs_iface *cs_iface; ++ u32 fault; ++ u64 info; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ fault = cs_iface->output->fault; ++ info = cs_iface->output->fault_info; ++ ++ if (queue && CS_EXCEPTION_TYPE(fault) == DRM_PANTHOR_EXCEPTION_CS_INHERIT_FAULT) { ++ u64 cs_extract = queue->iface.output->extract; ++ struct panthor_job *job; ++ ++ spin_lock(&queue->fence_ctx.lock); ++ list_for_each_entry(job, &queue->fence_ctx.in_flight_jobs, node) { ++ if (cs_extract >= job->ringbuf.end) ++ continue; ++ ++ if (cs_extract < job->ringbuf.start) ++ break; ++ ++ dma_fence_set_error(job->done_fence, -EINVAL); ++ } ++ spin_unlock(&queue->fence_ctx.lock); ++ } ++ ++ drm_warn(&ptdev->base, ++ "CSG slot %d CS slot: %d\n" ++ "CS_FAULT.EXCEPTION_TYPE: 0x%x (%s)\n" ++ "CS_FAULT.EXCEPTION_DATA: 0x%x\n" ++ "CS_FAULT_INFO.EXCEPTION_DATA: 0x%llx\n", ++ csg_id, cs_id, ++ (unsigned int)CS_EXCEPTION_TYPE(fault), ++ panthor_exception_name(ptdev, CS_EXCEPTION_TYPE(fault)), ++ (unsigned int)CS_EXCEPTION_DATA(fault), ++ info); ++} ++ ++static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ u32 renderpasses_in_flight, pending_frag_count; ++ struct panthor_heap_pool *heaps = NULL; ++ u64 heap_address, new_chunk_va = 0; ++ u32 vt_start, vt_end, frag_end; ++ int ret, csg_id; ++ ++ mutex_lock(&sched->lock); ++ csg_id = group->csg_id; ++ if (csg_id >= 0) { ++ struct panthor_fw_cs_iface *cs_iface; ++ ++ cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ heaps = panthor_vm_get_heap_pool(group->vm, false); ++ heap_address = cs_iface->output->heap_address; ++ vt_start = cs_iface->output->heap_vt_start; ++ vt_end = cs_iface->output->heap_vt_end; ++ frag_end = cs_iface->output->heap_frag_end; ++ renderpasses_in_flight = vt_start - frag_end; ++ pending_frag_count = vt_end - frag_end; ++ } ++ mutex_unlock(&sched->lock); ++ ++ /* The group got scheduled out, we stop here. We will get a new tiler OOM event ++ * when it's scheduled again. ++ */ ++ if (unlikely(csg_id < 0)) ++ return 0; ++ ++ if (!heaps || frag_end > vt_end || vt_end >= vt_start) { ++ ret = -EINVAL; ++ } else { ++ /* We do the allocation without holding the scheduler lock to avoid ++ * blocking the scheduling. ++ */ ++ ret = panthor_heap_grow(heaps, heap_address, ++ renderpasses_in_flight, ++ pending_frag_count, &new_chunk_va); ++ } ++ ++ if (ret && ret != -EBUSY) { ++ drm_warn(&ptdev->base, "Failed to extend the tiler heap\n"); ++ group->fatal_queues |= BIT(cs_id); ++ sched_queue_delayed_work(sched, tick, 0); ++ goto out_put_heap_pool; ++ } ++ ++ mutex_lock(&sched->lock); ++ csg_id = group->csg_id; ++ if (csg_id >= 0) { ++ struct panthor_fw_csg_iface *csg_iface; ++ struct panthor_fw_cs_iface *cs_iface; ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ ++ cs_iface->input->heap_start = new_chunk_va; ++ cs_iface->input->heap_end = new_chunk_va; ++ panthor_fw_update_reqs(cs_iface, req, cs_iface->output->ack, CS_TILER_OOM); ++ panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, BIT(cs_id)); ++ panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); ++ } ++ mutex_unlock(&sched->lock); ++ ++ /* We allocated a chunck, but couldn't link it to the heap ++ * context because the group was scheduled out while we were ++ * allocating memory. We need to return this chunk to the heap. ++ */ ++ if (unlikely(csg_id < 0 && new_chunk_va)) ++ panthor_heap_return_chunk(heaps, heap_address, new_chunk_va); ++ ++ ret = 0; ++ ++out_put_heap_pool: ++ panthor_heap_pool_put(heaps); ++ return ret; ++} ++ ++static void group_tiler_oom_work(struct work_struct *work) ++{ ++ struct panthor_group *group = ++ container_of(work, struct panthor_group, tiler_oom_work); ++ u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0); ++ ++ while (tiler_oom) { ++ u32 cs_id = ffs(tiler_oom) - 1; ++ ++ group_process_tiler_oom(group, cs_id); ++ tiler_oom &= ~BIT(cs_id); ++ } ++ ++ group_put(group); ++} ++ ++static void ++cs_slot_process_tiler_oom_event_locked(struct panthor_device *ptdev, ++ u32 csg_id, u32 cs_id) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ if (drm_WARN_ON(&ptdev->base, !group)) ++ return; ++ ++ atomic_or(BIT(cs_id), &group->tiler_oom); ++ ++ /* We don't use group_queue_work() here because we want to queue the ++ * work item to the heap_alloc_wq. ++ */ ++ group_get(group); ++ if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work)) ++ group_put(group); ++} ++ ++static bool cs_slot_process_irq_locked(struct panthor_device *ptdev, ++ u32 csg_id, u32 cs_id) ++{ ++ struct panthor_fw_cs_iface *cs_iface; ++ u32 req, ack, events; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ cs_iface = panthor_fw_get_cs_iface(ptdev, csg_id, cs_id); ++ req = cs_iface->input->req; ++ ack = cs_iface->output->ack; ++ events = (req ^ ack) & CS_EVT_MASK; ++ ++ if (events & CS_FATAL) ++ cs_slot_process_fatal_event_locked(ptdev, csg_id, cs_id); ++ ++ if (events & CS_FAULT) ++ cs_slot_process_fault_event_locked(ptdev, csg_id, cs_id); ++ ++ if (events & CS_TILER_OOM) ++ cs_slot_process_tiler_oom_event_locked(ptdev, csg_id, cs_id); ++ ++ /* We don't acknowledge the TILER_OOM event since its handling is ++ * deferred to a separate work. ++ */ ++ panthor_fw_update_reqs(cs_iface, req, ack, CS_FATAL | CS_FAULT); ++ ++ return (events & (CS_FAULT | CS_TILER_OOM)) != 0; ++} ++ ++static void csg_slot_sync_idle_state_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ struct panthor_fw_csg_iface *csg_iface; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ csg_slot->idle = csg_iface->output->status_state & CSG_STATUS_STATE_IS_IDLE; ++} ++ ++static void csg_slot_process_idle_event_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ sched->might_have_idle_groups = true; ++ ++ /* Schedule a tick so we can evict idle groups and schedule non-idle ++ * ones. This will also update runtime PM and devfreq busy/idle states, ++ * so the device can lower its frequency or get suspended. ++ */ ++ sched_queue_delayed_work(sched, tick, 0); ++} ++ ++static void csg_slot_sync_update_locked(struct panthor_device *ptdev, ++ u32 csg_id) ++{ ++ struct panthor_csg_slot *csg_slot = &ptdev->scheduler->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ if (group) ++ group_queue_work(group, sync_upd); ++ ++ sched_queue_work(ptdev->scheduler, sync_upd); ++} ++ ++static void ++csg_slot_process_progress_timer_event_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ struct panthor_group *group = csg_slot->group; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ drm_warn(&ptdev->base, "CSG slot %d progress timeout\n", csg_id); ++ ++ group = csg_slot->group; ++ if (!drm_WARN_ON(&ptdev->base, !group)) ++ group->timedout = true; ++ ++ sched_queue_delayed_work(sched, tick, 0); ++} ++ ++static void sched_process_csg_irq_locked(struct panthor_device *ptdev, u32 csg_id) ++{ ++ u32 req, ack, cs_irq_req, cs_irq_ack, cs_irqs, csg_events; ++ struct panthor_fw_csg_iface *csg_iface; ++ u32 ring_cs_db_mask = 0; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ if (drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) ++ return; ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ req = READ_ONCE(csg_iface->input->req); ++ ack = READ_ONCE(csg_iface->output->ack); ++ cs_irq_req = READ_ONCE(csg_iface->output->cs_irq_req); ++ cs_irq_ack = READ_ONCE(csg_iface->input->cs_irq_ack); ++ csg_events = (req ^ ack) & CSG_EVT_MASK; ++ ++ /* There may not be any pending CSG/CS interrupts to process */ ++ if (req == ack && cs_irq_req == cs_irq_ack) ++ return; ++ ++ /* Immediately set IRQ_ACK bits to be same as the IRQ_REQ bits before ++ * examining the CS_ACK & CS_REQ bits. This would ensure that Host ++ * doesn't miss an interrupt for the CS in the race scenario where ++ * whilst Host is servicing an interrupt for the CS, firmware sends ++ * another interrupt for that CS. ++ */ ++ csg_iface->input->cs_irq_ack = cs_irq_req; ++ ++ panthor_fw_update_reqs(csg_iface, req, ack, ++ CSG_SYNC_UPDATE | ++ CSG_IDLE | ++ CSG_PROGRESS_TIMER_EVENT); ++ ++ if (csg_events & CSG_IDLE) ++ csg_slot_process_idle_event_locked(ptdev, csg_id); ++ ++ if (csg_events & CSG_PROGRESS_TIMER_EVENT) ++ csg_slot_process_progress_timer_event_locked(ptdev, csg_id); ++ ++ cs_irqs = cs_irq_req ^ cs_irq_ack; ++ while (cs_irqs) { ++ u32 cs_id = ffs(cs_irqs) - 1; ++ ++ if (cs_slot_process_irq_locked(ptdev, csg_id, cs_id)) ++ ring_cs_db_mask |= BIT(cs_id); ++ ++ cs_irqs &= ~BIT(cs_id); ++ } ++ ++ if (csg_events & CSG_SYNC_UPDATE) ++ csg_slot_sync_update_locked(ptdev, csg_id); ++ ++ if (ring_cs_db_mask) ++ panthor_fw_toggle_reqs(csg_iface, doorbell_req, doorbell_ack, ring_cs_db_mask); ++ ++ panthor_fw_ring_csg_doorbells(ptdev, BIT(csg_id)); ++} ++ ++static void sched_process_idle_event_locked(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ /* Acknowledge the idle event and schedule a tick. */ ++ panthor_fw_update_reqs(glb_iface, req, glb_iface->output->ack, GLB_IDLE); ++ sched_queue_delayed_work(ptdev->scheduler, tick, 0); ++} ++ ++/** ++ * panthor_sched_process_global_irq() - Process the scheduling part of a global IRQ ++ * @ptdev: Device. ++ */ ++static void sched_process_global_irq_locked(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ u32 req, ack, evts; ++ ++ lockdep_assert_held(&ptdev->scheduler->lock); ++ ++ req = READ_ONCE(glb_iface->input->req); ++ ack = READ_ONCE(glb_iface->output->ack); ++ evts = (req ^ ack) & GLB_EVT_MASK; ++ ++ if (evts & GLB_IDLE) ++ sched_process_idle_event_locked(ptdev); ++} ++ ++static void process_fw_events_work(struct work_struct *work) ++{ ++ struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, ++ fw_events_work); ++ u32 events = atomic_xchg(&sched->fw_events, 0); ++ struct panthor_device *ptdev = sched->ptdev; ++ ++ mutex_lock(&sched->lock); ++ ++ if (events & JOB_INT_GLOBAL_IF) { ++ sched_process_global_irq_locked(ptdev); ++ events &= ~JOB_INT_GLOBAL_IF; ++ } ++ ++ while (events) { ++ u32 csg_id = ffs(events) - 1; ++ ++ sched_process_csg_irq_locked(ptdev, csg_id); ++ events &= ~BIT(csg_id); ++ } ++ ++ mutex_unlock(&sched->lock); ++} ++ ++/** ++ * panthor_sched_report_fw_events() - Report FW events to the scheduler. ++ */ ++void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events) ++{ ++ if (!ptdev->scheduler) ++ return; ++ ++ atomic_or(events, &ptdev->scheduler->fw_events); ++ sched_queue_work(ptdev->scheduler, fw_events); ++} ++ ++static const char *fence_get_driver_name(struct dma_fence *fence) ++{ ++ return "panthor"; ++} ++ ++static const char *queue_fence_get_timeline_name(struct dma_fence *fence) ++{ ++ return "queue-fence"; ++} ++ ++static const struct dma_fence_ops panthor_queue_fence_ops = { ++ .get_driver_name = fence_get_driver_name, ++ .get_timeline_name = queue_fence_get_timeline_name, ++}; ++ ++/** ++ */ ++struct panthor_csg_slots_upd_ctx { ++ u32 update_mask; ++ u32 timedout_mask; ++ struct { ++ u32 value; ++ u32 mask; ++ } requests[MAX_CSGS]; ++}; ++ ++static void csgs_upd_ctx_init(struct panthor_csg_slots_upd_ctx *ctx) ++{ ++ memset(ctx, 0, sizeof(*ctx)); ++} ++ ++static void csgs_upd_ctx_queue_reqs(struct panthor_device *ptdev, ++ struct panthor_csg_slots_upd_ctx *ctx, ++ u32 csg_id, u32 value, u32 mask) ++{ ++ if (drm_WARN_ON(&ptdev->base, !mask) || ++ drm_WARN_ON(&ptdev->base, csg_id >= ptdev->scheduler->csg_slot_count)) ++ return; ++ ++ ctx->requests[csg_id].value = (ctx->requests[csg_id].value & ~mask) | (value & mask); ++ ctx->requests[csg_id].mask |= mask; ++ ctx->update_mask |= BIT(csg_id); ++} ++ ++static int csgs_upd_ctx_apply_locked(struct panthor_device *ptdev, ++ struct panthor_csg_slots_upd_ctx *ctx) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ u32 update_slots = ctx->update_mask; ++ ++ lockdep_assert_held(&sched->lock); ++ ++ if (!ctx->update_mask) ++ return 0; ++ ++ while (update_slots) { ++ struct panthor_fw_csg_iface *csg_iface; ++ u32 csg_id = ffs(update_slots) - 1; ++ ++ update_slots &= ~BIT(csg_id); ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ panthor_fw_update_reqs(csg_iface, req, ++ ctx->requests[csg_id].value, ++ ctx->requests[csg_id].mask); ++ } ++ ++ panthor_fw_ring_csg_doorbells(ptdev, ctx->update_mask); ++ ++ update_slots = ctx->update_mask; ++ while (update_slots) { ++ struct panthor_fw_csg_iface *csg_iface; ++ u32 csg_id = ffs(update_slots) - 1; ++ u32 req_mask = ctx->requests[csg_id].mask, acked; ++ int ret; ++ ++ update_slots &= ~BIT(csg_id); ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ ++ ret = panthor_fw_csg_wait_acks(ptdev, csg_id, req_mask, &acked, 100); ++ ++ if (acked & CSG_ENDPOINT_CONFIG) ++ csg_slot_sync_priority_locked(ptdev, csg_id); ++ ++ if (acked & CSG_STATE_MASK) ++ csg_slot_sync_state_locked(ptdev, csg_id); ++ ++ if (acked & CSG_STATUS_UPDATE) { ++ csg_slot_sync_queues_state_locked(ptdev, csg_id); ++ csg_slot_sync_idle_state_locked(ptdev, csg_id); ++ } ++ ++ if (ret && acked != req_mask && ++ ((csg_iface->input->req ^ csg_iface->output->ack) & req_mask) != 0) { ++ drm_err(&ptdev->base, "CSG %d update request timedout", csg_id); ++ ctx->timedout_mask |= BIT(csg_id); ++ } ++ } ++ ++ if (ctx->timedout_mask) ++ return -ETIMEDOUT; ++ ++ return 0; ++} ++ ++struct panthor_sched_tick_ctx { ++ struct list_head old_groups[PANTHOR_CSG_PRIORITY_COUNT]; ++ struct list_head groups[PANTHOR_CSG_PRIORITY_COUNT]; ++ u32 idle_group_count; ++ u32 group_count; ++ enum panthor_csg_priority min_priority; ++ struct panthor_vm *vms[MAX_CS_PER_CSG]; ++ u32 as_count; ++ bool immediate_tick; ++ u32 csg_upd_failed_mask; ++}; ++ ++static bool ++tick_ctx_is_full(const struct panthor_scheduler *sched, ++ const struct panthor_sched_tick_ctx *ctx) ++{ ++ return ctx->group_count == sched->csg_slot_count; ++} ++ ++static bool ++group_is_idle(struct panthor_group *group) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ u32 inactive_queues; ++ ++ if (group->csg_id >= 0) ++ return ptdev->scheduler->csg_slots[group->csg_id].idle; ++ ++ inactive_queues = group->idle_queues | group->blocked_queues; ++ return hweight32(inactive_queues) == group->queue_count; ++} ++ ++static bool ++group_can_run(struct panthor_group *group) ++{ ++ return group->state != PANTHOR_CS_GROUP_TERMINATED && ++ !group->destroyed && group->fatal_queues == 0 && ++ !group->timedout; ++} ++ ++static void ++tick_ctx_pick_groups_from_list(const struct panthor_scheduler *sched, ++ struct panthor_sched_tick_ctx *ctx, ++ struct list_head *queue, ++ bool skip_idle_groups, ++ bool owned_by_tick_ctx) ++{ ++ struct panthor_group *group, *tmp; ++ ++ if (tick_ctx_is_full(sched, ctx)) ++ return; ++ ++ list_for_each_entry_safe(group, tmp, queue, run_node) { ++ u32 i; ++ ++ if (!group_can_run(group)) ++ continue; ++ ++ if (skip_idle_groups && group_is_idle(group)) ++ continue; ++ ++ for (i = 0; i < ctx->as_count; i++) { ++ if (ctx->vms[i] == group->vm) ++ break; ++ } ++ ++ if (i == ctx->as_count && ctx->as_count == sched->as_slot_count) ++ continue; ++ ++ if (!owned_by_tick_ctx) ++ group_get(group); ++ ++ list_move_tail(&group->run_node, &ctx->groups[group->priority]); ++ ctx->group_count++; ++ if (group_is_idle(group)) ++ ctx->idle_group_count++; ++ ++ if (i == ctx->as_count) ++ ctx->vms[ctx->as_count++] = group->vm; ++ ++ if (ctx->min_priority > group->priority) ++ ctx->min_priority = group->priority; ++ ++ if (tick_ctx_is_full(sched, ctx)) ++ return; ++ } ++} ++ ++static void ++tick_ctx_insert_old_group(struct panthor_scheduler *sched, ++ struct panthor_sched_tick_ctx *ctx, ++ struct panthor_group *group, ++ bool full_tick) ++{ ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; ++ struct panthor_group *other_group; ++ ++ if (!full_tick) { ++ list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); ++ return; ++ } ++ ++ /* Rotate to make sure groups with lower CSG slot ++ * priorities have a chance to get a higher CSG slot ++ * priority next time they get picked. This priority ++ * has an impact on resource request ordering, so it's ++ * important to make sure we don't let one group starve ++ * all other groups with the same group priority. ++ */ ++ list_for_each_entry(other_group, ++ &ctx->old_groups[csg_slot->group->priority], ++ run_node) { ++ struct panthor_csg_slot *other_csg_slot = &sched->csg_slots[other_group->csg_id]; ++ ++ if (other_csg_slot->priority > csg_slot->priority) { ++ list_add_tail(&csg_slot->group->run_node, &other_group->run_node); ++ return; ++ } ++ } ++ ++ list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); ++} ++ ++static void ++tick_ctx_init(struct panthor_scheduler *sched, ++ struct panthor_sched_tick_ctx *ctx, ++ bool full_tick) ++{ ++ struct panthor_device *ptdev = sched->ptdev; ++ struct panthor_csg_slots_upd_ctx upd_ctx; ++ int ret; ++ u32 i; ++ ++ memset(ctx, 0, sizeof(*ctx)); ++ csgs_upd_ctx_init(&upd_ctx); ++ ++ ctx->min_priority = PANTHOR_CSG_PRIORITY_COUNT; ++ for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { ++ INIT_LIST_HEAD(&ctx->groups[i]); ++ INIT_LIST_HEAD(&ctx->old_groups[i]); ++ } ++ ++ for (i = 0; i < sched->csg_slot_count; i++) { ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; ++ struct panthor_group *group = csg_slot->group; ++ struct panthor_fw_csg_iface *csg_iface; ++ ++ if (!group) ++ continue; ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, i); ++ group_get(group); ++ ++ /* If there was unhandled faults on the VM, force processing of ++ * CSG IRQs, so we can flag the faulty queue. ++ */ ++ if (panthor_vm_has_unhandled_faults(group->vm)) { ++ sched_process_csg_irq_locked(ptdev, i); ++ ++ /* No fatal fault reported, flag all queues as faulty. */ ++ if (!group->fatal_queues) ++ group->fatal_queues |= GENMASK(group->queue_count - 1, 0); ++ } ++ ++ tick_ctx_insert_old_group(sched, ctx, group, full_tick); ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, ++ csg_iface->output->ack ^ CSG_STATUS_UPDATE, ++ CSG_STATUS_UPDATE); ++ } ++ ++ ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); ++ if (ret) { ++ panthor_device_schedule_reset(ptdev); ++ ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; ++ } ++} ++ ++#define NUM_INSTRS_PER_SLOT 16 ++ ++static void ++group_term_post_processing(struct panthor_group *group) ++{ ++ struct panthor_job *job, *tmp; ++ LIST_HEAD(faulty_jobs); ++ bool cookie; ++ u32 i = 0; ++ ++ if (drm_WARN_ON(&group->ptdev->base, group_can_run(group))) ++ return; ++ ++ cookie = dma_fence_begin_signalling(); ++ for (i = 0; i < group->queue_count; i++) { ++ struct panthor_queue *queue = group->queues[i]; ++ struct panthor_syncobj_64b *syncobj; ++ int err; ++ ++ if (group->fatal_queues & BIT(i)) ++ err = -EINVAL; ++ else if (group->timedout) ++ err = -ETIMEDOUT; ++ else ++ err = -ECANCELED; ++ ++ if (!queue) ++ continue; ++ ++ spin_lock(&queue->fence_ctx.lock); ++ list_for_each_entry_safe(job, tmp, &queue->fence_ctx.in_flight_jobs, node) { ++ list_move_tail(&job->node, &faulty_jobs); ++ dma_fence_set_error(job->done_fence, err); ++ dma_fence_signal_locked(job->done_fence); ++ } ++ spin_unlock(&queue->fence_ctx.lock); ++ ++ /* Manually update the syncobj seqno to unblock waiters. */ ++ syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); ++ syncobj->status = ~0; ++ syncobj->seqno = atomic64_read(&queue->fence_ctx.seqno); ++ sched_queue_work(group->ptdev->scheduler, sync_upd); ++ } ++ dma_fence_end_signalling(cookie); ++ ++ list_for_each_entry_safe(job, tmp, &faulty_jobs, node) { ++ list_del_init(&job->node); ++ panthor_job_put(&job->base); ++ } ++} ++ ++static void group_term_work(struct work_struct *work) ++{ ++ struct panthor_group *group = ++ container_of(work, struct panthor_group, term_work); ++ ++ group_term_post_processing(group); ++ group_put(group); ++} ++ ++static void ++tick_ctx_cleanup(struct panthor_scheduler *sched, ++ struct panthor_sched_tick_ctx *ctx) ++{ ++ struct panthor_group *group, *tmp; ++ u32 i; ++ ++ for (i = 0; i < ARRAY_SIZE(ctx->old_groups); i++) { ++ list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) { ++ /* If everything went fine, we should only have groups ++ * to be terminated in the old_groups lists. ++ */ ++ drm_WARN_ON(&group->ptdev->base, !ctx->csg_upd_failed_mask && ++ group_can_run(group)); ++ ++ if (!group_can_run(group)) { ++ list_del_init(&group->run_node); ++ list_del_init(&group->wait_node); ++ group_queue_work(group, term); ++ } else if (group->csg_id >= 0) { ++ list_del_init(&group->run_node); ++ } else { ++ list_move(&group->run_node, ++ group_is_idle(group) ? ++ &sched->groups.idle[group->priority] : ++ &sched->groups.runnable[group->priority]); ++ } ++ group_put(group); ++ } ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(ctx->groups); i++) { ++ /* If everything went fine, the groups to schedule lists should ++ * be empty. ++ */ ++ drm_WARN_ON(&group->ptdev->base, ++ !ctx->csg_upd_failed_mask && !list_empty(&ctx->groups[i])); ++ ++ list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) { ++ if (group->csg_id >= 0) { ++ list_del_init(&group->run_node); ++ } else { ++ list_move(&group->run_node, ++ group_is_idle(group) ? ++ &sched->groups.idle[group->priority] : ++ &sched->groups.runnable[group->priority]); ++ } ++ group_put(group); ++ } ++ } ++} ++ ++static void ++tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *ctx) ++{ ++ struct panthor_group *group, *tmp; ++ struct panthor_device *ptdev = sched->ptdev; ++ struct panthor_csg_slot *csg_slot; ++ int prio, new_csg_prio = MAX_CSG_PRIO, i; ++ u32 csg_mod_mask = 0, free_csg_slots = 0; ++ struct panthor_csg_slots_upd_ctx upd_ctx; ++ int ret; ++ ++ csgs_upd_ctx_init(&upd_ctx); ++ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ /* Suspend or terminate evicted groups. */ ++ list_for_each_entry(group, &ctx->old_groups[prio], run_node) { ++ bool term = !group_can_run(group); ++ int csg_id = group->csg_id; ++ ++ if (drm_WARN_ON(&ptdev->base, csg_id < 0)) ++ continue; ++ ++ csg_slot = &sched->csg_slots[csg_id]; ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, ++ term ? CSG_STATE_TERMINATE : CSG_STATE_SUSPEND, ++ CSG_STATE_MASK); ++ } ++ ++ /* Update priorities on already running groups. */ ++ list_for_each_entry(group, &ctx->groups[prio], run_node) { ++ struct panthor_fw_csg_iface *csg_iface; ++ int csg_id = group->csg_id; ++ ++ if (csg_id < 0) { ++ new_csg_prio--; ++ continue; ++ } ++ ++ csg_slot = &sched->csg_slots[csg_id]; ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ if (csg_slot->priority == new_csg_prio) { ++ new_csg_prio--; ++ continue; ++ } ++ ++ panthor_fw_update_reqs(csg_iface, endpoint_req, ++ CSG_EP_REQ_PRIORITY(new_csg_prio), ++ CSG_EP_REQ_PRIORITY_MASK); ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, ++ csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, ++ CSG_ENDPOINT_CONFIG); ++ new_csg_prio--; ++ } ++ } ++ ++ ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); ++ if (ret) { ++ panthor_device_schedule_reset(ptdev); ++ ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; ++ return; ++ } ++ ++ /* Unbind evicted groups. */ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ list_for_each_entry(group, &ctx->old_groups[prio], run_node) { ++ /* This group is gone. Process interrupts to clear ++ * any pending interrupts before we start the new ++ * group. ++ */ ++ if (group->csg_id >= 0) ++ sched_process_csg_irq_locked(ptdev, group->csg_id); ++ ++ group_unbind_locked(group); ++ } ++ } ++ ++ for (i = 0; i < sched->csg_slot_count; i++) { ++ if (!sched->csg_slots[i].group) ++ free_csg_slots |= BIT(i); ++ } ++ ++ csgs_upd_ctx_init(&upd_ctx); ++ new_csg_prio = MAX_CSG_PRIO; ++ ++ /* Start new groups. */ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ list_for_each_entry(group, &ctx->groups[prio], run_node) { ++ int csg_id = group->csg_id; ++ struct panthor_fw_csg_iface *csg_iface; ++ ++ if (csg_id >= 0) { ++ new_csg_prio--; ++ continue; ++ } ++ ++ csg_id = ffs(free_csg_slots) - 1; ++ if (drm_WARN_ON(&ptdev->base, csg_id < 0)) ++ break; ++ ++ csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); ++ csg_slot = &sched->csg_slots[csg_id]; ++ csg_mod_mask |= BIT(csg_id); ++ group_bind_locked(group, csg_id); ++ csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--); ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, ++ group->state == PANTHOR_CS_GROUP_SUSPENDED ? ++ CSG_STATE_RESUME : CSG_STATE_START, ++ CSG_STATE_MASK); ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, ++ csg_iface->output->ack ^ CSG_ENDPOINT_CONFIG, ++ CSG_ENDPOINT_CONFIG); ++ free_csg_slots &= ~BIT(csg_id); ++ } ++ } ++ ++ ret = csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); ++ if (ret) { ++ panthor_device_schedule_reset(ptdev); ++ ctx->csg_upd_failed_mask |= upd_ctx.timedout_mask; ++ return; ++ } ++ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) { ++ list_del_init(&group->run_node); ++ ++ /* If the group has been destroyed while we were ++ * scheduling, ask for an immediate tick to ++ * re-evaluate as soon as possible and get rid of ++ * this dangling group. ++ */ ++ if (group->destroyed) ++ ctx->immediate_tick = true; ++ group_put(group); ++ } ++ ++ /* Return evicted groups to the idle or run queues. Groups ++ * that can no longer be run (because they've been destroyed ++ * or experienced an unrecoverable error) will be scheduled ++ * for destruction in tick_ctx_cleanup(). ++ */ ++ list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) { ++ if (!group_can_run(group)) ++ continue; ++ ++ if (group_is_idle(group)) ++ list_move_tail(&group->run_node, &sched->groups.idle[prio]); ++ else ++ list_move_tail(&group->run_node, &sched->groups.runnable[prio]); ++ group_put(group); ++ } ++ } ++ ++ sched->used_csg_slot_count = ctx->group_count; ++ sched->might_have_idle_groups = ctx->idle_group_count > 0; ++} ++ ++static u64 ++tick_ctx_update_resched_target(struct panthor_scheduler *sched, ++ const struct panthor_sched_tick_ctx *ctx) ++{ ++ /* We had space left, no need to reschedule until some external event happens. */ ++ if (!tick_ctx_is_full(sched, ctx)) ++ goto no_tick; ++ ++ /* If idle groups were scheduled, no need to wake up until some external ++ * event happens (group unblocked, new job submitted, ...). ++ */ ++ if (ctx->idle_group_count) ++ goto no_tick; ++ ++ if (drm_WARN_ON(&sched->ptdev->base, ctx->min_priority >= PANTHOR_CSG_PRIORITY_COUNT)) ++ goto no_tick; ++ ++ /* If there are groups of the same priority waiting, we need to ++ * keep the scheduler ticking, otherwise, we'll just wait for ++ * new groups with higher priority to be queued. ++ */ ++ if (!list_empty(&sched->groups.runnable[ctx->min_priority])) { ++ u64 resched_target = sched->last_tick + sched->tick_period; ++ ++ if (time_before64(sched->resched_target, sched->last_tick) || ++ time_before64(resched_target, sched->resched_target)) ++ sched->resched_target = resched_target; ++ ++ return sched->resched_target - sched->last_tick; ++ } ++ ++no_tick: ++ sched->resched_target = U64_MAX; ++ return U64_MAX; ++} ++ ++static void tick_work(struct work_struct *work) ++{ ++ struct panthor_scheduler *sched = container_of(work, struct panthor_scheduler, ++ tick_work.work); ++ struct panthor_device *ptdev = sched->ptdev; ++ struct panthor_sched_tick_ctx ctx; ++ u64 remaining_jiffies = 0, resched_delay; ++ u64 now = get_jiffies_64(); ++ int prio, ret, cookie; ++ ++ if (!drm_dev_enter(&ptdev->base, &cookie)) ++ return; ++ ++ ret = pm_runtime_resume_and_get(ptdev->base.dev); ++ if (drm_WARN_ON(&ptdev->base, ret)) ++ goto out_dev_exit; ++ ++ if (time_before64(now, sched->resched_target)) ++ remaining_jiffies = sched->resched_target - now; ++ ++ mutex_lock(&sched->lock); ++ if (panthor_device_reset_is_pending(sched->ptdev)) ++ goto out_unlock; ++ ++ tick_ctx_init(sched, &ctx, remaining_jiffies != 0); ++ if (ctx.csg_upd_failed_mask) ++ goto out_cleanup_ctx; ++ ++ if (remaining_jiffies) { ++ /* Scheduling forced in the middle of a tick. Only RT groups ++ * can preempt non-RT ones. Currently running RT groups can't be ++ * preempted. ++ */ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; ++ prio >= 0 && !tick_ctx_is_full(sched, &ctx); ++ prio--) { ++ tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], ++ true, true); ++ if (prio == PANTHOR_CSG_PRIORITY_RT) { ++ tick_ctx_pick_groups_from_list(sched, &ctx, ++ &sched->groups.runnable[prio], ++ true, false); ++ } ++ } ++ } ++ ++ /* First pick non-idle groups */ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; ++ prio >= 0 && !tick_ctx_is_full(sched, &ctx); ++ prio--) { ++ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.runnable[prio], ++ true, false); ++ tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], true, true); ++ } ++ ++ /* If we have free CSG slots left, pick idle groups */ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; ++ prio >= 0 && !tick_ctx_is_full(sched, &ctx); ++ prio--) { ++ /* Check the old_group queue first to avoid reprogramming the slots */ ++ tick_ctx_pick_groups_from_list(sched, &ctx, &ctx.old_groups[prio], false, true); ++ tick_ctx_pick_groups_from_list(sched, &ctx, &sched->groups.idle[prio], ++ false, false); ++ } ++ ++ tick_ctx_apply(sched, &ctx); ++ if (ctx.csg_upd_failed_mask) ++ goto out_cleanup_ctx; ++ ++ if (ctx.idle_group_count == ctx.group_count) { ++ panthor_devfreq_record_idle(sched->ptdev); ++ if (sched->pm.has_ref) { ++ pm_runtime_put_autosuspend(ptdev->base.dev); ++ sched->pm.has_ref = false; ++ } ++ } else { ++ panthor_devfreq_record_busy(sched->ptdev); ++ if (!sched->pm.has_ref) { ++ pm_runtime_get(ptdev->base.dev); ++ sched->pm.has_ref = true; ++ } ++ } ++ ++ sched->last_tick = now; ++ resched_delay = tick_ctx_update_resched_target(sched, &ctx); ++ if (ctx.immediate_tick) ++ resched_delay = 0; ++ ++ if (resched_delay != U64_MAX) ++ sched_queue_delayed_work(sched, tick, resched_delay); ++ ++out_cleanup_ctx: ++ tick_ctx_cleanup(sched, &ctx); ++ ++out_unlock: ++ mutex_unlock(&sched->lock); ++ pm_runtime_mark_last_busy(ptdev->base.dev); ++ pm_runtime_put_autosuspend(ptdev->base.dev); ++ ++out_dev_exit: ++ drm_dev_exit(cookie); ++} ++ ++static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx) ++{ ++ struct panthor_queue *queue = group->queues[queue_idx]; ++ union { ++ struct panthor_syncobj_64b sync64; ++ struct panthor_syncobj_32b sync32; ++ } *syncobj; ++ bool result; ++ u64 value; ++ ++ syncobj = panthor_queue_get_syncwait_obj(group, queue); ++ if (!syncobj) ++ return -EINVAL; ++ ++ value = queue->syncwait.sync64 ? ++ syncobj->sync64.seqno : ++ syncobj->sync32.seqno; ++ ++ if (queue->syncwait.gt) ++ result = value > queue->syncwait.ref; ++ else ++ result = value <= queue->syncwait.ref; ++ ++ if (result) ++ panthor_queue_put_syncwait_obj(queue); ++ ++ return result; ++} ++ ++static void sync_upd_work(struct work_struct *work) ++{ ++ struct panthor_scheduler *sched = container_of(work, ++ struct panthor_scheduler, ++ sync_upd_work); ++ struct panthor_group *group, *tmp; ++ bool immediate_tick = false; ++ ++ mutex_lock(&sched->lock); ++ list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) { ++ u32 tested_queues = group->blocked_queues; ++ u32 unblocked_queues = 0; ++ ++ while (tested_queues) { ++ u32 cs_id = ffs(tested_queues) - 1; ++ int ret; ++ ++ ret = panthor_queue_eval_syncwait(group, cs_id); ++ drm_WARN_ON(&group->ptdev->base, ret < 0); ++ if (ret) ++ unblocked_queues |= BIT(cs_id); ++ ++ tested_queues &= ~BIT(cs_id); ++ } ++ ++ if (unblocked_queues) { ++ group->blocked_queues &= ~unblocked_queues; ++ ++ if (group->csg_id < 0) { ++ list_move(&group->run_node, ++ &sched->groups.runnable[group->priority]); ++ if (group->priority == PANTHOR_CSG_PRIORITY_RT) ++ immediate_tick = true; ++ } ++ } ++ ++ if (!group->blocked_queues) ++ list_del_init(&group->wait_node); ++ } ++ mutex_unlock(&sched->lock); ++ ++ if (immediate_tick) ++ sched_queue_delayed_work(sched, tick, 0); ++} ++ ++static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) ++{ ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct list_head *queue = &sched->groups.runnable[group->priority]; ++ u64 delay_jiffies = 0; ++ bool was_idle; ++ u64 now; ++ ++ if (!group_can_run(group)) ++ return; ++ ++ /* All updated queues are blocked, no need to wake up the scheduler. */ ++ if ((queue_mask & group->blocked_queues) == queue_mask) ++ return; ++ ++ was_idle = group_is_idle(group); ++ group->idle_queues &= ~queue_mask; ++ ++ /* Don't mess up with the lists if we're in a middle of a reset. */ ++ if (atomic_read(&sched->reset.in_progress)) ++ return; ++ ++ if (was_idle && !group_is_idle(group)) ++ list_move_tail(&group->run_node, queue); ++ ++ /* RT groups are preemptive. */ ++ if (group->priority == PANTHOR_CSG_PRIORITY_RT) { ++ sched_queue_delayed_work(sched, tick, 0); ++ return; ++ } ++ ++ /* Some groups might be idle, force an immediate tick to ++ * re-evaluate. ++ */ ++ if (sched->might_have_idle_groups) { ++ sched_queue_delayed_work(sched, tick, 0); ++ return; ++ } ++ ++ /* Scheduler is ticking, nothing to do. */ ++ if (sched->resched_target != U64_MAX) { ++ /* If there are free slots, force immediating ticking. */ ++ if (sched->used_csg_slot_count < sched->csg_slot_count) ++ sched_queue_delayed_work(sched, tick, 0); ++ ++ return; ++ } ++ ++ /* Scheduler tick was off, recalculate the resched_target based on the ++ * last tick event, and queue the scheduler work. ++ */ ++ now = get_jiffies_64(); ++ sched->resched_target = sched->last_tick + sched->tick_period; ++ if (sched->used_csg_slot_count == sched->csg_slot_count && ++ time_before64(now, sched->resched_target)) ++ delay_jiffies = min_t(unsigned long, sched->resched_target - now, ULONG_MAX); ++ ++ sched_queue_delayed_work(sched, tick, delay_jiffies); ++} ++ ++static void queue_stop(struct panthor_queue *queue, ++ struct panthor_job *bad_job) ++{ ++ drm_sched_stop(&queue->scheduler, bad_job ? &bad_job->base : NULL); ++} ++ ++static void queue_start(struct panthor_queue *queue) ++{ ++ struct panthor_job *job; ++ ++ /* Re-assign the parent fences. */ ++ list_for_each_entry(job, &queue->scheduler.pending_list, base.list) ++ job->base.s_fence->parent = dma_fence_get(job->done_fence); ++ ++ drm_sched_start(&queue->scheduler, true); ++} ++ ++static void panthor_group_stop(struct panthor_group *group) ++{ ++ struct panthor_scheduler *sched = group->ptdev->scheduler; ++ ++ lockdep_assert_held(&sched->reset.lock); ++ ++ for (u32 i = 0; i < group->queue_count; i++) ++ queue_stop(group->queues[i], NULL); ++ ++ group_get(group); ++ list_move_tail(&group->run_node, &sched->reset.stopped_groups); ++} ++ ++static void panthor_group_start(struct panthor_group *group) ++{ ++ struct panthor_scheduler *sched = group->ptdev->scheduler; ++ ++ lockdep_assert_held(&group->ptdev->scheduler->reset.lock); ++ ++ for (u32 i = 0; i < group->queue_count; i++) ++ queue_start(group->queues[i]); ++ ++ if (group_can_run(group)) { ++ list_move_tail(&group->run_node, ++ group_is_idle(group) ? ++ &sched->groups.idle[group->priority] : ++ &sched->groups.runnable[group->priority]); ++ } else { ++ list_del_init(&group->run_node); ++ list_del_init(&group->wait_node); ++ group_queue_work(group, term); ++ } ++ ++ group_put(group); ++} ++ ++static void panthor_sched_immediate_tick(struct panthor_device *ptdev) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ ++ sched_queue_delayed_work(sched, tick, 0); ++} ++ ++/** ++ * panthor_sched_report_mmu_fault() - Report MMU faults to the scheduler. ++ */ ++void panthor_sched_report_mmu_fault(struct panthor_device *ptdev) ++{ ++ /* Force a tick to immediately kill faulty groups. */ ++ if (ptdev->scheduler) ++ panthor_sched_immediate_tick(ptdev); ++} ++ ++void panthor_sched_resume(struct panthor_device *ptdev) ++{ ++ /* Force a tick to re-evaluate after a resume. */ ++ panthor_sched_immediate_tick(ptdev); ++} ++ ++void panthor_sched_suspend(struct panthor_device *ptdev) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_csg_slots_upd_ctx upd_ctx; ++ u64 suspended_slots, faulty_slots; ++ struct panthor_group *group; ++ u32 i; ++ ++ mutex_lock(&sched->lock); ++ csgs_upd_ctx_init(&upd_ctx); ++ for (i = 0; i < sched->csg_slot_count; i++) { ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; ++ ++ if (csg_slot->group) { ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, ++ CSG_STATE_SUSPEND, ++ CSG_STATE_MASK); ++ } ++ } ++ ++ suspended_slots = upd_ctx.update_mask; ++ ++ csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); ++ suspended_slots &= ~upd_ctx.timedout_mask; ++ faulty_slots = upd_ctx.timedout_mask; ++ ++ if (faulty_slots) { ++ u32 slot_mask = faulty_slots; ++ ++ drm_err(&ptdev->base, "CSG suspend failed, escalating to termination"); ++ csgs_upd_ctx_init(&upd_ctx); ++ while (slot_mask) { ++ u32 csg_id = ffs(slot_mask) - 1; ++ ++ csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, ++ CSG_STATE_TERMINATE, ++ CSG_STATE_MASK); ++ slot_mask &= ~BIT(csg_id); ++ } ++ ++ csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); ++ ++ slot_mask = upd_ctx.timedout_mask; ++ while (slot_mask) { ++ u32 csg_id = ffs(slot_mask) - 1; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ ++ /* Terminate command timedout, but the soft-reset will ++ * automatically terminate all active groups, so let's ++ * force the state to halted here. ++ */ ++ if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED) ++ csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; ++ slot_mask &= ~BIT(csg_id); ++ } ++ } ++ ++ /* Flush L2 and LSC caches to make sure suspend state is up-to-date. ++ * If the flush fails, flag all queues for termination. ++ */ ++ if (suspended_slots) { ++ bool flush_caches_failed = false; ++ u32 slot_mask = suspended_slots; ++ ++ if (panthor_gpu_flush_caches(ptdev, CACHE_CLEAN, CACHE_CLEAN, 0)) ++ flush_caches_failed = true; ++ ++ while (slot_mask) { ++ u32 csg_id = ffs(slot_mask) - 1; ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[csg_id]; ++ ++ if (flush_caches_failed) ++ csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; ++ else ++ csg_slot_sync_update_locked(ptdev, csg_id); ++ ++ slot_mask &= ~BIT(csg_id); ++ } ++ ++ if (flush_caches_failed) ++ faulty_slots |= suspended_slots; ++ } ++ ++ for (i = 0; i < sched->csg_slot_count; i++) { ++ struct panthor_csg_slot *csg_slot = &sched->csg_slots[i]; ++ ++ group = csg_slot->group; ++ if (!group) ++ continue; ++ ++ group_get(group); ++ ++ if (group->csg_id >= 0) ++ sched_process_csg_irq_locked(ptdev, group->csg_id); ++ ++ group_unbind_locked(group); ++ ++ drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node)); ++ ++ if (group_can_run(group)) { ++ list_add(&group->run_node, ++ &sched->groups.idle[group->priority]); ++ } else { ++ /* We don't bother stopping the scheduler if the group is ++ * faulty, the group termination work will finish the job. ++ */ ++ list_del_init(&group->wait_node); ++ group_queue_work(group, term); ++ } ++ group_put(group); ++ } ++ mutex_unlock(&sched->lock); ++} ++ ++void panthor_sched_pre_reset(struct panthor_device *ptdev) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_group *group, *group_tmp; ++ u32 i; ++ ++ mutex_lock(&sched->reset.lock); ++ atomic_set(&sched->reset.in_progress, true); ++ ++ /* Cancel all scheduler works. Once this is done, these works can't be ++ * scheduled again until the reset operation is complete. ++ */ ++ cancel_work_sync(&sched->sync_upd_work); ++ cancel_delayed_work_sync(&sched->tick_work); ++ ++ panthor_sched_suspend(ptdev); ++ ++ /* Stop all groups that might still accept jobs, so we don't get passed ++ * new jobs while we're resetting. ++ */ ++ for (i = 0; i < ARRAY_SIZE(sched->groups.runnable); i++) { ++ /* All groups should be in the idle lists. */ ++ drm_WARN_ON(&ptdev->base, !list_empty(&sched->groups.runnable[i])); ++ list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node) ++ panthor_group_stop(group); ++ } ++ ++ for (i = 0; i < ARRAY_SIZE(sched->groups.idle); i++) { ++ list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node) ++ panthor_group_stop(group); ++ } ++ ++ mutex_unlock(&sched->reset.lock); ++} ++ ++void panthor_sched_post_reset(struct panthor_device *ptdev) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_group *group, *group_tmp; ++ ++ mutex_lock(&sched->reset.lock); ++ ++ list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) ++ panthor_group_start(group); ++ ++ /* We're done resetting the GPU, clear the reset.in_progress bit so we can ++ * kick the scheduler. ++ */ ++ atomic_set(&sched->reset.in_progress, false); ++ mutex_unlock(&sched->reset.lock); ++ ++ sched_queue_delayed_work(sched, tick, 0); ++ ++ sched_queue_work(sched, sync_upd); ++} ++ ++static void group_sync_upd_work(struct work_struct *work) ++{ ++ struct panthor_group *group = ++ container_of(work, struct panthor_group, sync_upd_work); ++ struct panthor_job *job, *job_tmp; ++ LIST_HEAD(done_jobs); ++ u32 queue_idx; ++ bool cookie; ++ ++ cookie = dma_fence_begin_signalling(); ++ for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { ++ struct panthor_queue *queue = group->queues[queue_idx]; ++ struct panthor_syncobj_64b *syncobj; ++ ++ if (!queue) ++ continue; ++ ++ syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj)); ++ ++ spin_lock(&queue->fence_ctx.lock); ++ list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { ++ if (!job->call_info.size) ++ continue; ++ ++ if (syncobj->seqno < job->done_fence->seqno) ++ break; ++ ++ list_move_tail(&job->node, &done_jobs); ++ dma_fence_signal_locked(job->done_fence); ++ } ++ spin_unlock(&queue->fence_ctx.lock); ++ } ++ dma_fence_end_signalling(cookie); ++ ++ list_for_each_entry_safe(job, job_tmp, &done_jobs, node) { ++ list_del_init(&job->node); ++ panthor_job_put(&job->base); ++ } ++ ++ group_put(group); ++} ++ ++static struct dma_fence * ++queue_run_job(struct drm_sched_job *sched_job) ++{ ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ struct panthor_group *group = job->group; ++ struct panthor_queue *queue = group->queues[job->queue_idx]; ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ u32 ringbuf_size = panthor_kernel_bo_size(queue->ringbuf); ++ u32 ringbuf_insert = queue->iface.input->insert & (ringbuf_size - 1); ++ u64 addr_reg = ptdev->csif_info.cs_reg_count - ++ ptdev->csif_info.unpreserved_cs_reg_count; ++ u64 val_reg = addr_reg + 2; ++ u64 sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) + ++ job->queue_idx * sizeof(struct panthor_syncobj_64b); ++ u32 waitall_mask = GENMASK(sched->sb_slot_count - 1, 0); ++ struct dma_fence *done_fence; ++ int ret; ++ ++ u64 call_instrs[NUM_INSTRS_PER_SLOT] = { ++ /* MOV32 rX+2, cs.latest_flush */ ++ (2ull << 56) | (val_reg << 48) | job->call_info.latest_flush, ++ ++ /* FLUSH_CACHE2.clean_inv_all.no_wait.signal(0) rX+2 */ ++ (36ull << 56) | (0ull << 48) | (val_reg << 40) | (0 << 16) | 0x233, ++ ++ /* MOV48 rX:rX+1, cs.start */ ++ (1ull << 56) | (addr_reg << 48) | job->call_info.start, ++ ++ /* MOV32 rX+2, cs.size */ ++ (2ull << 56) | (val_reg << 48) | job->call_info.size, ++ ++ /* WAIT(0) => waits for FLUSH_CACHE2 instruction */ ++ (3ull << 56) | (1 << 16), ++ ++ /* CALL rX:rX+1, rX+2 */ ++ (32ull << 56) | (addr_reg << 40) | (val_reg << 32), ++ ++ /* MOV48 rX:rX+1, sync_addr */ ++ (1ull << 56) | (addr_reg << 48) | sync_addr, ++ ++ /* MOV48 rX+2, #1 */ ++ (1ull << 56) | (val_reg << 48) | 1, ++ ++ /* WAIT(all) */ ++ (3ull << 56) | (waitall_mask << 16), ++ ++ /* SYNC_ADD64.system_scope.propage_err.nowait rX:rX+1, rX+2*/ ++ (51ull << 56) | (0ull << 48) | (addr_reg << 40) | (val_reg << 32) | (0 << 16) | 1, ++ ++ /* ERROR_BARRIER, so we can recover from faults at job ++ * boundaries. ++ */ ++ (47ull << 56), ++ }; ++ ++ /* Need to be cacheline aligned to please the prefetcher. */ ++ static_assert(sizeof(call_instrs) % 64 == 0, ++ "call_instrs is not aligned on a cacheline"); ++ ++ /* Stream size is zero, nothing to do => return a NULL fence and let ++ * drm_sched signal the parent. ++ */ ++ if (!job->call_info.size) ++ return NULL; ++ ++ ret = pm_runtime_resume_and_get(ptdev->base.dev); ++ if (drm_WARN_ON(&ptdev->base, ret)) ++ return ERR_PTR(ret); ++ ++ mutex_lock(&sched->lock); ++ if (!group_can_run(group)) { ++ done_fence = ERR_PTR(-ECANCELED); ++ goto out_unlock; ++ } ++ ++ dma_fence_init(job->done_fence, ++ &panthor_queue_fence_ops, ++ &queue->fence_ctx.lock, ++ queue->fence_ctx.id, ++ atomic64_inc_return(&queue->fence_ctx.seqno)); ++ ++ memcpy(queue->ringbuf->kmap + ringbuf_insert, ++ call_instrs, sizeof(call_instrs)); ++ ++ panthor_job_get(&job->base); ++ spin_lock(&queue->fence_ctx.lock); ++ list_add_tail(&job->node, &queue->fence_ctx.in_flight_jobs); ++ spin_unlock(&queue->fence_ctx.lock); ++ ++ job->ringbuf.start = queue->iface.input->insert; ++ job->ringbuf.end = job->ringbuf.start + sizeof(call_instrs); ++ ++ /* Make sure the ring buffer is updated before the INSERT ++ * register. ++ */ ++ wmb(); ++ ++ queue->iface.input->extract = queue->iface.output->extract; ++ queue->iface.input->insert = job->ringbuf.end; ++ ++ if (group->csg_id < 0) { ++ /* If the queue is blocked, we want to keep the timeout running, so we ++ * can detect unbounded waits and kill the group when that happens. ++ * Otherwise, we suspend the timeout so the time we spend waiting for ++ * a CSG slot is not counted. ++ */ ++ if (!(group->blocked_queues & BIT(job->queue_idx)) && ++ !queue->timeout_suspended) { ++ queue->remaining_time = drm_sched_suspend_timeout(&queue->scheduler); ++ queue->timeout_suspended = true; ++ } ++ ++ group_schedule_locked(group, BIT(job->queue_idx)); ++ } else { ++ gpu_write(ptdev, CSF_DOORBELL(queue->doorbell_id), 1); ++ if (!sched->pm.has_ref && ++ !(group->blocked_queues & BIT(job->queue_idx))) { ++ pm_runtime_get(ptdev->base.dev); ++ sched->pm.has_ref = true; ++ } ++ } ++ ++ done_fence = dma_fence_get(job->done_fence); ++ ++out_unlock: ++ mutex_unlock(&sched->lock); ++ pm_runtime_mark_last_busy(ptdev->base.dev); ++ pm_runtime_put_autosuspend(ptdev->base.dev); ++ ++ return done_fence; ++} ++ ++static enum drm_gpu_sched_stat ++queue_timedout_job(struct drm_sched_job *sched_job) ++{ ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ struct panthor_group *group = job->group; ++ struct panthor_device *ptdev = group->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_queue *queue = group->queues[job->queue_idx]; ++ ++ drm_warn(&ptdev->base, "job timeout\n"); ++ ++ drm_WARN_ON(&ptdev->base, atomic_read(&sched->reset.in_progress)); ++ ++ queue_stop(queue, job); ++ ++ mutex_lock(&sched->lock); ++ group->timedout = true; ++ if (group->csg_id >= 0) { ++ sched_queue_delayed_work(ptdev->scheduler, tick, 0); ++ } else { ++ /* Remove from the run queues, so the scheduler can't ++ * pick the group on the next tick. ++ */ ++ list_del_init(&group->run_node); ++ list_del_init(&group->wait_node); ++ ++ group_queue_work(group, term); ++ } ++ mutex_unlock(&sched->lock); ++ ++ queue_start(queue); ++ ++ return DRM_GPU_SCHED_STAT_NOMINAL; ++} ++ ++static void queue_free_job(struct drm_sched_job *sched_job) ++{ ++ drm_sched_job_cleanup(sched_job); ++ panthor_job_put(sched_job); ++} ++ ++static const struct drm_sched_backend_ops panthor_queue_sched_ops = { ++ .run_job = queue_run_job, ++ .timedout_job = queue_timedout_job, ++ .free_job = queue_free_job, ++}; ++ ++static struct panthor_queue * ++group_create_queue(struct panthor_group *group, ++ const struct drm_panthor_queue_create *args) ++{ ++ struct drm_gpu_scheduler *drm_sched; ++ struct panthor_queue *queue; ++ int ret; ++ ++ if (args->pad[0] || args->pad[1] || args->pad[2]) ++ return ERR_PTR(-EINVAL); ++ ++ if (args->ringbuf_size < SZ_4K || args->ringbuf_size > SZ_64K || ++ !is_power_of_2(args->ringbuf_size)) ++ return ERR_PTR(-EINVAL); ++ ++ if (args->priority > CSF_MAX_QUEUE_PRIO) ++ return ERR_PTR(-EINVAL); ++ ++ queue = kzalloc(sizeof(*queue), GFP_KERNEL); ++ if (!queue) ++ return ERR_PTR(-ENOMEM); ++ ++ queue->fence_ctx.id = dma_fence_context_alloc(1); ++ spin_lock_init(&queue->fence_ctx.lock); ++ INIT_LIST_HEAD(&queue->fence_ctx.in_flight_jobs); ++ ++ queue->priority = args->priority; ++ ++ queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm, ++ args->ringbuf_size, ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++ if (IS_ERR(queue->ringbuf)) { ++ ret = PTR_ERR(queue->ringbuf); ++ goto err_free_queue; ++ } ++ ++ ret = panthor_kernel_bo_vmap(queue->ringbuf); ++ if (ret) ++ goto err_free_queue; ++ ++ queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev, ++ &queue->iface.input, ++ &queue->iface.output, ++ &queue->iface.input_fw_va, ++ &queue->iface.output_fw_va); ++ if (IS_ERR(queue->iface.mem)) { ++ ret = PTR_ERR(queue->iface.mem); ++ goto err_free_queue; ++ } ++ ++ ret = drm_sched_init(&queue->scheduler, &panthor_queue_sched_ops, ++ group->ptdev->scheduler->wq, 1, ++ args->ringbuf_size / (NUM_INSTRS_PER_SLOT * sizeof(u64)), ++ 0, msecs_to_jiffies(JOB_TIMEOUT_MS), ++ group->ptdev->reset.wq, ++ NULL, "panthor-queue", group->ptdev->base.dev); ++ if (ret) ++ goto err_free_queue; ++ ++ drm_sched = &queue->scheduler; ++ ret = drm_sched_entity_init(&queue->entity, 0, &drm_sched, 1, NULL); ++ ++ return queue; ++ ++err_free_queue: ++ group_free_queue(group, queue); ++ return ERR_PTR(ret); ++} ++ ++#define MAX_GROUPS_PER_POOL 128 ++ ++int panthor_group_create(struct panthor_file *pfile, ++ const struct drm_panthor_group_create *group_args, ++ const struct drm_panthor_queue_create *queue_args) ++{ ++ struct panthor_device *ptdev = pfile->ptdev; ++ struct panthor_group_pool *gpool = pfile->groups; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); ++ struct panthor_group *group = NULL; ++ u32 gid, i, suspend_size; ++ int ret; ++ ++ if (group_args->pad) ++ return -EINVAL; ++ ++ if (group_args->priority > PANTHOR_CSG_PRIORITY_HIGH) ++ return -EINVAL; ++ ++ if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) || ++ (group_args->fragment_core_mask & ~ptdev->gpu_info.shader_present) || ++ (group_args->tiler_core_mask & ~ptdev->gpu_info.tiler_present)) ++ return -EINVAL; ++ ++ if (hweight64(group_args->compute_core_mask) < group_args->max_compute_cores || ++ hweight64(group_args->fragment_core_mask) < group_args->max_fragment_cores || ++ hweight64(group_args->tiler_core_mask) < group_args->max_tiler_cores) ++ return -EINVAL; ++ ++ group = kzalloc(sizeof(*group), GFP_KERNEL); ++ if (!group) ++ return -ENOMEM; ++ ++ spin_lock_init(&group->fatal_lock); ++ kref_init(&group->refcount); ++ group->state = PANTHOR_CS_GROUP_CREATED; ++ group->csg_id = -1; ++ ++ group->ptdev = ptdev; ++ group->max_compute_cores = group_args->max_compute_cores; ++ group->compute_core_mask = group_args->compute_core_mask; ++ group->max_fragment_cores = group_args->max_fragment_cores; ++ group->fragment_core_mask = group_args->fragment_core_mask; ++ group->max_tiler_cores = group_args->max_tiler_cores; ++ group->tiler_core_mask = group_args->tiler_core_mask; ++ group->priority = group_args->priority; ++ ++ INIT_LIST_HEAD(&group->wait_node); ++ INIT_LIST_HEAD(&group->run_node); ++ INIT_WORK(&group->term_work, group_term_work); ++ INIT_WORK(&group->sync_upd_work, group_sync_upd_work); ++ INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work); ++ INIT_WORK(&group->release_work, group_release_work); ++ ++ group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id); ++ if (!group->vm) { ++ ret = -EINVAL; ++ goto err_put_group; ++ } ++ ++ suspend_size = csg_iface->control->suspend_size; ++ group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); ++ if (IS_ERR(group->suspend_buf)) { ++ ret = PTR_ERR(group->suspend_buf); ++ group->suspend_buf = NULL; ++ goto err_put_group; ++ } ++ ++ suspend_size = csg_iface->control->protm_suspend_size; ++ group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); ++ if (IS_ERR(group->protm_suspend_buf)) { ++ ret = PTR_ERR(group->protm_suspend_buf); ++ group->protm_suspend_buf = NULL; ++ goto err_put_group; ++ } ++ ++ group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm, ++ group_args->queues.count * ++ sizeof(struct panthor_syncobj_64b), ++ DRM_PANTHOR_BO_NO_MMAP, ++ DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | ++ DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED, ++ PANTHOR_VM_KERNEL_AUTO_VA); ++ if (IS_ERR(group->syncobjs)) { ++ ret = PTR_ERR(group->syncobjs); ++ goto err_put_group; ++ } ++ ++ ret = panthor_kernel_bo_vmap(group->syncobjs); ++ if (ret) ++ goto err_put_group; ++ ++ memset(group->syncobjs->kmap, 0, ++ group_args->queues.count * sizeof(struct panthor_syncobj_64b)); ++ ++ for (i = 0; i < group_args->queues.count; i++) { ++ group->queues[i] = group_create_queue(group, &queue_args[i]); ++ if (IS_ERR(group->queues[i])) { ++ ret = PTR_ERR(group->queues[i]); ++ group->queues[i] = NULL; ++ goto err_put_group; ++ } ++ ++ group->queue_count++; ++ } ++ ++ group->idle_queues = GENMASK(group->queue_count - 1, 0); ++ ++ ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL); ++ if (ret) ++ goto err_put_group; ++ ++ mutex_lock(&sched->reset.lock); ++ if (atomic_read(&sched->reset.in_progress)) { ++ panthor_group_stop(group); ++ } else { ++ mutex_lock(&sched->lock); ++ list_add_tail(&group->run_node, ++ &sched->groups.idle[group->priority]); ++ mutex_unlock(&sched->lock); ++ } ++ mutex_unlock(&sched->reset.lock); ++ ++ return gid; ++ ++err_put_group: ++ group_put(group); ++ return ret; ++} ++ ++int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) ++{ ++ struct panthor_group_pool *gpool = pfile->groups; ++ struct panthor_device *ptdev = pfile->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_group *group; ++ ++ group = xa_erase(&gpool->xa, group_handle); ++ if (!group) ++ return -EINVAL; ++ ++ for (u32 i = 0; i < group->queue_count; i++) { ++ if (group->queues[i]) ++ drm_sched_entity_destroy(&group->queues[i]->entity); ++ } ++ ++ mutex_lock(&sched->reset.lock); ++ mutex_lock(&sched->lock); ++ group->destroyed = true; ++ if (group->csg_id >= 0) { ++ sched_queue_delayed_work(sched, tick, 0); ++ } else if (!atomic_read(&sched->reset.in_progress)) { ++ /* Remove from the run queues, so the scheduler can't ++ * pick the group on the next tick. ++ */ ++ list_del_init(&group->run_node); ++ list_del_init(&group->wait_node); ++ group_queue_work(group, term); ++ } ++ mutex_unlock(&sched->lock); ++ mutex_unlock(&sched->reset.lock); ++ ++ group_put(group); ++ return 0; ++} ++ ++int panthor_group_get_state(struct panthor_file *pfile, ++ struct drm_panthor_group_get_state *get_state) ++{ ++ struct panthor_group_pool *gpool = pfile->groups; ++ struct panthor_device *ptdev = pfile->ptdev; ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ struct panthor_group *group; ++ ++ if (get_state->pad) ++ return -EINVAL; ++ ++ group = group_get(xa_load(&gpool->xa, get_state->group_handle)); ++ if (!group) ++ return -EINVAL; ++ ++ memset(get_state, 0, sizeof(*get_state)); ++ ++ mutex_lock(&sched->lock); ++ if (group->timedout) ++ get_state->state |= DRM_PANTHOR_GROUP_STATE_TIMEDOUT; ++ if (group->fatal_queues) { ++ get_state->state |= DRM_PANTHOR_GROUP_STATE_FATAL_FAULT; ++ get_state->fatal_queues = group->fatal_queues; ++ } ++ mutex_unlock(&sched->lock); ++ ++ group_put(group); ++ return 0; ++} ++ ++int panthor_group_pool_create(struct panthor_file *pfile) ++{ ++ struct panthor_group_pool *gpool; ++ ++ gpool = kzalloc(sizeof(*gpool), GFP_KERNEL); ++ if (!gpool) ++ return -ENOMEM; ++ ++ xa_init_flags(&gpool->xa, XA_FLAGS_ALLOC1); ++ pfile->groups = gpool; ++ return 0; ++} ++ ++void panthor_group_pool_destroy(struct panthor_file *pfile) ++{ ++ struct panthor_group_pool *gpool = pfile->groups; ++ struct panthor_group *group; ++ unsigned long i; ++ ++ if (IS_ERR_OR_NULL(gpool)) ++ return; ++ ++ xa_for_each(&gpool->xa, i, group) ++ panthor_group_destroy(pfile, i); ++ ++ xa_destroy(&gpool->xa); ++ kfree(gpool); ++ pfile->groups = NULL; ++} ++ ++static void job_release(struct kref *ref) ++{ ++ struct panthor_job *job = container_of(ref, struct panthor_job, refcount); ++ ++ drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node)); ++ ++ if (job->base.s_fence) ++ drm_sched_job_cleanup(&job->base); ++ ++ if (job->done_fence && job->done_fence->ops) ++ dma_fence_put(job->done_fence); ++ else ++ dma_fence_free(job->done_fence); ++ ++ group_put(job->group); ++ ++ kfree(job); ++} ++ ++struct drm_sched_job *panthor_job_get(struct drm_sched_job *sched_job) ++{ ++ if (sched_job) { ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ ++ kref_get(&job->refcount); ++ } ++ ++ return sched_job; ++} ++ ++void panthor_job_put(struct drm_sched_job *sched_job) ++{ ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ ++ if (sched_job) ++ kref_put(&job->refcount, job_release); ++} ++ ++struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job) ++{ ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ ++ return job->group->vm; ++} ++ ++struct drm_sched_job * ++panthor_job_create(struct panthor_file *pfile, ++ u16 group_handle, ++ const struct drm_panthor_queue_submit *qsubmit) ++{ ++ struct panthor_group_pool *gpool = pfile->groups; ++ struct panthor_job *job; ++ int ret; ++ ++ if (qsubmit->pad) ++ return ERR_PTR(-EINVAL); ++ ++ /* If stream_addr is zero, so stream_size should be. */ ++ if ((qsubmit->stream_size == 0) != (qsubmit->stream_addr == 0)) ++ return ERR_PTR(-EINVAL); ++ ++ /* Make sure the address is aligned on 64-byte (cacheline) and the size is ++ * aligned on 8-byte (instruction size). ++ */ ++ if ((qsubmit->stream_addr & 63) || (qsubmit->stream_size & 7)) ++ return ERR_PTR(-EINVAL); ++ ++ /* bits 24:30 must be zero. */ ++ if (qsubmit->latest_flush & GENMASK(30, 24)) ++ return ERR_PTR(-EINVAL); ++ ++ job = kzalloc(sizeof(*job), GFP_KERNEL); ++ if (!job) ++ return ERR_PTR(-ENOMEM); ++ ++ kref_init(&job->refcount); ++ job->queue_idx = qsubmit->queue_index; ++ job->call_info.size = qsubmit->stream_size; ++ job->call_info.start = qsubmit->stream_addr; ++ job->call_info.latest_flush = qsubmit->latest_flush; ++ INIT_LIST_HEAD(&job->node); ++ ++ job->group = group_get(xa_load(&gpool->xa, group_handle)); ++ if (!job->group) { ++ ret = -EINVAL; ++ goto err_put_job; ++ } ++ ++ if (job->queue_idx >= job->group->queue_count || ++ !job->group->queues[job->queue_idx]) { ++ ret = -EINVAL; ++ goto err_put_job; ++ } ++ ++ job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL); ++ if (!job->done_fence) { ++ ret = -ENOMEM; ++ goto err_put_job; ++ } ++ ++ ret = drm_sched_job_init(&job->base, ++ &job->group->queues[job->queue_idx]->entity, ++ 1, job->group); ++ if (ret) ++ goto err_put_job; ++ ++ return &job->base; ++ ++err_put_job: ++ panthor_job_put(&job->base); ++ return ERR_PTR(ret); ++} ++ ++void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched_job) ++{ ++ struct panthor_job *job = container_of(sched_job, struct panthor_job, base); ++ ++ /* Still not sure why we want USAGE_WRITE for external objects, since I ++ * was assuming this would be handled through explicit syncs being imported ++ * to external BOs with DMA_BUF_IOCTL_IMPORT_SYNC_FILE, but other drivers ++ * seem to pass DMA_RESV_USAGE_WRITE, so there must be a good reason. ++ */ ++ panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, ++ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE); ++} ++ ++void panthor_sched_unplug(struct panthor_device *ptdev) ++{ ++ struct panthor_scheduler *sched = ptdev->scheduler; ++ ++ cancel_delayed_work_sync(&sched->tick_work); ++ ++ mutex_lock(&sched->lock); ++ if (sched->pm.has_ref) { ++ pm_runtime_put(ptdev->base.dev); ++ sched->pm.has_ref = false; ++ } ++ mutex_unlock(&sched->lock); ++} ++ ++static void panthor_sched_fini(struct drm_device *ddev, void *res) ++{ ++ struct panthor_scheduler *sched = res; ++ int prio; ++ ++ if (!sched || !sched->csg_slot_count) ++ return; ++ ++ cancel_delayed_work_sync(&sched->tick_work); ++ ++ if (sched->wq) ++ destroy_workqueue(sched->wq); ++ ++ if (sched->heap_alloc_wq) ++ destroy_workqueue(sched->heap_alloc_wq); ++ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ drm_WARN_ON(ddev, !list_empty(&sched->groups.runnable[prio])); ++ drm_WARN_ON(ddev, !list_empty(&sched->groups.idle[prio])); ++ } ++ ++ drm_WARN_ON(ddev, !list_empty(&sched->groups.waiting)); ++} ++ ++int panthor_sched_init(struct panthor_device *ptdev) ++{ ++ struct panthor_fw_global_iface *glb_iface = panthor_fw_get_glb_iface(ptdev); ++ struct panthor_fw_csg_iface *csg_iface = panthor_fw_get_csg_iface(ptdev, 0); ++ struct panthor_fw_cs_iface *cs_iface = panthor_fw_get_cs_iface(ptdev, 0, 0); ++ struct panthor_scheduler *sched; ++ u32 gpu_as_count, num_groups; ++ int prio, ret; ++ ++ sched = drmm_kzalloc(&ptdev->base, sizeof(*sched), GFP_KERNEL); ++ if (!sched) ++ return -ENOMEM; ++ ++ /* The highest bit in JOB_INT_* is reserved for globabl IRQs. That ++ * leaves 31 bits for CSG IRQs, hence the MAX_CSGS clamp here. ++ */ ++ num_groups = min_t(u32, MAX_CSGS, glb_iface->control->group_num); ++ ++ /* The FW-side scheduler might deadlock if two groups with the same ++ * priority try to access a set of resources that overlaps, with part ++ * of the resources being allocated to one group and the other part to ++ * the other group, both groups waiting for the remaining resources to ++ * be allocated. To avoid that, it is recommended to assign each CSG a ++ * different priority. In theory we could allow several groups to have ++ * the same CSG priority if they don't request the same resources, but ++ * that makes the scheduling logic more complicated, so let's clamp ++ * the number of CSG slots to MAX_CSG_PRIO + 1 for now. ++ */ ++ num_groups = min_t(u32, MAX_CSG_PRIO + 1, num_groups); ++ ++ /* We need at least one AS for the MCU and one for the GPU contexts. */ ++ gpu_as_count = hweight32(ptdev->gpu_info.as_present & GENMASK(31, 1)); ++ if (!gpu_as_count) { ++ drm_err(&ptdev->base, "Not enough AS (%d, expected at least 2)", ++ gpu_as_count + 1); ++ return -EINVAL; ++ } ++ ++ sched->ptdev = ptdev; ++ sched->sb_slot_count = CS_FEATURES_SCOREBOARDS(cs_iface->control->features); ++ sched->csg_slot_count = num_groups; ++ sched->cs_slot_count = csg_iface->control->stream_num; ++ sched->as_slot_count = gpu_as_count; ++ ptdev->csif_info.csg_slot_count = sched->csg_slot_count; ++ ptdev->csif_info.cs_slot_count = sched->cs_slot_count; ++ ptdev->csif_info.scoreboard_slot_count = sched->sb_slot_count; ++ ++ sched->last_tick = 0; ++ sched->resched_target = U64_MAX; ++ sched->tick_period = msecs_to_jiffies(10); ++ INIT_DELAYED_WORK(&sched->tick_work, tick_work); ++ INIT_WORK(&sched->sync_upd_work, sync_upd_work); ++ INIT_WORK(&sched->fw_events_work, process_fw_events_work); ++ ++ ret = drmm_mutex_init(&ptdev->base, &sched->lock); ++ if (ret) ++ return ret; ++ ++ for (prio = PANTHOR_CSG_PRIORITY_COUNT - 1; prio >= 0; prio--) { ++ INIT_LIST_HEAD(&sched->groups.runnable[prio]); ++ INIT_LIST_HEAD(&sched->groups.idle[prio]); ++ } ++ INIT_LIST_HEAD(&sched->groups.waiting); ++ ++ ret = drmm_mutex_init(&ptdev->base, &sched->reset.lock); ++ if (ret) ++ return ret; ++ ++ INIT_LIST_HEAD(&sched->reset.stopped_groups); ++ ++ /* sched->heap_alloc_wq will be used for heap chunk allocation on ++ * tiler OOM events, which means we can't use the same workqueue for ++ * the scheduler because works queued by the scheduler are in ++ * the dma-signalling path. Allocate a dedicated heap_alloc_wq to ++ * work around this limitation. ++ * ++ * FIXME: Ultimately, what we need is a failable/non-blocking GEM ++ * allocation path that we can call when a heap OOM is reported. The ++ * FW is smart enough to fall back on other methods if the kernel can't ++ * allocate memory, and fail the tiling job if none of these ++ * countermeasures worked. ++ * ++ * Set WQ_MEM_RECLAIM on sched->wq to unblock the situation when the ++ * system is running out of memory. ++ */ ++ sched->heap_alloc_wq = alloc_workqueue("panthor-heap-alloc", WQ_UNBOUND, 0); ++ sched->wq = alloc_workqueue("panthor-csf-sched", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); ++ if (!sched->wq || !sched->heap_alloc_wq) { ++ panthor_sched_fini(&ptdev->base, sched); ++ drm_err(&ptdev->base, "Failed to allocate the workqueues"); ++ return -ENOMEM; ++ } ++ ++ ret = drmm_add_action_or_reset(&ptdev->base, panthor_sched_fini, sched); ++ if (ret) ++ return ret; ++ ++ ptdev->scheduler = sched; ++ return 0; ++} +diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h +new file mode 100644 +index 00000000000000..66438b1f331f69 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_sched.h +@@ -0,0 +1,50 @@ ++/* SPDX-License-Identifier: GPL-2.0 or MIT */ ++/* Copyright 2023 Collabora ltd. */ ++ ++#ifndef __PANTHOR_SCHED_H__ ++#define __PANTHOR_SCHED_H__ ++ ++struct drm_exec; ++struct dma_fence; ++struct drm_file; ++struct drm_gem_object; ++struct drm_sched_job; ++struct drm_panthor_group_create; ++struct drm_panthor_queue_create; ++struct drm_panthor_group_get_state; ++struct drm_panthor_queue_submit; ++struct panthor_device; ++struct panthor_file; ++struct panthor_group_pool; ++struct panthor_job; ++ ++int panthor_group_create(struct panthor_file *pfile, ++ const struct drm_panthor_group_create *group_args, ++ const struct drm_panthor_queue_create *queue_args); ++int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle); ++int panthor_group_get_state(struct panthor_file *pfile, ++ struct drm_panthor_group_get_state *get_state); ++ ++struct drm_sched_job * ++panthor_job_create(struct panthor_file *pfile, ++ u16 group_handle, ++ const struct drm_panthor_queue_submit *qsubmit); ++struct drm_sched_job *panthor_job_get(struct drm_sched_job *job); ++struct panthor_vm *panthor_job_vm(struct drm_sched_job *sched_job); ++void panthor_job_put(struct drm_sched_job *job); ++void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *job); ++ ++int panthor_group_pool_create(struct panthor_file *pfile); ++void panthor_group_pool_destroy(struct panthor_file *pfile); ++ ++int panthor_sched_init(struct panthor_device *ptdev); ++void panthor_sched_unplug(struct panthor_device *ptdev); ++void panthor_sched_pre_reset(struct panthor_device *ptdev); ++void panthor_sched_post_reset(struct panthor_device *ptdev); ++void panthor_sched_suspend(struct panthor_device *ptdev); ++void panthor_sched_resume(struct panthor_device *ptdev); ++ ++void panthor_sched_report_mmu_fault(struct panthor_device *ptdev); ++void panthor_sched_report_fw_events(struct panthor_device *ptdev, u32 events); ++ ++#endif + +From 1bee600b2d8420705317dcd88c5281a361778b1e Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:25 +0100 +Subject: [PATCH 090/150] drm/panthor: Add the driver frontend block + +This is the last piece missing to expose the driver to the outside +world. + +This is basically a wrapper between the ioctls and the other logical +blocks. + +v6: +- Add Maxime's and Heiko's acks +- Return a page-aligned BO size to userspace +- Keep header inclusion alphabetically ordered + +v5: +- Account for the drm_exec_init() prototype change +- Include platform_device.h + +v4: +- Add an ioctl to let the UMD query the VM state +- Fix kernel doc +- Let panthor_device_init() call panthor_device_init() +- Fix cleanup ordering in the panthor_init() error path +- Add Steve's and Liviu's R-b + +v3: +- Add acks for the MIT/GPL2 relicensing +- Fix 32-bit support +- Account for panthor_vm and panthor_sched changes +- Simplify the resv preparation/update logic +- Use a linked list rather than xarray for list of signals. +- Simplify panthor_get_uobj_array by returning the newly allocated + array. +- Drop the "DOC" for job submission helpers and move the relevant + comments to panthor_ioctl_group_submit(). +- Add helpers sync_op_is_signal()/sync_op_is_wait(). +- Simplify return type of panthor_submit_ctx_add_sync_signal() and + panthor_submit_ctx_get_sync_signal(). +- Drop WARN_ON from panthor_submit_ctx_add_job(). +- Fix typos in comments. + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-12-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 1473 +++++++++++++++++++++++++ + 1 file changed, 1473 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/panthor_drv.c + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +new file mode 100644 +index 00000000000000..ff484506229fb4 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -0,0 +1,1473 @@ ++// SPDX-License-Identifier: GPL-2.0 or MIT ++/* Copyright 2018 Marty E. Plummer */ ++/* Copyright 2019 Linaro, Ltd., Rob Herring */ ++/* Copyright 2019 Collabora ltd. */ ++ ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "panthor_device.h" ++#include "panthor_fw.h" ++#include "panthor_gem.h" ++#include "panthor_gpu.h" ++#include "panthor_heap.h" ++#include "panthor_mmu.h" ++#include "panthor_regs.h" ++#include "panthor_sched.h" ++ ++/** ++ * DOC: user <-> kernel object copy helpers. ++ */ ++ ++/** ++ * panthor_set_uobj() - Copy kernel object to user object. ++ * @usr_ptr: Users pointer. ++ * @usr_size: Size of the user object. ++ * @min_size: Minimum size for this object. ++ * @kern_size: Size of the kernel object. ++ * @in: Address of the kernel object to copy. ++ * ++ * Helper automating kernel -> user object copies. ++ * ++ * Don't use this function directly, use PANTHOR_UOBJ_SET() instead. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_set_uobj(u64 usr_ptr, u32 usr_size, u32 min_size, u32 kern_size, const void *in) ++{ ++ /* User size shouldn't be smaller than the minimal object size. */ ++ if (usr_size < min_size) ++ return -EINVAL; ++ ++ if (copy_to_user(u64_to_user_ptr(usr_ptr), in, min_t(u32, usr_size, kern_size))) ++ return -EFAULT; ++ ++ /* When the kernel object is smaller than the user object, we fill the gap with ++ * zeros. ++ */ ++ if (usr_size > kern_size && ++ clear_user(u64_to_user_ptr(usr_ptr + kern_size), usr_size - kern_size)) { ++ return -EFAULT; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_get_uobj_array() - Copy a user object array into a kernel accessible object array. ++ * @in: The object array to copy. ++ * @min_stride: Minimum array stride. ++ * @obj_size: Kernel object size. ++ * ++ * Helper automating user -> kernel object copies. ++ * ++ * Don't use this function directly, use PANTHOR_UOBJ_GET_ARRAY() instead. ++ * ++ * Return: newly allocated object array or an ERR_PTR on error. ++ */ ++static void * ++panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride, ++ u32 obj_size) ++{ ++ int ret = 0; ++ void *out_alloc; ++ ++ /* User stride must be at least the minimum object size, otherwise it might ++ * lack useful information. ++ */ ++ if (in->stride < min_stride) ++ return ERR_PTR(-EINVAL); ++ ++ if (!in->count) ++ return NULL; ++ ++ out_alloc = kvmalloc_array(in->count, obj_size, GFP_KERNEL); ++ if (!out_alloc) ++ return ERR_PTR(-ENOMEM); ++ ++ if (obj_size == in->stride) { ++ /* Fast path when user/kernel have the same uAPI header version. */ ++ if (copy_from_user(out_alloc, u64_to_user_ptr(in->array), ++ (unsigned long)obj_size * in->count)) ++ ret = -EFAULT; ++ } else { ++ void __user *in_ptr = u64_to_user_ptr(in->array); ++ void *out_ptr = out_alloc; ++ ++ /* If the sizes differ, we need to copy elements one by one. */ ++ for (u32 i = 0; i < in->count; i++) { ++ ret = copy_struct_from_user(out_ptr, obj_size, in_ptr, in->stride); ++ if (ret) ++ break; ++ ++ out_ptr += obj_size; ++ in_ptr += in->stride; ++ } ++ } ++ ++ if (ret) { ++ kvfree(out_alloc); ++ return ERR_PTR(ret); ++ } ++ ++ return out_alloc; ++} ++ ++/** ++ * PANTHOR_UOBJ_MIN_SIZE_INTERNAL() - Get the minimum user object size ++ * @_typename: Object type. ++ * @_last_mandatory_field: Last mandatory field. ++ * ++ * Get the minimum user object size based on the last mandatory field name, ++ * A.K.A, the name of the last field of the structure at the time this ++ * structure was added to the uAPI. ++ * ++ * Don't use directly, use PANTHOR_UOBJ_DECL() instead. ++ */ ++#define PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field) \ ++ (offsetof(_typename, _last_mandatory_field) + \ ++ sizeof(((_typename *)NULL)->_last_mandatory_field)) ++ ++/** ++ * PANTHOR_UOBJ_DECL() - Declare a new uAPI object whose subject to ++ * evolutions. ++ * @_typename: Object type. ++ * @_last_mandatory_field: Last mandatory field. ++ * ++ * Should be used to extend the PANTHOR_UOBJ_MIN_SIZE() list. ++ */ ++#define PANTHOR_UOBJ_DECL(_typename, _last_mandatory_field) \ ++ _typename : PANTHOR_UOBJ_MIN_SIZE_INTERNAL(_typename, _last_mandatory_field) ++ ++/** ++ * PANTHOR_UOBJ_MIN_SIZE() - Get the minimum size of a given uAPI object ++ * @_obj_name: Object to get the minimum size of. ++ * ++ * Don't use this macro directly, it's automatically called by ++ * PANTHOR_UOBJ_{SET,GET_ARRAY}(). ++ */ ++#define PANTHOR_UOBJ_MIN_SIZE(_obj_name) \ ++ _Generic(_obj_name, \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_vm_bind_op, syncs)) ++ ++/** ++ * PANTHOR_UOBJ_SET() - Copy a kernel object to a user object. ++ * @_dest_usr_ptr: User pointer to copy to. ++ * @_usr_size: Size of the user object. ++ * @_src_obj: Kernel object to copy (not a pointer). ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++#define PANTHOR_UOBJ_SET(_dest_usr_ptr, _usr_size, _src_obj) \ ++ panthor_set_uobj(_dest_usr_ptr, _usr_size, \ ++ PANTHOR_UOBJ_MIN_SIZE(_src_obj), \ ++ sizeof(_src_obj), &(_src_obj)) ++ ++/** ++ * PANTHOR_UOBJ_GET_ARRAY() - Copy a user object array to a kernel accessible ++ * object array. ++ * @_dest_array: Local variable that will hold the newly allocated kernel ++ * object array. ++ * @_uobj_array: The drm_panthor_obj_array object describing the user object ++ * array. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++#define PANTHOR_UOBJ_GET_ARRAY(_dest_array, _uobj_array) \ ++ ({ \ ++ typeof(_dest_array) _tmp; \ ++ _tmp = panthor_get_uobj_array(_uobj_array, \ ++ PANTHOR_UOBJ_MIN_SIZE((_dest_array)[0]), \ ++ sizeof((_dest_array)[0])); \ ++ if (!IS_ERR(_tmp)) \ ++ _dest_array = _tmp; \ ++ PTR_ERR_OR_ZERO(_tmp); \ ++ }) ++ ++/** ++ * struct panthor_sync_signal - Represent a synchronization object point to attach ++ * our job fence to. ++ * ++ * This structure is here to keep track of fences that are currently bound to ++ * a specific syncobj point. ++ * ++ * At the beginning of a job submission, the fence ++ * is retrieved from the syncobj itself, and can be NULL if no fence was attached ++ * to this point. ++ * ++ * At the end, it points to the fence of the last job that had a ++ * %DRM_PANTHOR_SYNC_OP_SIGNAL on this syncobj. ++ * ++ * With jobs being submitted in batches, the fence might change several times during ++ * the process, allowing one job to wait on a job that's part of the same submission ++ * but appears earlier in the drm_panthor_group_submit::queue_submits array. ++ */ ++struct panthor_sync_signal { ++ /** @node: list_head to track signal ops within a submit operation */ ++ struct list_head node; ++ ++ /** @handle: The syncobj handle. */ ++ u32 handle; ++ ++ /** ++ * @point: The syncobj point. ++ * ++ * Zero for regular syncobjs, and non-zero for timeline syncobjs. ++ */ ++ u64 point; ++ ++ /** ++ * @syncobj: The sync object pointed by @handle. ++ */ ++ struct drm_syncobj *syncobj; ++ ++ /** ++ * @chain: Chain object used to link the new fence to an existing ++ * timeline syncobj. ++ * ++ * NULL for regular syncobj, non-NULL for timeline syncobjs. ++ */ ++ struct dma_fence_chain *chain; ++ ++ /** ++ * @fence: The fence to assign to the syncobj or syncobj-point. ++ */ ++ struct dma_fence *fence; ++}; ++ ++/** ++ * struct panthor_job_ctx - Job context ++ */ ++struct panthor_job_ctx { ++ /** @job: The job that is about to be submitted to drm_sched. */ ++ struct drm_sched_job *job; ++ ++ /** @syncops: Array of sync operations. */ ++ struct drm_panthor_sync_op *syncops; ++ ++ /** @syncop_count: Number of sync operations. */ ++ u32 syncop_count; ++}; ++ ++/** ++ * struct panthor_submit_ctx - Submission context ++ * ++ * Anything that's related to a submission (%DRM_IOCTL_PANTHOR_VM_BIND or ++ * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT) is kept here, so we can automate the ++ * initialization and cleanup steps. ++ */ ++struct panthor_submit_ctx { ++ /** @file: DRM file this submission happens on. */ ++ struct drm_file *file; ++ ++ /** ++ * @signals: List of struct panthor_sync_signal. ++ * ++ * %DRM_PANTHOR_SYNC_OP_SIGNAL operations will be recorded here, ++ * and %DRM_PANTHOR_SYNC_OP_WAIT will first check if an entry ++ * matching the syncobj+point exists before calling ++ * drm_syncobj_find_fence(). This allows us to describe dependencies ++ * existing between jobs that are part of the same batch. ++ */ ++ struct list_head signals; ++ ++ /** @jobs: Array of jobs. */ ++ struct panthor_job_ctx *jobs; ++ ++ /** @job_count: Number of entries in the @jobs array. */ ++ u32 job_count; ++ ++ /** @exec: drm_exec context used to acquire and prepare resv objects. */ ++ struct drm_exec exec; ++}; ++ ++#define PANTHOR_SYNC_OP_FLAGS_MASK \ ++ (DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK | DRM_PANTHOR_SYNC_OP_SIGNAL) ++ ++static bool sync_op_is_signal(const struct drm_panthor_sync_op *sync_op) ++{ ++ return !!(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL); ++} ++ ++static bool sync_op_is_wait(const struct drm_panthor_sync_op *sync_op) ++{ ++ /* Note that DRM_PANTHOR_SYNC_OP_WAIT == 0 */ ++ return !(sync_op->flags & DRM_PANTHOR_SYNC_OP_SIGNAL); ++} ++ ++/** ++ * panthor_check_sync_op() - Check drm_panthor_sync_op fields ++ * @sync_op: The sync operation to check. ++ * ++ * Return: 0 on success, -EINVAL otherwise. ++ */ ++static int ++panthor_check_sync_op(const struct drm_panthor_sync_op *sync_op) ++{ ++ u8 handle_type; ++ ++ if (sync_op->flags & ~PANTHOR_SYNC_OP_FLAGS_MASK) ++ return -EINVAL; ++ ++ handle_type = sync_op->flags & DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_MASK; ++ if (handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ && ++ handle_type != DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_TIMELINE_SYNCOBJ) ++ return -EINVAL; ++ ++ if (handle_type == DRM_PANTHOR_SYNC_OP_HANDLE_TYPE_SYNCOBJ && ++ sync_op->timeline_value != 0) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++/** ++ * panthor_sync_signal_free() - Release resources and free a panthor_sync_signal object ++ * @sig_sync: Signal object to free. ++ */ ++static void ++panthor_sync_signal_free(struct panthor_sync_signal *sig_sync) ++{ ++ if (!sig_sync) ++ return; ++ ++ drm_syncobj_put(sig_sync->syncobj); ++ dma_fence_chain_free(sig_sync->chain); ++ dma_fence_put(sig_sync->fence); ++ kfree(sig_sync); ++} ++ ++/** ++ * panthor_submit_ctx_add_sync_signal() - Add a signal operation to a submit context ++ * @ctx: Context to add the signal operation to. ++ * @handle: Syncobj handle. ++ * @point: Syncobj point. ++ * ++ * Return: 0 on success, otherwise negative error value. ++ */ ++static int ++panthor_submit_ctx_add_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point) ++{ ++ struct panthor_sync_signal *sig_sync; ++ struct dma_fence *cur_fence; ++ int ret; ++ ++ sig_sync = kzalloc(sizeof(*sig_sync), GFP_KERNEL); ++ if (!sig_sync) ++ return -ENOMEM; ++ ++ sig_sync->handle = handle; ++ sig_sync->point = point; ++ ++ if (point > 0) { ++ sig_sync->chain = dma_fence_chain_alloc(); ++ if (!sig_sync->chain) { ++ ret = -ENOMEM; ++ goto err_free_sig_sync; ++ } ++ } ++ ++ sig_sync->syncobj = drm_syncobj_find(ctx->file, handle); ++ if (!sig_sync->syncobj) { ++ ret = -EINVAL; ++ goto err_free_sig_sync; ++ } ++ ++ /* Retrieve the current fence attached to that point. It's ++ * perfectly fine to get a NULL fence here, it just means there's ++ * no fence attached to that point yet. ++ */ ++ if (!drm_syncobj_find_fence(ctx->file, handle, point, 0, &cur_fence)) ++ sig_sync->fence = cur_fence; ++ ++ list_add_tail(&sig_sync->node, &ctx->signals); ++ ++ return 0; ++ ++err_free_sig_sync: ++ panthor_sync_signal_free(sig_sync); ++ return ret; ++} ++ ++/** ++ * panthor_submit_ctx_search_sync_signal() - Search an existing signal operation in a ++ * submit context. ++ * @ctx: Context to search the signal operation in. ++ * @handle: Syncobj handle. ++ * @point: Syncobj point. ++ * ++ * Return: A valid panthor_sync_signal object if found, NULL otherwise. ++ */ ++static struct panthor_sync_signal * ++panthor_submit_ctx_search_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point) ++{ ++ struct panthor_sync_signal *sig_sync; ++ ++ list_for_each_entry(sig_sync, &ctx->signals, node) { ++ if (handle == sig_sync->handle && point == sig_sync->point) ++ return sig_sync; ++ } ++ ++ return NULL; ++} ++ ++/** ++ * panthor_submit_ctx_add_job() - Add a job to a submit context ++ * @ctx: Context to search the signal operation in. ++ * @idx: Index of the job in the context. ++ * @job: Job to add. ++ * @syncs: Sync operations provided by userspace. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_add_job(struct panthor_submit_ctx *ctx, u32 idx, ++ struct drm_sched_job *job, ++ const struct drm_panthor_obj_array *syncs) ++{ ++ int ret; ++ ++ ctx->jobs[idx].job = job; ++ ++ ret = PANTHOR_UOBJ_GET_ARRAY(ctx->jobs[idx].syncops, syncs); ++ if (ret) ++ return ret; ++ ++ ctx->jobs[idx].syncop_count = syncs->count; ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_get_sync_signal() - Search signal operation and add one if none was found. ++ * @ctx: Context to search the signal operation in. ++ * @handle: Syncobj handle. ++ * @point: Syncobj point. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_get_sync_signal(struct panthor_submit_ctx *ctx, u32 handle, u64 point) ++{ ++ struct panthor_sync_signal *sig_sync; ++ ++ sig_sync = panthor_submit_ctx_search_sync_signal(ctx, handle, point); ++ if (sig_sync) ++ return 0; ++ ++ return panthor_submit_ctx_add_sync_signal(ctx, handle, point); ++} ++ ++/** ++ * panthor_submit_ctx_update_job_sync_signal_fences() - Update fences ++ * on the signal operations specified by a job. ++ * @ctx: Context to search the signal operation in. ++ * @job_idx: Index of the job to operate on. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_update_job_sync_signal_fences(struct panthor_submit_ctx *ctx, ++ u32 job_idx) ++{ ++ struct panthor_device *ptdev = container_of(ctx->file->minor->dev, ++ struct panthor_device, ++ base); ++ struct dma_fence *done_fence = &ctx->jobs[job_idx].job->s_fence->finished; ++ const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops; ++ u32 sync_op_count = ctx->jobs[job_idx].syncop_count; ++ ++ for (u32 i = 0; i < sync_op_count; i++) { ++ struct dma_fence *old_fence; ++ struct panthor_sync_signal *sig_sync; ++ ++ if (!sync_op_is_signal(&sync_ops[i])) ++ continue; ++ ++ sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle, ++ sync_ops[i].timeline_value); ++ if (drm_WARN_ON(&ptdev->base, !sig_sync)) ++ return -EINVAL; ++ ++ old_fence = sig_sync->fence; ++ sig_sync->fence = dma_fence_get(done_fence); ++ dma_fence_put(old_fence); ++ ++ if (drm_WARN_ON(&ptdev->base, !sig_sync->fence)) ++ return -EINVAL; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_collect_job_signal_ops() - Iterate over all job signal operations ++ * and add them to the context. ++ * @ctx: Context to search the signal operation in. ++ * @job_idx: Index of the job to operate on. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_collect_job_signal_ops(struct panthor_submit_ctx *ctx, ++ u32 job_idx) ++{ ++ const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops; ++ u32 sync_op_count = ctx->jobs[job_idx].syncop_count; ++ ++ for (u32 i = 0; i < sync_op_count; i++) { ++ int ret; ++ ++ if (!sync_op_is_signal(&sync_ops[i])) ++ continue; ++ ++ ret = panthor_check_sync_op(&sync_ops[i]); ++ if (ret) ++ return ret; ++ ++ ret = panthor_submit_ctx_get_sync_signal(ctx, ++ sync_ops[i].handle, ++ sync_ops[i].timeline_value); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_push_fences() - Iterate over the signal array, and for each entry, push ++ * the currently assigned fence to the associated syncobj. ++ * @ctx: Context to push fences on. ++ * ++ * This is the last step of a submission procedure, and is done once we know the submission ++ * is effective and job fences are guaranteed to be signaled in finite time. ++ */ ++static void ++panthor_submit_ctx_push_fences(struct panthor_submit_ctx *ctx) ++{ ++ struct panthor_sync_signal *sig_sync; ++ ++ list_for_each_entry(sig_sync, &ctx->signals, node) { ++ if (sig_sync->chain) { ++ drm_syncobj_add_point(sig_sync->syncobj, sig_sync->chain, ++ sig_sync->fence, sig_sync->point); ++ sig_sync->chain = NULL; ++ } else { ++ drm_syncobj_replace_fence(sig_sync->syncobj, sig_sync->fence); ++ } ++ } ++} ++ ++/** ++ * panthor_submit_ctx_add_sync_deps_to_job() - Add sync wait operations as ++ * job dependencies. ++ * @ctx: Submit context. ++ * @job_idx: Index of the job to operate on. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_add_sync_deps_to_job(struct panthor_submit_ctx *ctx, ++ u32 job_idx) ++{ ++ struct panthor_device *ptdev = container_of(ctx->file->minor->dev, ++ struct panthor_device, ++ base); ++ const struct drm_panthor_sync_op *sync_ops = ctx->jobs[job_idx].syncops; ++ struct drm_sched_job *job = ctx->jobs[job_idx].job; ++ u32 sync_op_count = ctx->jobs[job_idx].syncop_count; ++ int ret = 0; ++ ++ for (u32 i = 0; i < sync_op_count; i++) { ++ struct panthor_sync_signal *sig_sync; ++ struct dma_fence *fence; ++ ++ if (!sync_op_is_wait(&sync_ops[i])) ++ continue; ++ ++ ret = panthor_check_sync_op(&sync_ops[i]); ++ if (ret) ++ return ret; ++ ++ sig_sync = panthor_submit_ctx_search_sync_signal(ctx, sync_ops[i].handle, ++ sync_ops[i].timeline_value); ++ if (sig_sync) { ++ if (drm_WARN_ON(&ptdev->base, !sig_sync->fence)) ++ return -EINVAL; ++ ++ fence = dma_fence_get(sig_sync->fence); ++ } else { ++ ret = drm_syncobj_find_fence(ctx->file, sync_ops[i].handle, ++ sync_ops[i].timeline_value, ++ 0, &fence); ++ if (ret) ++ return ret; ++ } ++ ++ ret = drm_sched_job_add_dependency(job, fence); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_collect_jobs_signal_ops() - Collect all signal operations ++ * and add them to the submit context. ++ * @ctx: Submit context. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_collect_jobs_signal_ops(struct panthor_submit_ctx *ctx) ++{ ++ for (u32 i = 0; i < ctx->job_count; i++) { ++ int ret; ++ ++ ret = panthor_submit_ctx_collect_job_signal_ops(ctx, i); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_add_deps_and_arm_jobs() - Add jobs dependencies and arm jobs ++ * @ctx: Submit context. ++ * ++ * Must be called after the resv preparation has been taken care of. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int ++panthor_submit_ctx_add_deps_and_arm_jobs(struct panthor_submit_ctx *ctx) ++{ ++ for (u32 i = 0; i < ctx->job_count; i++) { ++ int ret; ++ ++ ret = panthor_submit_ctx_add_sync_deps_to_job(ctx, i); ++ if (ret) ++ return ret; ++ ++ drm_sched_job_arm(ctx->jobs[i].job); ++ ++ ret = panthor_submit_ctx_update_job_sync_signal_fences(ctx, i); ++ if (ret) ++ return ret; ++ } ++ ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_push_jobs() - Push jobs to their scheduling entities. ++ * @ctx: Submit context. ++ * @upd_resvs: Callback used to update reservation objects that were previously ++ * preapred. ++ */ ++static void ++panthor_submit_ctx_push_jobs(struct panthor_submit_ctx *ctx, ++ void (*upd_resvs)(struct drm_exec *, struct drm_sched_job *)) ++{ ++ for (u32 i = 0; i < ctx->job_count; i++) { ++ upd_resvs(&ctx->exec, ctx->jobs[i].job); ++ drm_sched_entity_push_job(ctx->jobs[i].job); ++ ++ /* Job is owned by the scheduler now. */ ++ ctx->jobs[i].job = NULL; ++ } ++ ++ panthor_submit_ctx_push_fences(ctx); ++} ++ ++/** ++ * panthor_submit_ctx_init() - Initializes a submission context ++ * @ctx: Submit context to initialize. ++ * @file: drm_file this submission happens on. ++ * @job_count: Number of jobs that will be submitted. ++ * ++ * Return: 0 on success, a negative error code otherwise. ++ */ ++static int panthor_submit_ctx_init(struct panthor_submit_ctx *ctx, ++ struct drm_file *file, u32 job_count) ++{ ++ ctx->jobs = kvmalloc_array(job_count, sizeof(*ctx->jobs), ++ GFP_KERNEL | __GFP_ZERO); ++ if (!ctx->jobs) ++ return -ENOMEM; ++ ++ ctx->file = file; ++ ctx->job_count = job_count; ++ INIT_LIST_HEAD(&ctx->signals); ++ drm_exec_init(&ctx->exec, ++ DRM_EXEC_INTERRUPTIBLE_WAIT | DRM_EXEC_IGNORE_DUPLICATES, ++ 0); ++ return 0; ++} ++ ++/** ++ * panthor_submit_ctx_cleanup() - Cleanup a submission context ++ * @ctx: Submit context to cleanup. ++ * @job_put: Job put callback. ++ */ ++static void panthor_submit_ctx_cleanup(struct panthor_submit_ctx *ctx, ++ void (*job_put)(struct drm_sched_job *)) ++{ ++ struct panthor_sync_signal *sig_sync, *tmp; ++ unsigned long i; ++ ++ drm_exec_fini(&ctx->exec); ++ ++ list_for_each_entry_safe(sig_sync, tmp, &ctx->signals, node) ++ panthor_sync_signal_free(sig_sync); ++ ++ for (i = 0; i < ctx->job_count; i++) { ++ job_put(ctx->jobs[i].job); ++ kvfree(ctx->jobs[i].syncops); ++ } ++ ++ kvfree(ctx->jobs); ++} ++ ++static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct drm_file *file) ++{ ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ struct drm_panthor_dev_query *args = data; ++ ++ if (!args->pointer) { ++ switch (args->type) { ++ case DRM_PANTHOR_DEV_QUERY_GPU_INFO: ++ args->size = sizeof(ptdev->gpu_info); ++ return 0; ++ ++ case DRM_PANTHOR_DEV_QUERY_CSIF_INFO: ++ args->size = sizeof(ptdev->csif_info); ++ return 0; ++ ++ default: ++ return -EINVAL; ++ } ++ } ++ ++ switch (args->type) { ++ case DRM_PANTHOR_DEV_QUERY_GPU_INFO: ++ return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->gpu_info); ++ ++ case DRM_PANTHOR_DEV_QUERY_CSIF_INFO: ++ return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->csif_info); ++ ++ default: ++ return -EINVAL; ++ } ++} ++ ++#define PANTHOR_VM_CREATE_FLAGS 0 ++ ++static int panthor_ioctl_vm_create(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_vm_create *args = data; ++ int cookie, ret; ++ ++ if (!drm_dev_enter(ddev, &cookie)) ++ return -ENODEV; ++ ++ ret = panthor_vm_pool_create_vm(ptdev, pfile->vms, args); ++ if (ret >= 0) { ++ args->id = ret; ++ ret = 0; ++ } ++ ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static int panthor_ioctl_vm_destroy(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_vm_destroy *args = data; ++ ++ if (args->pad) ++ return -EINVAL; ++ ++ return panthor_vm_pool_destroy_vm(pfile->vms, args->id); ++} ++ ++#define PANTHOR_BO_FLAGS DRM_PANTHOR_BO_NO_MMAP ++ ++static int panthor_ioctl_bo_create(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_bo_create *args = data; ++ struct panthor_vm *vm = NULL; ++ int cookie, ret; ++ ++ if (!drm_dev_enter(ddev, &cookie)) ++ return -ENODEV; ++ ++ if (!args->size || args->pad || ++ (args->flags & ~PANTHOR_BO_FLAGS)) { ++ ret = -EINVAL; ++ goto out_dev_exit; ++ } ++ ++ if (args->exclusive_vm_id) { ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->exclusive_vm_id); ++ if (!vm) { ++ ret = -EINVAL; ++ goto out_dev_exit; ++ } ++ } ++ ++ ret = panthor_gem_create_with_handle(file, ddev, vm, &args->size, ++ args->flags, &args->handle); ++ ++ panthor_vm_put(vm); ++ ++out_dev_exit: ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static int panthor_ioctl_bo_mmap_offset(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_panthor_bo_mmap_offset *args = data; ++ struct drm_gem_object *obj; ++ int ret; ++ ++ if (args->pad) ++ return -EINVAL; ++ ++ obj = drm_gem_object_lookup(file, args->handle); ++ if (!obj) ++ return -ENOENT; ++ ++ ret = drm_gem_create_mmap_offset(obj); ++ if (ret) ++ goto out; ++ ++ args->offset = drm_vma_node_offset_addr(&obj->vma_node); ++ ++out: ++ drm_gem_object_put(obj); ++ return ret; ++} ++ ++static int panthor_ioctl_group_submit(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_group_submit *args = data; ++ struct drm_panthor_queue_submit *jobs_args; ++ struct panthor_submit_ctx ctx; ++ int ret = 0, cookie; ++ ++ if (args->pad) ++ return -EINVAL; ++ ++ if (!drm_dev_enter(ddev, &cookie)) ++ return -ENODEV; ++ ++ ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->queue_submits); ++ if (ret) ++ goto out_dev_exit; ++ ++ ret = panthor_submit_ctx_init(&ctx, file, args->queue_submits.count); ++ if (ret) ++ goto out_free_jobs_args; ++ ++ /* Create jobs and attach sync operations */ ++ for (u32 i = 0; i < args->queue_submits.count; i++) { ++ const struct drm_panthor_queue_submit *qsubmit = &jobs_args[i]; ++ struct drm_sched_job *job; ++ ++ job = panthor_job_create(pfile, args->group_handle, qsubmit); ++ if (IS_ERR(job)) { ++ ret = PTR_ERR(job); ++ goto out_cleanup_submit_ctx; ++ } ++ ++ ret = panthor_submit_ctx_add_job(&ctx, i, job, &qsubmit->syncs); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ } ++ ++ /* ++ * Collect signal operations on all jobs, such that each job can pick ++ * from it for its dependencies and update the fence to signal when the ++ * job is submitted. ++ */ ++ ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ ++ /* ++ * We acquire/prepare revs on all jobs before proceeding with the ++ * dependency registration. ++ * ++ * This is solving two problems: ++ * 1. drm_sched_job_arm() and drm_sched_entity_push_job() must be ++ * protected by a lock to make sure no concurrent access to the same ++ * entity get interleaved, which would mess up with the fence seqno ++ * ordering. Luckily, one of the resv being acquired is the VM resv, ++ * and a scheduling entity is only bound to a single VM. As soon as ++ * we acquire the VM resv, we should be safe. ++ * 2. Jobs might depend on fences that were issued by previous jobs in ++ * the same batch, so we can't add dependencies on all jobs before ++ * arming previous jobs and registering the fence to the signal ++ * array, otherwise we might miss dependencies, or point to an ++ * outdated fence. ++ */ ++ if (args->queue_submits.count > 0) { ++ /* All jobs target the same group, so they also point to the same VM. */ ++ struct panthor_vm *vm = panthor_job_vm(ctx.jobs[0].job); ++ ++ drm_exec_until_all_locked(&ctx.exec) { ++ ret = panthor_vm_prepare_mapped_bos_resvs(&ctx.exec, vm, ++ args->queue_submits.count); ++ } ++ ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ } ++ ++ /* ++ * Now that resvs are locked/prepared, we can iterate over each job to ++ * add the dependencies, arm the job fence, register the job fence to ++ * the signal array. ++ */ ++ ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ ++ /* Nothing can fail after that point, so we can make our job fences ++ * visible to the outside world. Push jobs and set the job fences to ++ * the resv slots we reserved. This also pushes the fences to the ++ * syncobjs that are part of the signal array. ++ */ ++ panthor_submit_ctx_push_jobs(&ctx, panthor_job_update_resvs); ++ ++out_cleanup_submit_ctx: ++ panthor_submit_ctx_cleanup(&ctx, panthor_job_put); ++ ++out_free_jobs_args: ++ kvfree(jobs_args); ++ ++out_dev_exit: ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static int panthor_ioctl_group_destroy(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_group_destroy *args = data; ++ ++ if (args->pad) ++ return -EINVAL; ++ ++ return panthor_group_destroy(pfile, args->group_handle); ++} ++ ++static int panthor_ioctl_group_create(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_group_create *args = data; ++ struct drm_panthor_queue_create *queue_args; ++ int ret; ++ ++ if (!args->queues.count) ++ return -EINVAL; ++ ++ ret = PANTHOR_UOBJ_GET_ARRAY(queue_args, &args->queues); ++ if (ret) ++ return ret; ++ ++ ret = panthor_group_create(pfile, args, queue_args); ++ if (ret >= 0) { ++ args->group_handle = ret; ++ ret = 0; ++ } ++ ++ kvfree(queue_args); ++ return ret; ++} ++ ++static int panthor_ioctl_group_get_state(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_group_get_state *args = data; ++ ++ return panthor_group_get_state(pfile, args); ++} ++ ++static int panthor_ioctl_tiler_heap_create(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_tiler_heap_create *args = data; ++ struct panthor_heap_pool *pool; ++ struct panthor_vm *vm; ++ int ret; ++ ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id); ++ if (!vm) ++ return -EINVAL; ++ ++ pool = panthor_vm_get_heap_pool(vm, true); ++ if (IS_ERR(pool)) { ++ ret = PTR_ERR(pool); ++ goto out_put_vm; ++ } ++ ++ ret = panthor_heap_create(pool, ++ args->initial_chunk_count, ++ args->chunk_size, ++ args->max_chunks, ++ args->target_in_flight, ++ &args->tiler_heap_ctx_gpu_va, ++ &args->first_heap_chunk_gpu_va); ++ if (ret < 0) ++ goto out_put_heap_pool; ++ ++ /* Heap pools are per-VM. We combine the VM and HEAP id to make ++ * a unique heap handle. ++ */ ++ args->handle = (args->vm_id << 16) | ret; ++ ret = 0; ++ ++out_put_heap_pool: ++ panthor_heap_pool_put(pool); ++ ++out_put_vm: ++ panthor_vm_put(vm); ++ return ret; ++} ++ ++static int panthor_ioctl_tiler_heap_destroy(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_tiler_heap_destroy *args = data; ++ struct panthor_heap_pool *pool; ++ struct panthor_vm *vm; ++ int ret; ++ ++ if (args->pad) ++ return -EINVAL; ++ ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->handle >> 16); ++ if (!vm) ++ return -EINVAL; ++ ++ pool = panthor_vm_get_heap_pool(vm, false); ++ if (!pool) { ++ ret = -EINVAL; ++ goto out_put_vm; ++ } ++ ++ ret = panthor_heap_destroy(pool, args->handle & GENMASK(15, 0)); ++ panthor_heap_pool_put(pool); ++ ++out_put_vm: ++ panthor_vm_put(vm); ++ return ret; ++} ++ ++static int panthor_ioctl_vm_bind_async(struct drm_device *ddev, ++ struct drm_panthor_vm_bind *args, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_vm_bind_op *jobs_args; ++ struct panthor_submit_ctx ctx; ++ struct panthor_vm *vm; ++ int ret = 0; ++ ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id); ++ if (!vm) ++ return -EINVAL; ++ ++ ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops); ++ if (ret) ++ goto out_put_vm; ++ ++ ret = panthor_submit_ctx_init(&ctx, file, args->ops.count); ++ if (ret) ++ goto out_free_jobs_args; ++ ++ for (u32 i = 0; i < args->ops.count; i++) { ++ struct drm_panthor_vm_bind_op *op = &jobs_args[i]; ++ struct drm_sched_job *job; ++ ++ job = panthor_vm_bind_job_create(file, vm, op); ++ if (IS_ERR(job)) { ++ ret = PTR_ERR(job); ++ goto out_cleanup_submit_ctx; ++ } ++ ++ ret = panthor_submit_ctx_add_job(&ctx, i, job, &op->syncs); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ } ++ ++ ret = panthor_submit_ctx_collect_jobs_signal_ops(&ctx); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ ++ /* Prepare reservation objects for each VM_BIND job. */ ++ drm_exec_until_all_locked(&ctx.exec) { ++ for (u32 i = 0; i < ctx.job_count; i++) { ++ ret = panthor_vm_bind_job_prepare_resvs(&ctx.exec, ctx.jobs[i].job); ++ drm_exec_retry_on_contention(&ctx.exec); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ } ++ } ++ ++ ret = panthor_submit_ctx_add_deps_and_arm_jobs(&ctx); ++ if (ret) ++ goto out_cleanup_submit_ctx; ++ ++ /* Nothing can fail after that point. */ ++ panthor_submit_ctx_push_jobs(&ctx, panthor_vm_bind_job_update_resvs); ++ ++out_cleanup_submit_ctx: ++ panthor_submit_ctx_cleanup(&ctx, panthor_vm_bind_job_put); ++ ++out_free_jobs_args: ++ kvfree(jobs_args); ++ ++out_put_vm: ++ panthor_vm_put(vm); ++ return ret; ++} ++ ++static int panthor_ioctl_vm_bind_sync(struct drm_device *ddev, ++ struct drm_panthor_vm_bind *args, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_vm_bind_op *jobs_args; ++ struct panthor_vm *vm; ++ int ret; ++ ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id); ++ if (!vm) ++ return -EINVAL; ++ ++ ret = PANTHOR_UOBJ_GET_ARRAY(jobs_args, &args->ops); ++ if (ret) ++ goto out_put_vm; ++ ++ for (u32 i = 0; i < args->ops.count; i++) { ++ ret = panthor_vm_bind_exec_sync_op(file, vm, &jobs_args[i]); ++ if (ret) { ++ /* Update ops.count so the user knows where things failed. */ ++ args->ops.count = i; ++ break; ++ } ++ } ++ ++ kvfree(jobs_args); ++ ++out_put_vm: ++ panthor_vm_put(vm); ++ return ret; ++} ++ ++#define PANTHOR_VM_BIND_FLAGS DRM_PANTHOR_VM_BIND_ASYNC ++ ++static int panthor_ioctl_vm_bind(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct drm_panthor_vm_bind *args = data; ++ int cookie, ret; ++ ++ if (!drm_dev_enter(ddev, &cookie)) ++ return -ENODEV; ++ ++ if (args->flags & DRM_PANTHOR_VM_BIND_ASYNC) ++ ret = panthor_ioctl_vm_bind_async(ddev, args, file); ++ else ++ ret = panthor_ioctl_vm_bind_sync(ddev, args, file); ++ ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static int panthor_ioctl_vm_get_state(struct drm_device *ddev, void *data, ++ struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ struct drm_panthor_vm_get_state *args = data; ++ struct panthor_vm *vm; ++ ++ vm = panthor_vm_pool_get_vm(pfile->vms, args->vm_id); ++ if (!vm) ++ return -EINVAL; ++ ++ if (panthor_vm_is_unusable(vm)) ++ args->state = DRM_PANTHOR_VM_STATE_UNUSABLE; ++ else ++ args->state = DRM_PANTHOR_VM_STATE_USABLE; ++ ++ panthor_vm_put(vm); ++ return 0; ++} ++ ++static int ++panthor_open(struct drm_device *ddev, struct drm_file *file) ++{ ++ struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); ++ struct panthor_file *pfile; ++ int ret; ++ ++ if (!try_module_get(THIS_MODULE)) ++ return -EINVAL; ++ ++ pfile = kzalloc(sizeof(*pfile), GFP_KERNEL); ++ if (!pfile) { ++ ret = -ENOMEM; ++ goto err_put_mod; ++ } ++ ++ pfile->ptdev = ptdev; ++ ++ ret = panthor_vm_pool_create(pfile); ++ if (ret) ++ goto err_free_file; ++ ++ ret = panthor_group_pool_create(pfile); ++ if (ret) ++ goto err_destroy_vm_pool; ++ ++ file->driver_priv = pfile; ++ return 0; ++ ++err_destroy_vm_pool: ++ panthor_vm_pool_destroy(pfile); ++ ++err_free_file: ++ kfree(pfile); ++ ++err_put_mod: ++ module_put(THIS_MODULE); ++ return ret; ++} ++ ++static void ++panthor_postclose(struct drm_device *ddev, struct drm_file *file) ++{ ++ struct panthor_file *pfile = file->driver_priv; ++ ++ panthor_group_pool_destroy(pfile); ++ panthor_vm_pool_destroy(pfile); ++ ++ kfree(pfile); ++ module_put(THIS_MODULE); ++} ++ ++static const struct drm_ioctl_desc panthor_drm_driver_ioctls[] = { ++#define PANTHOR_IOCTL(n, func, flags) \ ++ DRM_IOCTL_DEF_DRV(PANTHOR_##n, panthor_ioctl_##func, flags) ++ ++ PANTHOR_IOCTL(DEV_QUERY, dev_query, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(VM_CREATE, vm_create, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(VM_DESTROY, vm_destroy, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(VM_BIND, vm_bind, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(VM_GET_STATE, vm_get_state, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(BO_CREATE, bo_create, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(BO_MMAP_OFFSET, bo_mmap_offset, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(GROUP_CREATE, group_create, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(GROUP_DESTROY, group_destroy, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(GROUP_GET_STATE, group_get_state, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(TILER_HEAP_CREATE, tiler_heap_create, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(TILER_HEAP_DESTROY, tiler_heap_destroy, DRM_RENDER_ALLOW), ++ PANTHOR_IOCTL(GROUP_SUBMIT, group_submit, DRM_RENDER_ALLOW), ++}; ++ ++static int panthor_mmap(struct file *filp, struct vm_area_struct *vma) ++{ ++ struct drm_file *file = filp->private_data; ++ struct panthor_file *pfile = file->driver_priv; ++ struct panthor_device *ptdev = pfile->ptdev; ++ u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT; ++ int ret, cookie; ++ ++ if (!drm_dev_enter(file->minor->dev, &cookie)) ++ return -ENODEV; ++ ++ if (panthor_device_mmio_offset(offset) >= DRM_PANTHOR_USER_MMIO_OFFSET) ++ ret = panthor_device_mmap_io(ptdev, vma); ++ else ++ ret = drm_gem_mmap(filp, vma); ++ ++ drm_dev_exit(cookie); ++ return ret; ++} ++ ++static const struct file_operations panthor_drm_driver_fops = { ++ .open = drm_open, ++ .release = drm_release, ++ .unlocked_ioctl = drm_ioctl, ++ .compat_ioctl = drm_compat_ioctl, ++ .poll = drm_poll, ++ .read = drm_read, ++ .llseek = noop_llseek, ++ .mmap = panthor_mmap, ++}; ++ ++#ifdef CONFIG_DEBUG_FS ++static void panthor_debugfs_init(struct drm_minor *minor) ++{ ++ panthor_mmu_debugfs_init(minor); ++} ++#endif ++ ++/* ++ * PanCSF driver version: ++ * - 1.0 - initial interface ++ */ ++static const struct drm_driver panthor_drm_driver = { ++ .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ | ++ DRIVER_SYNCOBJ_TIMELINE | DRIVER_GEM_GPUVA, ++ .open = panthor_open, ++ .postclose = panthor_postclose, ++ .ioctls = panthor_drm_driver_ioctls, ++ .num_ioctls = ARRAY_SIZE(panthor_drm_driver_ioctls), ++ .fops = &panthor_drm_driver_fops, ++ .name = "panthor", ++ .desc = "Panthor DRM driver", ++ .date = "20230801", ++ .major = 1, ++ .minor = 0, ++ ++ .gem_create_object = panthor_gem_create_object, ++ .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, ++#ifdef CONFIG_DEBUG_FS ++ .debugfs_init = panthor_debugfs_init, ++#endif ++}; ++ ++static int panthor_probe(struct platform_device *pdev) ++{ ++ struct panthor_device *ptdev; ++ ++ ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver, ++ struct panthor_device, base); ++ if (!ptdev) ++ return -ENOMEM; ++ ++ platform_set_drvdata(pdev, ptdev); ++ ++ return panthor_device_init(ptdev); ++} ++ ++static void panthor_remove(struct platform_device *pdev) ++{ ++ struct panthor_device *ptdev = platform_get_drvdata(pdev); ++ ++ panthor_device_unplug(ptdev); ++} ++ ++static const struct of_device_id dt_match[] = { ++ { .compatible = "rockchip,rk3588-mali" }, ++ { .compatible = "arm,mali-valhall-csf" }, ++ {} ++}; ++MODULE_DEVICE_TABLE(of, dt_match); ++ ++static DEFINE_RUNTIME_DEV_PM_OPS(panthor_pm_ops, ++ panthor_device_suspend, ++ panthor_device_resume, ++ NULL); ++ ++static struct platform_driver panthor_driver = { ++ .probe = panthor_probe, ++ .remove_new = panthor_remove, ++ .driver = { ++ .name = "panthor", ++ .pm = &panthor_pm_ops, ++ .of_match_table = dt_match, ++ }, ++}; ++ ++/* ++ * Workqueue used to cleanup stuff. ++ * ++ * We create a dedicated workqueue so we can drain on unplug and ++ * make sure all resources are freed before the module is unloaded. ++ */ ++struct workqueue_struct *panthor_cleanup_wq; ++ ++static int __init panthor_init(void) ++{ ++ int ret; ++ ++ ret = panthor_mmu_pt_cache_init(); ++ if (ret) ++ return ret; ++ ++ panthor_cleanup_wq = alloc_workqueue("panthor-cleanup", WQ_UNBOUND, 0); ++ if (!panthor_cleanup_wq) { ++ pr_err("panthor: Failed to allocate the workqueues"); ++ ret = -ENOMEM; ++ goto err_mmu_pt_cache_fini; ++ } ++ ++ ret = platform_driver_register(&panthor_driver); ++ if (ret) ++ goto err_destroy_cleanup_wq; ++ ++ return 0; ++ ++err_destroy_cleanup_wq: ++ destroy_workqueue(panthor_cleanup_wq); ++ ++err_mmu_pt_cache_fini: ++ panthor_mmu_pt_cache_fini(); ++ return ret; ++} ++module_init(panthor_init); ++ ++static void __exit panthor_exit(void) ++{ ++ platform_driver_unregister(&panthor_driver); ++ destroy_workqueue(panthor_cleanup_wq); ++ panthor_mmu_pt_cache_fini(); ++} ++module_exit(panthor_exit); ++ ++MODULE_AUTHOR("Panthor Project Developers"); ++MODULE_DESCRIPTION("Panthor DRM Driver"); ++MODULE_LICENSE("Dual MIT/GPL"); + +From 1a7f5f2dfdb8d80ac69e642ccd4fd0c6f262505c Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:26 +0100 +Subject: [PATCH 091/150] drm/panthor: Allow driver compilation + +Now that all blocks are available, we can add/update Kconfig/Makefile +files to allow compilation. + +v6: +- Add Maxime's and Heiko's acks +- Keep source files alphabetically ordered in the Makefile + +v4: +- Add Steve's R-b + +v3: +- Add a dep on DRM_GPUVM +- Fix dependencies in Kconfig +- Expand help text to (hopefully) describe which GPUs are to be + supported by this driver and which are for panfrost. + +Co-developed-by: Steven Price +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Acked-by: Steven Price # MIT+GPL2 relicensing,Arm +Acked-by: Grant Likely # MIT+GPL2 relicensing,Linaro +Acked-by: Boris Brezillon # MIT+GPL2 relicensing,Collabora +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-13-boris.brezillon@collabora.com +--- + drivers/gpu/drm/Kconfig | 2 ++ + drivers/gpu/drm/Makefile | 1 + + drivers/gpu/drm/panthor/Kconfig | 23 +++++++++++++++++++++++ + drivers/gpu/drm/panthor/Makefile | 14 ++++++++++++++ + 4 files changed, 40 insertions(+) + create mode 100644 drivers/gpu/drm/panthor/Kconfig + create mode 100644 drivers/gpu/drm/panthor/Makefile + +diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig +index ddbbe187f04195..ad3d8284419ad3 100644 +--- a/drivers/gpu/drm/Kconfig ++++ b/drivers/gpu/drm/Kconfig +@@ -428,6 +428,8 @@ source "drivers/gpu/drm/lima/Kconfig" + + source "drivers/gpu/drm/panfrost/Kconfig" + ++source "drivers/gpu/drm/panthor/Kconfig" ++ + source "drivers/gpu/drm/aspeed/Kconfig" + + source "drivers/gpu/drm/mcde/Kconfig" +diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile +index 1f167b96acfbe2..dbb6173fc10b32 100644 +--- a/drivers/gpu/drm/Makefile ++++ b/drivers/gpu/drm/Makefile +@@ -144,6 +144,7 @@ obj-$(CONFIG_DRM_XEN) += xen/ + obj-$(CONFIG_DRM_VBOXVIDEO) += vboxvideo/ + obj-$(CONFIG_DRM_LIMA) += lima/ + obj-$(CONFIG_DRM_PANFROST) += panfrost/ ++obj-$(CONFIG_DRM_PANTHOR) += panthor/ + obj-$(CONFIG_DRM_ASPEED_GFX) += aspeed/ + obj-$(CONFIG_DRM_MCDE) += mcde/ + obj-$(CONFIG_DRM_TIDSS) += tidss/ +diff --git a/drivers/gpu/drm/panthor/Kconfig b/drivers/gpu/drm/panthor/Kconfig +new file mode 100644 +index 00000000000000..55b40ad07f3b07 +--- /dev/null ++++ b/drivers/gpu/drm/panthor/Kconfig +@@ -0,0 +1,23 @@ ++# SPDX-License-Identifier: GPL-2.0 or MIT ++ ++config DRM_PANTHOR ++ tristate "Panthor (DRM support for ARM Mali CSF-based GPUs)" ++ depends on DRM ++ depends on ARM || ARM64 || COMPILE_TEST ++ depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE ++ depends on MMU ++ select DEVFREQ_GOV_SIMPLE_ONDEMAND ++ select DRM_EXEC ++ select DRM_GEM_SHMEM_HELPER ++ select DRM_GPUVM ++ select DRM_SCHED ++ select IOMMU_IO_PGTABLE_LPAE ++ select IOMMU_SUPPORT ++ select PM_DEVFREQ ++ help ++ DRM driver for ARM Mali CSF-based GPUs. ++ ++ This driver is for Mali (or Immortalis) Valhall Gxxx GPUs. ++ ++ Note that the Mali-G68 and Mali-G78, while Valhall architecture, will ++ be supported with the panfrost driver as they are not CSF GPUs. +diff --git a/drivers/gpu/drm/panthor/Makefile b/drivers/gpu/drm/panthor/Makefile +new file mode 100644 +index 00000000000000..15294719b09cbe +--- /dev/null ++++ b/drivers/gpu/drm/panthor/Makefile +@@ -0,0 +1,14 @@ ++# SPDX-License-Identifier: GPL-2.0 or MIT ++ ++panthor-y := \ ++ panthor_devfreq.o \ ++ panthor_device.o \ ++ panthor_drv.o \ ++ panthor_fw.o \ ++ panthor_gem.o \ ++ panthor_gpu.o \ ++ panthor_heap.o \ ++ panthor_mmu.o \ ++ panthor_sched.o ++ ++obj-$(CONFIG_DRM_PANTHOR) += panthor.o + +From 50151fb2c76b8e059e82b2e01922ce3b7f723da5 Mon Sep 17 00:00:00 2001 +From: Liviu Dudau +Date: Thu, 29 Feb 2024 17:22:27 +0100 +Subject: [PATCH 092/150] dt-bindings: gpu: mali-valhall-csf: Add support for + Arm Mali CSF GPUs + +Arm has introduced a new v10 GPU architecture that replaces the Job +Manager +interface with a new Command Stream Frontend. It adds firmware driven +command stream queues that can be used by kernel and user space to +submit +jobs to the GPU. + +Add the initial schema for the device tree that is based on support for +RK3588 SoC. The minimum number of clocks is one for the IP, but on +Rockchip +platforms they will tend to expose the semi-independent clocks for +better +power management. + +v6: +- Add Maxime's and Heiko's acks + +v5: +- Move the opp-table node under the gpu node + +v4: +- Fix formatting issue + +v3: +- Cleanup commit message to remove redundant text +- Added opp-table property and re-ordered entries +- Clarified power-domains and power-domain-names requirements for +RK3588. +- Cleaned up example + +Note: power-domains and power-domain-names requirements for other +platforms +are still work in progress, hence the bindings are left incomplete here. + +v2: +- New commit + +Signed-off-by: Liviu Dudau +Cc: Krzysztof Kozlowski +Cc: Rob Herring +Cc: Conor Dooley +Cc: devicetree@vger.kernel.org +Signed-off-by: Boris Brezillon +Reviewed-by: Rob Herring +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-14-boris.brezillon@collabora.com +--- + Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml b/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml +index a5b4e00217587c..f9e69e86b033ae 100644 +--- a/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml ++++ b/Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml +@@ -144,4 +144,4 @@ examples: + }; + }; + +-... ++... +\ No newline at end of file + +From 906b1a5a4518945fcb843463625dbd7dc36ff77a Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 29 Feb 2024 17:22:28 +0100 +Subject: [PATCH 093/150] drm/panthor: Add an entry to MAINTAINERS + +Add an entry for the Panthor driver to the MAINTAINERS file. + +v6: +- Add Maxime's and Heiko's acks + +v4: +- Add Steve's R-b + +v3: +- Add bindings document as an 'F:' line. +- Add Steven and Liviu as co-maintainers. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Maxime Ripard +Acked-by: Heiko Stuebner +Link: https://patchwork.freedesktop.org/patch/msgid/20240229162230.2634044-15-boris.brezillon@collabora.com +--- + MAINTAINERS | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/MAINTAINERS b/MAINTAINERS +index 13d1078808bb5f..38213a4527b165 100644 +--- a/MAINTAINERS ++++ b/MAINTAINERS +@@ -1622,6 +1622,17 @@ T: git git://anongit.freedesktop.org/drm/drm-misc + F: drivers/gpu/drm/panfrost/ + F: include/uapi/drm/panfrost_drm.h + ++ARM MALI PANTHOR DRM DRIVER ++M: Boris Brezillon ++M: Steven Price ++M: Liviu Dudau ++L: dri-devel@lists.freedesktop.org ++S: Supported ++T: git git://anongit.freedesktop.org/drm/drm-misc ++F: Documentation/devicetree/bindings/gpu/arm,mali-valhall-csf.yaml ++F: drivers/gpu/drm/panthor/ ++F: include/uapi/drm/panthor_drm.h ++ + ARM MALI-DP DRM DRIVER + M: Liviu Dudau + M: Brian Starkey + +From 7604c5dd533b58b8a6126e1f17a66834bfb6d092 Mon Sep 17 00:00:00 2001 +From: boogie +Date: Thu, 14 Mar 2024 00:27:14 +0100 +Subject: [PATCH 094/150] drm/panthor: remove debugfs + +--- + drivers/gpu/drm/panthor/panthor_drv.c | 11 ------ + drivers/gpu/drm/panthor/panthor_mmu.c | 51 --------------------------- + drivers/gpu/drm/panthor/panthor_mmu.h | 4 --- + 3 files changed, 66 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index ff484506229fb4..32fbcba3933b18 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -10,7 +10,6 @@ + #include + #include + +-#include + #include + #include + #include +@@ -1347,13 +1346,6 @@ static const struct file_operations panthor_drm_driver_fops = { + .mmap = panthor_mmap, + }; + +-#ifdef CONFIG_DEBUG_FS +-static void panthor_debugfs_init(struct drm_minor *minor) +-{ +- panthor_mmu_debugfs_init(minor); +-} +-#endif +- + /* + * PanCSF driver version: + * - 1.0 - initial interface +@@ -1374,9 +1366,6 @@ static const struct drm_driver panthor_drm_driver = { + + .gem_create_object = panthor_gem_create_object, + .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, +-#ifdef CONFIG_DEBUG_FS +- .debugfs_init = panthor_debugfs_init, +-#endif + }; + + static int panthor_probe(struct platform_device *pdev) +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index fdd35249169fdb..1ca4e6adc99aa5 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -2,7 +2,6 @@ + /* Copyright 2019 Linaro, Ltd, Rob Herring */ + /* Copyright 2023 Collabora ltd. */ + +-#include + #include + #include + #include +@@ -2695,56 +2694,6 @@ int panthor_mmu_init(struct panthor_device *ptdev) + return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq); + } + +-#ifdef CONFIG_DEBUG_FS +-static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m) +-{ +- int ret; +- +- mutex_lock(&vm->op_lock); +- ret = drm_debugfs_gpuva_info(m, &vm->base); +- mutex_unlock(&vm->op_lock); +- +- return ret; +-} +- +-static int show_each_vm(struct seq_file *m, void *arg) +-{ +- struct drm_info_node *node = (struct drm_info_node *)m->private; +- struct drm_device *ddev = node->minor->dev; +- struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); +- int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data; +- struct panthor_vm *vm; +- int ret = 0; +- +- mutex_lock(&ptdev->mmu->vm.lock); +- list_for_each_entry(vm, &ptdev->mmu->vm.list, node) { +- ret = show(vm, m); +- if (ret < 0) +- break; +- +- seq_puts(m, "\n"); +- } +- mutex_unlock(&ptdev->mmu->vm.lock); +- +- return ret; +-} +- +-static struct drm_info_list panthor_mmu_debugfs_list[] = { +- DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas), +-}; +- +-/** +- * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries +- * @minor: Minor. +- */ +-void panthor_mmu_debugfs_init(struct drm_minor *minor) +-{ +- drm_debugfs_create_files(panthor_mmu_debugfs_list, +- ARRAY_SIZE(panthor_mmu_debugfs_list), +- minor->debugfs_root, minor); +-} +-#endif /* CONFIG_DEBUG_FS */ +- + /** + * panthor_mmu_pt_cache_init() - Initialize the page table cache. + * +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h +index f3c1ed19f973f6..e6a2198b009b76 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.h ++++ b/drivers/gpu/drm/panthor/panthor_mmu.h +@@ -95,8 +95,4 @@ void panthor_vm_update_resvs(struct panthor_vm *vm, struct drm_exec *exec, + int panthor_mmu_pt_cache_init(void); + void panthor_mmu_pt_cache_fini(void); + +-#ifdef CONFIG_DEBUG_FS +-void panthor_mmu_debugfs_init(struct drm_minor *minor); +-#endif +- + #endif + +From 77e4953abc4f8370bc62b7d7c7cac313be95353b Mon Sep 17 00:00:00 2001 +From: Rob Clark +Date: Fri, 3 Sep 2021 11:47:52 -0700 +Subject: [PATCH 095/150] dma-buf/dma-fence: Add deadline awareness +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Add a way to hint to the fence signaler of an upcoming deadline, such as +vblank, which the fence waiter would prefer not to miss. This is to aid +the fence signaler in making power management decisions, like boosting +frequency as the deadline approaches and awareness of missing deadlines +so that can be factored in to the frequency scaling. + +v2: Drop dma_fence::deadline and related logic to filter duplicate + deadlines, to avoid increasing dma_fence size. The fence-context + implementation will need similar logic to track deadlines of all + the fences on the same timeline. [ckoenig] +v3: Clarify locking wrt. set_deadline callback +v4: Clarify in docs comment that this is a hint +v5: Drop DMA_FENCE_FLAG_HAS_DEADLINE_BIT. +v6: More docs +v7: Fix typo, clarify past deadlines + +Signed-off-by: Rob Clark +Reviewed-by: Christian König +Acked-by: Pekka Paalanen +Reviewed-by: Bagas Sanjaya +--- + Documentation/driver-api/dma-buf.rst | 6 +++ + drivers/dma-buf/dma-fence.c | 59 ++++++++++++++++++++++++++++ + include/linux/dma-fence.h | 22 +++++++++++ + 3 files changed, 87 insertions(+) + +diff --git a/Documentation/driver-api/dma-buf.rst b/Documentation/driver-api/dma-buf.rst +index 36a76cbe90954f..c010ad76f001a0 100644 +--- a/Documentation/driver-api/dma-buf.rst ++++ b/Documentation/driver-api/dma-buf.rst +@@ -158,6 +158,12 @@ DMA Fence Signalling Annotations + .. kernel-doc:: drivers/dma-buf/dma-fence.c + :doc: fence signalling annotation + ++DMA Fence Deadline Hints ++~~~~~~~~~~~~~~~~~~~~~~~~ ++ ++.. kernel-doc:: drivers/dma-buf/dma-fence.c ++ :doc: deadline hints ++ + DMA Fences Functions Reference + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c +index eef4786aaf862f..8aa8f8cb7071ef 100644 +--- a/drivers/dma-buf/dma-fence.c ++++ b/drivers/dma-buf/dma-fence.c +@@ -913,6 +913,65 @@ dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t count, + } + EXPORT_SYMBOL(dma_fence_wait_any_timeout); + ++/** ++ * DOC: deadline hints ++ * ++ * In an ideal world, it would be possible to pipeline a workload sufficiently ++ * that a utilization based device frequency governor could arrive at a minimum ++ * frequency that meets the requirements of the use-case, in order to minimize ++ * power consumption. But in the real world there are many workloads which ++ * defy this ideal. For example, but not limited to: ++ * ++ * * Workloads that ping-pong between device and CPU, with alternating periods ++ * of CPU waiting for device, and device waiting on CPU. This can result in ++ * devfreq and cpufreq seeing idle time in their respective domains and in ++ * result reduce frequency. ++ * ++ * * Workloads that interact with a periodic time based deadline, such as double ++ * buffered GPU rendering vs vblank sync'd page flipping. In this scenario, ++ * missing a vblank deadline results in an *increase* in idle time on the GPU ++ * (since it has to wait an additional vblank period), sending a signal to ++ * the GPU's devfreq to reduce frequency, when in fact the opposite is what is ++ * needed. ++ * ++ * To this end, deadline hint(s) can be set on a &dma_fence via &dma_fence_set_deadline. ++ * The deadline hint provides a way for the waiting driver, or userspace, to ++ * convey an appropriate sense of urgency to the signaling driver. ++ * ++ * A deadline hint is given in absolute ktime (CLOCK_MONOTONIC for userspace ++ * facing APIs). The time could either be some point in the future (such as ++ * the vblank based deadline for page-flipping, or the start of a compositor's ++ * composition cycle), or the current time to indicate an immediate deadline ++ * hint (Ie. forward progress cannot be made until this fence is signaled). ++ * ++ * Multiple deadlines may be set on a given fence, even in parallel. See the ++ * documentation for &dma_fence_ops.set_deadline. ++ * ++ * The deadline hint is just that, a hint. The driver that created the fence ++ * may react by increasing frequency, making different scheduling choices, etc. ++ * Or doing nothing at all. ++ */ ++ ++/** ++ * dma_fence_set_deadline - set desired fence-wait deadline hint ++ * @fence: the fence that is to be waited on ++ * @deadline: the time by which the waiter hopes for the fence to be ++ * signaled ++ * ++ * Give the fence signaler a hint about an upcoming deadline, such as ++ * vblank, by which point the waiter would prefer the fence to be ++ * signaled by. This is intended to give feedback to the fence signaler ++ * to aid in power management decisions, such as boosting GPU frequency ++ * if a periodic vblank deadline is approaching but the fence is not ++ * yet signaled.. ++ */ ++void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline) ++{ ++ if (fence->ops->set_deadline && !dma_fence_is_signaled(fence)) ++ fence->ops->set_deadline(fence, deadline); ++} ++EXPORT_SYMBOL(dma_fence_set_deadline); ++ + /** + * dma_fence_describe - Dump fence describtion into seq_file + * @fence: the 6fence to describe +diff --git a/include/linux/dma-fence.h b/include/linux/dma-fence.h +index b79097b9070b38..b3772edca2e6e0 100644 +--- a/include/linux/dma-fence.h ++++ b/include/linux/dma-fence.h +@@ -257,6 +257,26 @@ struct dma_fence_ops { + */ + void (*timeline_value_str)(struct dma_fence *fence, + char *str, int size); ++ ++ /** ++ * @set_deadline: ++ * ++ * Callback to allow a fence waiter to inform the fence signaler of ++ * an upcoming deadline, such as vblank, by which point the waiter ++ * would prefer the fence to be signaled by. This is intended to ++ * give feedback to the fence signaler to aid in power management ++ * decisions, such as boosting GPU frequency. ++ * ++ * This is called without &dma_fence.lock held, it can be called ++ * multiple times and from any context. Locking is up to the callee ++ * if it has some state to manage. If multiple deadlines are set, ++ * the expectation is to track the soonest one. If the deadline is ++ * before the current time, it should be interpreted as an immediate ++ * deadline. ++ * ++ * This callback is optional. ++ */ ++ void (*set_deadline)(struct dma_fence *fence, ktime_t deadline); + }; + + void dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops, +@@ -617,6 +637,8 @@ static inline signed long dma_fence_wait(struct dma_fence *fence, bool intr) + return ret < 0 ? ret : 0; + } + ++void dma_fence_set_deadline(struct dma_fence *fence, ktime_t deadline); ++ + struct dma_fence *dma_fence_get_stub(void); + struct dma_fence *dma_fence_allocate_private_stub(ktime_t timestamp); + u64 dma_fence_context_alloc(unsigned num); + +From efc60a1fa7d292d708c87f7a53df83c4cc158b9e Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Mon, 17 Oct 2022 20:22:11 +0300 +Subject: [PATCH 096/150] drm/gem: Take reservation lock for vmap/vunmap + operations +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The new common dma-buf locking convention will require buffer importers +to hold the reservation lock around mapping operations. Make DRM GEM core +to take the lock around the vmapping operations and update DRM drivers to +use the locked functions for the case where DRM core now holds the lock. +This patch prepares DRM core and drivers to the common dynamic dma-buf +locking convention. + +Acked-by: Christian König +Signed-off-by: Dmitry Osipenko +Link: https://patchwork.freedesktop.org/patch/msgid/20221017172229.42269-4-dmitry.osipenko@collabora.com +--- + drivers/gpu/drm/drm_client.c | 4 ++-- + drivers/gpu/drm/drm_gem.c | 24 ++++++++++++++++++++ + drivers/gpu/drm/drm_gem_dma_helper.c | 6 ++--- + drivers/gpu/drm/drm_gem_framebuffer_helper.c | 6 ++--- + drivers/gpu/drm/drm_gem_ttm_helper.c | 9 +------- + drivers/gpu/drm/lima/lima_sched.c | 4 ++-- + drivers/gpu/drm/panfrost/panfrost_dump.c | 4 ++-- + drivers/gpu/drm/panfrost/panfrost_perfcnt.c | 6 ++--- + drivers/gpu/drm/qxl/qxl_object.c | 17 +++++++------- + drivers/gpu/drm/qxl/qxl_prime.c | 4 ++-- + include/drm/drm_gem.h | 3 +++ + 11 files changed, 54 insertions(+), 33 deletions(-) + +diff --git a/drivers/gpu/drm/drm_client.c b/drivers/gpu/drm/drm_client.c +index dcbeeb68ca6411..11de319ac6d90e 100644 +--- a/drivers/gpu/drm/drm_client.c ++++ b/drivers/gpu/drm/drm_client.c +@@ -344,7 +344,7 @@ drm_client_buffer_vmap(struct drm_client_buffer *buffer, + * fd_install step out of the driver backend hooks, to make that + * final step optional for internal users. + */ +- ret = drm_gem_vmap(buffer->gem, map); ++ ret = drm_gem_vmap_unlocked(buffer->gem, map); + if (ret) + return ret; + +@@ -366,7 +366,7 @@ void drm_client_buffer_vunmap(struct drm_client_buffer *buffer) + { + struct iosys_map *map = &buffer->map; + +- drm_gem_vunmap(buffer->gem, map); ++ drm_gem_vunmap_unlocked(buffer->gem, map); + } + EXPORT_SYMBOL(drm_client_buffer_vunmap); + +diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c +index 4e58e500c2a5fd..482a7964b851c7 100644 +--- a/drivers/gpu/drm/drm_gem.c ++++ b/drivers/gpu/drm/drm_gem.c +@@ -1174,6 +1174,8 @@ int drm_gem_vmap(struct drm_gem_object *obj, struct iosys_map *map) + { + int ret; + ++ dma_resv_assert_held(obj->resv); ++ + if (!obj->funcs->vmap) + return -EOPNOTSUPP; + +@@ -1189,6 +1191,8 @@ EXPORT_SYMBOL(drm_gem_vmap); + + void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map) + { ++ dma_resv_assert_held(obj->resv); ++ + if (iosys_map_is_null(map)) + return; + +@@ -1200,6 +1204,26 @@ void drm_gem_vunmap(struct drm_gem_object *obj, struct iosys_map *map) + } + EXPORT_SYMBOL(drm_gem_vunmap); + ++int drm_gem_vmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map) ++{ ++ int ret; ++ ++ dma_resv_lock(obj->resv, NULL); ++ ret = drm_gem_vmap(obj, map); ++ dma_resv_unlock(obj->resv); ++ ++ return ret; ++} ++EXPORT_SYMBOL(drm_gem_vmap_unlocked); ++ ++void drm_gem_vunmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map) ++{ ++ dma_resv_lock(obj->resv, NULL); ++ drm_gem_vunmap(obj, map); ++ dma_resv_unlock(obj->resv); ++} ++EXPORT_SYMBOL(drm_gem_vunmap_unlocked); ++ + /** + * drm_gem_lock_reservations - Sets up the ww context and acquires + * the lock on an array of GEM objects. +diff --git a/drivers/gpu/drm/drm_gem_dma_helper.c b/drivers/gpu/drm/drm_gem_dma_helper.c +index 2daad6c6bfb280..df89fbd2d35cd6 100644 +--- a/drivers/gpu/drm/drm_gem_dma_helper.c ++++ b/drivers/gpu/drm/drm_gem_dma_helper.c +@@ -230,7 +230,7 @@ void drm_gem_dma_free(struct drm_gem_dma_object *dma_obj) + + if (gem_obj->import_attach) { + if (dma_obj->vaddr) +- dma_buf_vunmap(gem_obj->import_attach->dmabuf, &map); ++ dma_buf_vunmap_unlocked(gem_obj->import_attach->dmabuf, &map); + drm_prime_gem_destroy(gem_obj, dma_obj->sgt); + } else if (dma_obj->vaddr) { + if (dma_obj->map_noncoherent) +@@ -581,7 +581,7 @@ drm_gem_dma_prime_import_sg_table_vmap(struct drm_device *dev, + struct iosys_map map; + int ret; + +- ret = dma_buf_vmap(attach->dmabuf, &map); ++ ret = dma_buf_vmap_unlocked(attach->dmabuf, &map); + if (ret) { + DRM_ERROR("Failed to vmap PRIME buffer\n"); + return ERR_PTR(ret); +@@ -589,7 +589,7 @@ drm_gem_dma_prime_import_sg_table_vmap(struct drm_device *dev, + + obj = drm_gem_dma_prime_import_sg_table(dev, attach, sgt); + if (IS_ERR(obj)) { +- dma_buf_vunmap(attach->dmabuf, &map); ++ dma_buf_vunmap_unlocked(attach->dmabuf, &map); + return obj; + } + +diff --git a/drivers/gpu/drm/drm_gem_framebuffer_helper.c b/drivers/gpu/drm/drm_gem_framebuffer_helper.c +index 880a4975507fc1..e35e224e6303a2 100644 +--- a/drivers/gpu/drm/drm_gem_framebuffer_helper.c ++++ b/drivers/gpu/drm/drm_gem_framebuffer_helper.c +@@ -354,7 +354,7 @@ int drm_gem_fb_vmap(struct drm_framebuffer *fb, struct iosys_map *map, + ret = -EINVAL; + goto err_drm_gem_vunmap; + } +- ret = drm_gem_vmap(obj, &map[i]); ++ ret = drm_gem_vmap_unlocked(obj, &map[i]); + if (ret) + goto err_drm_gem_vunmap; + } +@@ -376,7 +376,7 @@ int drm_gem_fb_vmap(struct drm_framebuffer *fb, struct iosys_map *map, + obj = drm_gem_fb_get_obj(fb, i); + if (!obj) + continue; +- drm_gem_vunmap(obj, &map[i]); ++ drm_gem_vunmap_unlocked(obj, &map[i]); + } + return ret; + } +@@ -403,7 +403,7 @@ void drm_gem_fb_vunmap(struct drm_framebuffer *fb, struct iosys_map *map) + continue; + if (iosys_map_is_null(&map[i])) + continue; +- drm_gem_vunmap(obj, &map[i]); ++ drm_gem_vunmap_unlocked(obj, &map[i]); + } + } + EXPORT_SYMBOL(drm_gem_fb_vunmap); +diff --git a/drivers/gpu/drm/drm_gem_ttm_helper.c b/drivers/gpu/drm/drm_gem_ttm_helper.c +index e5fc875990c4f3..d5962a34c01d5f 100644 +--- a/drivers/gpu/drm/drm_gem_ttm_helper.c ++++ b/drivers/gpu/drm/drm_gem_ttm_helper.c +@@ -64,13 +64,8 @@ int drm_gem_ttm_vmap(struct drm_gem_object *gem, + struct iosys_map *map) + { + struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(gem); +- int ret; +- +- dma_resv_lock(gem->resv, NULL); +- ret = ttm_bo_vmap(bo, map); +- dma_resv_unlock(gem->resv); + +- return ret; ++ return ttm_bo_vmap(bo, map); + } + EXPORT_SYMBOL(drm_gem_ttm_vmap); + +@@ -87,9 +82,7 @@ void drm_gem_ttm_vunmap(struct drm_gem_object *gem, + { + struct ttm_buffer_object *bo = drm_gem_ttm_of_gem(gem); + +- dma_resv_lock(gem->resv, NULL); + ttm_bo_vunmap(bo, map); +- dma_resv_unlock(gem->resv); + } + EXPORT_SYMBOL(drm_gem_ttm_vunmap); + +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index 3c3c765de51169..b59269db0ffa65 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -371,7 +371,7 @@ static void lima_sched_build_error_task_list(struct lima_sched_task *task) + } else { + buffer_chunk->size = lima_bo_size(bo); + +- ret = drm_gem_shmem_vmap(&bo->base, &map); ++ ret = drm_gem_vmap_unlocked(&bo->base.base, &map); + if (ret) { + kvfree(et); + goto out; +@@ -379,7 +379,7 @@ static void lima_sched_build_error_task_list(struct lima_sched_task *task) + + memcpy(buffer_chunk + 1, map.vaddr, buffer_chunk->size); + +- drm_gem_shmem_vunmap(&bo->base, &map); ++ drm_gem_vunmap_unlocked(&bo->base.base, &map); + } + + buffer_chunk = (void *)(buffer_chunk + 1) + buffer_chunk->size; +diff --git a/drivers/gpu/drm/panfrost/panfrost_dump.c b/drivers/gpu/drm/panfrost/panfrost_dump.c +index 6bd0634e2d5805..e7942ac449c68c 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_dump.c ++++ b/drivers/gpu/drm/panfrost/panfrost_dump.c +@@ -209,7 +209,7 @@ void panfrost_core_dump(struct panfrost_job *job) + goto dump_header; + } + +- ret = drm_gem_shmem_vmap(&bo->base, &map); ++ ret = drm_gem_vmap_unlocked(&bo->base.base, &map); + if (ret) { + dev_err(pfdev->dev, "Panfrost Dump: couldn't map Buffer Object\n"); + iter.hdr->bomap.valid = 0; +@@ -236,7 +236,7 @@ void panfrost_core_dump(struct panfrost_job *job) + vaddr = map.vaddr; + memcpy(iter.data, vaddr, bo->base.base.size); + +- drm_gem_shmem_vunmap(&bo->base, &map); ++ drm_gem_vunmap_unlocked(&bo->base.base, &map); + + iter.hdr->bomap.valid = 1; + +diff --git a/drivers/gpu/drm/panfrost/panfrost_perfcnt.c b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c +index bc0df93f7f2156..ba9b6e2b263637 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_perfcnt.c ++++ b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c +@@ -106,7 +106,7 @@ static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, + goto err_close_bo; + } + +- ret = drm_gem_shmem_vmap(bo, &map); ++ ret = drm_gem_vmap_unlocked(&bo->base, &map); + if (ret) + goto err_put_mapping; + perfcnt->buf = map.vaddr; +@@ -165,7 +165,7 @@ static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, + return 0; + + err_vunmap: +- drm_gem_shmem_vunmap(bo, &map); ++ drm_gem_vunmap_unlocked(&bo->base, &map); + err_put_mapping: + panfrost_gem_mapping_put(perfcnt->mapping); + err_close_bo: +@@ -195,7 +195,7 @@ static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, + GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_OFF)); + + perfcnt->user = NULL; +- drm_gem_shmem_vunmap(&perfcnt->mapping->obj->base, &map); ++ drm_gem_vunmap_unlocked(&perfcnt->mapping->obj->base.base, &map); + perfcnt->buf = NULL; + panfrost_gem_close(&perfcnt->mapping->obj->base.base, file_priv); + panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu); +diff --git a/drivers/gpu/drm/qxl/qxl_object.c b/drivers/gpu/drm/qxl/qxl_object.c +index 695d9308d1f08b..06a58dad5f5cf4 100644 +--- a/drivers/gpu/drm/qxl/qxl_object.c ++++ b/drivers/gpu/drm/qxl/qxl_object.c +@@ -168,9 +168,16 @@ int qxl_bo_vmap_locked(struct qxl_bo *bo, struct iosys_map *map) + bo->map_count++; + goto out; + } +- r = ttm_bo_vmap(&bo->tbo, &bo->map); ++ ++ r = __qxl_bo_pin(bo); + if (r) + return r; ++ ++ r = ttm_bo_vmap(&bo->tbo, &bo->map); ++ if (r) { ++ __qxl_bo_unpin(bo); ++ return r; ++ } + bo->map_count = 1; + + /* TODO: Remove kptr in favor of map everywhere. */ +@@ -192,12 +199,6 @@ int qxl_bo_vmap(struct qxl_bo *bo, struct iosys_map *map) + if (r) + return r; + +- r = __qxl_bo_pin(bo); +- if (r) { +- qxl_bo_unreserve(bo); +- return r; +- } +- + r = qxl_bo_vmap_locked(bo, map); + qxl_bo_unreserve(bo); + return r; +@@ -247,6 +248,7 @@ void qxl_bo_vunmap_locked(struct qxl_bo *bo) + return; + bo->kptr = NULL; + ttm_bo_vunmap(&bo->tbo, &bo->map); ++ __qxl_bo_unpin(bo); + } + + int qxl_bo_vunmap(struct qxl_bo *bo) +@@ -258,7 +260,6 @@ int qxl_bo_vunmap(struct qxl_bo *bo) + return r; + + qxl_bo_vunmap_locked(bo); +- __qxl_bo_unpin(bo); + qxl_bo_unreserve(bo); + return 0; + } +diff --git a/drivers/gpu/drm/qxl/qxl_prime.c b/drivers/gpu/drm/qxl/qxl_prime.c +index 142d01415acb37..9169c26357d366 100644 +--- a/drivers/gpu/drm/qxl/qxl_prime.c ++++ b/drivers/gpu/drm/qxl/qxl_prime.c +@@ -59,7 +59,7 @@ int qxl_gem_prime_vmap(struct drm_gem_object *obj, struct iosys_map *map) + struct qxl_bo *bo = gem_to_qxl_bo(obj); + int ret; + +- ret = qxl_bo_vmap(bo, map); ++ ret = qxl_bo_vmap_locked(bo, map); + if (ret < 0) + return ret; + +@@ -71,5 +71,5 @@ void qxl_gem_prime_vunmap(struct drm_gem_object *obj, + { + struct qxl_bo *bo = gem_to_qxl_bo(obj); + +- qxl_bo_vunmap(bo); ++ qxl_bo_vunmap_locked(bo); + } +diff --git a/include/drm/drm_gem.h b/include/drm/drm_gem.h +index 58b69fa1cf5dbf..83afab00e11f4d 100644 +--- a/include/drm/drm_gem.h ++++ b/include/drm/drm_gem.h +@@ -476,6 +476,9 @@ struct page **drm_gem_get_pages(struct drm_gem_object *obj); + void drm_gem_put_pages(struct drm_gem_object *obj, struct page **pages, + bool dirty, bool accessed); + ++int drm_gem_vmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map); ++void drm_gem_vunmap_unlocked(struct drm_gem_object *obj, struct iosys_map *map); ++ + int drm_gem_objects_lookup(struct drm_file *filp, void __user *bo_handles, + int count, struct drm_gem_object ***objs_out); + struct drm_gem_object *drm_gem_object_lookup(struct drm_file *filp, u32 handle); + +From 31152686b858736f378eb95c79e90358e2604e41 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Mon, 17 Oct 2022 20:22:09 +0300 +Subject: [PATCH 097/150] dma-buf: Add unlocked variant of vmapping functions +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Add unlocked variant of dma_buf_vmap/vunmap() that will be utilized +by drivers that don't take the reservation lock explicitly. + +Acked-by: Sumit Semwal +Acked-by: Christian König +Signed-off-by: Dmitry Osipenko +Link: https://patchwork.freedesktop.org/patch/msgid/20221017172229.42269-2-dmitry.osipenko@collabora.com +--- + drivers/dma-buf/dma-buf.c | 43 +++++++++++++++++++++++++++++++++++++++ + include/linux/dma-buf.h | 2 ++ + 2 files changed, 45 insertions(+) + +diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c +index f804618f9bb730..f4c9827afc75ec 100644 +--- a/drivers/dma-buf/dma-buf.c ++++ b/drivers/dma-buf/dma-buf.c +@@ -1656,6 +1656,33 @@ int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map) + } + EXPORT_SYMBOL_NS_GPL(dma_buf_vmap, DMA_BUF); + ++/** ++ * dma_buf_vmap_unlocked - Create virtual mapping for the buffer object into kernel ++ * address space. Same restrictions as for vmap and friends apply. ++ * @dmabuf: [in] buffer to vmap ++ * @map: [out] returns the vmap pointer ++ * ++ * Unlocked version of dma_buf_vmap() ++ * ++ * Returns 0 on success, or a negative errno code otherwise. ++ */ ++int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map) ++{ ++ int ret; ++ ++ iosys_map_clear(map); ++ ++ if (WARN_ON(!dmabuf)) ++ return -EINVAL; ++ ++ dma_resv_lock(dmabuf->resv, NULL); ++ ret = dma_buf_vmap(dmabuf, map); ++ dma_resv_unlock(dmabuf->resv); ++ ++ return ret; ++} ++EXPORT_SYMBOL_NS_GPL(dma_buf_vmap_unlocked, DMA_BUF); ++ + /** + * dma_buf_vunmap - Unmap a vmap obtained by dma_buf_vmap. + * @dmabuf: [in] buffer to vunmap +@@ -1680,6 +1707,22 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) + } + EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap, DMA_BUF); + ++/** ++ * dma_buf_vunmap_unlocked - Unmap a vmap obtained by dma_buf_vmap. ++ * @dmabuf: [in] buffer to vunmap ++ * @map: [in] vmap pointer to vunmap ++ */ ++void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map) ++{ ++ if (WARN_ON(!dmabuf)) ++ return; ++ ++ dma_resv_lock(dmabuf->resv, NULL); ++ dma_buf_vunmap(dmabuf, map); ++ dma_resv_unlock(dmabuf->resv); ++} ++EXPORT_SYMBOL_NS_GPL(dma_buf_vunmap_unlocked, DMA_BUF); ++ + #ifdef CONFIG_DEBUG_FS + static int dma_buf_debug_show(struct seq_file *s, void *unused) + { +diff --git a/include/linux/dma-buf.h b/include/linux/dma-buf.h +index 54f9f7db635ad7..9db6e9808995c8 100644 +--- a/include/linux/dma-buf.h ++++ b/include/linux/dma-buf.h +@@ -686,4 +686,6 @@ static inline size_t dma_buf_get_peak_size(void) { return 0; } + static inline size_t dma_buf_get_total_size(void) { return 0; } + #endif + ++int dma_buf_vmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map); ++void dma_buf_vunmap_unlocked(struct dma_buf *dmabuf, struct iosys_map *map); + #endif /* __DMA_BUF_H__ */ + +From 22758ab1abf763a73c3f87a52f04654b5d1c9e26 Mon Sep 17 00:00:00 2001 +From: Dmitry Osipenko +Date: Mon, 17 Oct 2022 20:22:23 +0300 +Subject: [PATCH 098/150] dma-buf: Move dma_buf_vmap() to dynamic locking + specification +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Move dma_buf_vmap/vunmap() functions to the dynamic locking +specification by asserting that the reservation lock is held. + +Acked-by: Sumit Semwal +Acked-by: Christian König +Signed-off-by: Dmitry Osipenko +Link: https://patchwork.freedesktop.org/patch/msgid/20221017172229.42269-16-dmitry.osipenko@collabora.com +--- + drivers/dma-buf/dma-buf.c | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c +index f4c9827afc75ec..1c0deb1e08242a 100644 +--- a/drivers/dma-buf/dma-buf.c ++++ b/drivers/dma-buf/dma-buf.c +@@ -1628,6 +1628,8 @@ int dma_buf_vmap(struct dma_buf *dmabuf, struct iosys_map *map) + if (WARN_ON(!dmabuf)) + return -EINVAL; + ++ dma_resv_assert_held(dmabuf->resv); ++ + if (!dmabuf->ops->vmap) + return -EINVAL; + +@@ -1693,6 +1695,8 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) + if (WARN_ON(!dmabuf)) + return; + ++ dma_resv_assert_held(dmabuf->resv); ++ + BUG_ON(iosys_map_is_null(&dmabuf->vmap_ptr)); + BUG_ON(dmabuf->vmapping_counter == 0); + BUG_ON(!iosys_map_is_equal(&dmabuf->vmap_ptr, map)); + +From 045dde9dc9f3f95e0f3d487ab2ccf51c752dc15e Mon Sep 17 00:00:00 2001 +From: Donald Robson +Date: Wed, 22 Nov 2023 16:34:23 +0000 +Subject: [PATCH 099/150] drm/gpuvm: Helper to get range of unmap from a remap + op. + +Determining the start and range of the unmap stage of a remap op is a +common piece of code currently implemented by multiple drivers. Add a +helper for this. + +Changes since v7: +- Renamed helper to drm_gpuva_op_remap_to_unmap_range() +- Improved documentation + +Changes since v6: +- Remove use of __always_inline + +Signed-off-by: Donald Robson +Signed-off-by: Sarah Walker +Reviewed-by: Danilo Krummrich +Link: https://lore.kernel.org/r/8a0a5b5eeec459d3c60fcdaa5a638ad14a18a59e.1700668843.git.donald.robson@imgtec.com +Signed-off-by: Maxime Ripard +--- + include/drm/drm_gpuvm.h | 28 ++++++++++++++++++++++++++++ + 1 file changed, 28 insertions(+) + +diff --git a/include/drm/drm_gpuvm.h b/include/drm/drm_gpuvm.h +index 7921eb322b83fe..e37871dcc0894a 100644 +--- a/include/drm/drm_gpuvm.h ++++ b/include/drm/drm_gpuvm.h +@@ -1196,4 +1196,32 @@ void drm_gpuva_remap(struct drm_gpuva *prev, + + void drm_gpuva_unmap(struct drm_gpuva_op_unmap *op); + ++/** ++ * drm_gpuva_op_remap_to_unmap_range() - Helper to get the start and range of ++ * the unmap stage of a remap op. ++ * @op: Remap op. ++ * @start_addr: Output pointer for the start of the required unmap. ++ * @range: Output pointer for the length of the required unmap. ++ * ++ * The given start address and range will be set such that they represent the ++ * range of the address space that was previously covered by the mapping being ++ * re-mapped, but is now empty. ++ */ ++static inline void ++drm_gpuva_op_remap_to_unmap_range(const struct drm_gpuva_op_remap *op, ++ u64 *start_addr, u64 *range) ++{ ++ const u64 va_start = op->prev ? ++ op->prev->va.addr + op->prev->va.range : ++ op->unmap->va->va.addr; ++ const u64 va_end = op->next ? ++ op->next->va.addr : ++ op->unmap->va->va.addr + op->unmap->va->va.range; ++ ++ if (start_addr) ++ *start_addr = va_start; ++ if (range) ++ *range = va_end - va_start; ++} ++ + #endif /* __DRM_GPUVM_H__ */ + +From 464c15697c5ae4c70dc13bc3995429f231401dd5 Mon Sep 17 00:00:00 2001 +From: Thomas Zimmermann +Date: Tue, 20 Jun 2023 09:59:57 +0200 +Subject: [PATCH 100/150] drm: Enable PRIME import/export for all drivers + +Call drm_gem_prime_handle_to_fd() and drm_gem_prime_fd_to_handle() by +default if no PRIME import/export helpers have been set. Both functions +are the default for almost all drivers. + +DRM drivers implement struct drm_driver.gem_prime_import_sg_table +to import dma-buf objects from other drivers. Having the function +drm_gem_prime_fd_to_handle() functions set by default allows each +driver to import dma-buf objects to itself, even without support for +other drivers. + +For drm_gem_prime_handle_to_fd() it is similar: using it by default +allows each driver to export to itself, even without support for other +drivers. + +This functionality enables userspace to share per-driver buffers +across process boundaries via PRIME (e.g., wlroots requires this +functionality). The patch generalizes a pattern that has previously +been implemented by GEM VRAM helpers [1] to work with any driver. +For example, gma500 can now run the wlroots-based sway compositor. + +v2: + * clean up docs and TODO comments (Simon, Zack) + * clean up style in drm_getcap() + +Signed-off-by: Thomas Zimmermann +Link: https://lore.kernel.org/dri-devel/20230302143502.500661-1-contact@emersion.fr/ # 1 +Reviewed-by: Simon Ser +Acked-by: Alex Deucher +Reviewed-by: Jeffrey Hugo +Link: https://patchwork.freedesktop.org/patch/msgid/20230620080252.16368-2-tzimmermann@suse.de +--- + drivers/gpu/drm/drm_ioctl.c | 3 +-- + drivers/gpu/drm/drm_prime.c | 21 ++++++++++++--------- + include/drm/drm_drv.h | 12 ++---------- + 3 files changed, 15 insertions(+), 21 deletions(-) + +diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c +index 4b839f17f315ab..4f6895096c7048 100644 +--- a/drivers/gpu/drm/drm_ioctl.c ++++ b/drivers/gpu/drm/drm_ioctl.c +@@ -245,8 +245,7 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_ + req->value = 1; + return 0; + case DRM_CAP_PRIME: +- req->value |= dev->driver->prime_fd_to_handle ? DRM_PRIME_CAP_IMPORT : 0; +- req->value |= dev->driver->prime_handle_to_fd ? DRM_PRIME_CAP_EXPORT : 0; ++ req->value = DRM_PRIME_CAP_IMPORT | DRM_PRIME_CAP_EXPORT; + return 0; + case DRM_CAP_SYNCOBJ: + req->value = drm_core_check_feature(dev, DRIVER_SYNCOBJ); +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index eb09e86044c6d4..e6a2a40daf289c 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -372,11 +372,12 @@ int drm_prime_fd_to_handle_ioctl(struct drm_device *dev, void *data, + { + struct drm_prime_handle *args = data; + +- if (!dev->driver->prime_fd_to_handle) +- return -ENOSYS; ++ if (dev->driver->prime_fd_to_handle) { ++ return dev->driver->prime_fd_to_handle(dev, file_priv, args->fd, ++ &args->handle); ++ } + +- return dev->driver->prime_fd_to_handle(dev, file_priv, +- args->fd, &args->handle); ++ return drm_gem_prime_fd_to_handle(dev, file_priv, args->fd, &args->handle); + } + + static struct dma_buf *export_and_register_object(struct drm_device *dev, +@@ -518,15 +519,17 @@ int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data, + { + struct drm_prime_handle *args = data; + +- if (!dev->driver->prime_handle_to_fd) +- return -ENOSYS; +- + /* check flags are valid */ + if (args->flags & ~(DRM_CLOEXEC | DRM_RDWR)) + return -EINVAL; + +- return dev->driver->prime_handle_to_fd(dev, file_priv, +- args->handle, args->flags, &args->fd); ++ if (dev->driver->prime_handle_to_fd) { ++ return dev->driver->prime_handle_to_fd(dev, file_priv, ++ args->handle, args->flags, ++ &args->fd); ++ } ++ return drm_gem_prime_handle_to_fd(dev, file_priv, args->handle, ++ args->flags, &args->fd); + } + + /** +diff --git a/include/drm/drm_drv.h b/include/drm/drm_drv.h +index 4ae996ce8d4037..f20c7374e4d9ed 100644 +--- a/include/drm/drm_drv.h ++++ b/include/drm/drm_drv.h +@@ -307,22 +307,14 @@ struct drm_driver { + /** + * @prime_handle_to_fd: + * +- * Main PRIME export function. Should be implemented with +- * drm_gem_prime_handle_to_fd() for GEM based drivers. +- * +- * For an in-depth discussion see :ref:`PRIME buffer sharing +- * documentation `. ++ * PRIME export function. Only used by vmwgfx. + */ + int (*prime_handle_to_fd)(struct drm_device *dev, struct drm_file *file_priv, + uint32_t handle, uint32_t flags, int *prime_fd); + /** + * @prime_fd_to_handle: + * +- * Main PRIME import function. Should be implemented with +- * drm_gem_prime_fd_to_handle() for GEM based drivers. +- * +- * For an in-depth discussion see :ref:`PRIME buffer sharing +- * documentation `. ++ * PRIME import function. Only used by vmwgfx. + */ + int (*prime_fd_to_handle)(struct drm_device *dev, struct drm_file *file_priv, + int prime_fd, uint32_t *handle); + +From 4d6619ab9f3fcd8be1efc930b0fac546a14bbd4a Mon Sep 17 00:00:00 2001 +From: Thomas Zimmermann +Date: Tue, 20 Jun 2023 09:59:58 +0200 +Subject: [PATCH 101/150] [MOD] drm: Clear fd/handle callbacks in struct + drm_driver + +Clear all assignments of struct drm_driver's fd/handle callbacks to +drm_gem_prime_fd_to_handle() and drm_gem_prime_handle_to_fd(). These +functions are called by default. Add a TODO item to convert vmwgfx +to the defaults as well. + +v2: + * remove TODO item (Zack) + * also update amdgpu's amdgpu_partition_driver + +Signed-off-by: Thomas Zimmermann +Reviewed-by: Simon Ser +Acked-by: Alex Deucher +Acked-by: Jeffrey Hugo # qaic +Link: https://patchwork.freedesktop.org/patch/msgid/20230620080252.16368-3-tzimmermann@suse.de +--- + drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 2 -- + drivers/gpu/drm/armada/armada_drv.c | 2 -- + drivers/gpu/drm/drm_prime.c | 13 ++++--------- + drivers/gpu/drm/etnaviv/etnaviv_drv.c | 2 -- + drivers/gpu/drm/exynos/exynos_drm_drv.c | 2 -- + drivers/gpu/drm/i915/i915_driver.c | 2 -- + drivers/gpu/drm/lima/lima_drv.c | 2 -- + drivers/gpu/drm/mediatek/mtk_drm_drv.c | 2 -- + drivers/gpu/drm/msm/msm_drv.c | 2 -- + drivers/gpu/drm/nouveau/nouveau_drm.c | 2 -- + drivers/gpu/drm/omapdrm/omap_drv.c | 2 -- + drivers/gpu/drm/panfrost/panfrost_drv.c | 2 -- + drivers/gpu/drm/pl111/pl111_drv.c | 2 -- + drivers/gpu/drm/qxl/qxl_drv.c | 2 -- + drivers/gpu/drm/radeon/radeon_drv.c | 2 -- + drivers/gpu/drm/rcar-du/rcar_du_drv.c | 2 -- + drivers/gpu/drm/rockchip/rockchip_drm_drv.c | 2 -- + drivers/gpu/drm/tegra/drm.c | 2 -- + drivers/gpu/drm/v3d/v3d_drv.c | 2 -- + drivers/gpu/drm/virtio/virtgpu_drv.c | 2 -- + drivers/gpu/drm/xen/xen_drm_front.c | 2 -- + include/drm/drm_gem_dma_helper.h | 2 -- + include/drm/drm_gem_vram_helper.h | 3 +-- + 23 files changed, 5 insertions(+), 53 deletions(-) + +diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +index be4d1aeef47b44..db7c1f8bc95b31 100644 +--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c ++++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +@@ -2810,8 +2810,6 @@ static const struct drm_driver amdgpu_kms_driver = { + .fops = &amdgpu_driver_kms_fops, + .release = &amdgpu_driver_release_kms, + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = amdgpu_gem_prime_import, + .gem_prime_mmap = drm_gem_prime_mmap, + +diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c +index 688ba358f5319b..147bf1723bffdb 100644 +--- a/drivers/gpu/drm/armada/armada_drv.c ++++ b/drivers/gpu/drm/armada/armada_drv.c +@@ -38,8 +38,6 @@ DEFINE_DRM_GEM_FOPS(armada_drm_fops); + + static const struct drm_driver armada_drm_driver = { + .lastclose = drm_fb_helper_lastclose, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = armada_gem_prime_import, + .dumb_create = armada_gem_dumb_create, + .major = 1, +diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c +index e6a2a40daf289c..c631a200698582 100644 +--- a/drivers/gpu/drm/drm_prime.c ++++ b/drivers/gpu/drm/drm_prime.c +@@ -51,15 +51,10 @@ MODULE_IMPORT_NS(DMA_BUF); + * between applications, they can't be guessed like the globally unique GEM + * names. + * +- * Drivers that support the PRIME API implement the +- * &drm_driver.prime_handle_to_fd and &drm_driver.prime_fd_to_handle operations. +- * GEM based drivers must use drm_gem_prime_handle_to_fd() and +- * drm_gem_prime_fd_to_handle() to implement these. For GEM based drivers the +- * actual driver interfaces is provided through the &drm_gem_object_funcs.export +- * and &drm_driver.gem_prime_import hooks. +- * +- * &dma_buf_ops implementations for GEM drivers are all individually exported +- * for drivers which need to overwrite or reimplement some of them. ++ * Drivers that support the PRIME API implement the drm_gem_object_funcs.export ++ * and &drm_driver.gem_prime_import hooks. &dma_buf_ops implementations for ++ * drivers are all individually exported for drivers which need to overwrite ++ * or reimplement some of them. + * + * Reference Counting for GEM Drivers + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c +index 1d2b4fb4bcf8b7..7ebce549fb9144 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c +@@ -474,8 +474,6 @@ static const struct drm_driver etnaviv_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_RENDER, + .open = etnaviv_open, + .postclose = etnaviv_postclose, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = etnaviv_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + #ifdef CONFIG_DEBUG_FS +diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c +index 16c539657f730c..1b2951edc26070 100644 +--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c ++++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c +@@ -111,8 +111,6 @@ static const struct drm_driver exynos_drm_driver = { + .lastclose = drm_fb_helper_lastclose, + .postclose = exynos_drm_postclose, + .dumb_create = exynos_drm_gem_dumb_create, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = exynos_drm_gem_prime_import, + .gem_prime_import_sg_table = exynos_drm_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, +diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c +index be0ebed2a360fd..449f5b9e6044bf 100644 +--- a/drivers/gpu/drm/i915/i915_driver.c ++++ b/drivers/gpu/drm/i915/i915_driver.c +@@ -1921,8 +1921,6 @@ static const struct drm_driver i915_drm_driver = { + .lastclose = i915_driver_lastclose, + .postclose = i915_driver_postclose, + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = i915_gem_prime_import, + + .dumb_create = i915_gem_dumb_create, +diff --git a/drivers/gpu/drm/lima/lima_drv.c b/drivers/gpu/drm/lima/lima_drv.c +index 39cab4a55f5728..51efdc107314fc 100644 +--- a/drivers/gpu/drm/lima/lima_drv.c ++++ b/drivers/gpu/drm/lima/lima_drv.c +@@ -276,9 +276,7 @@ static const struct drm_driver lima_drm_driver = { + .patchlevel = 0, + + .gem_create_object = lima_gem_create_object, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, + .gem_prime_mmap = drm_gem_prime_mmap, + }; + +diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c +index 25639fbfd374af..c7fa26f45780d5 100644 +--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c ++++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c +@@ -472,8 +472,6 @@ static const struct drm_driver mtk_drm_driver = { + + .dumb_create = mtk_drm_gem_dumb_create, + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = mtk_drm_gem_prime_import, + .gem_prime_import_sg_table = mtk_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, +diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c +index f982a827be7ca0..73539e6bea94af 100644 +--- a/drivers/gpu/drm/msm/msm_drv.c ++++ b/drivers/gpu/drm/msm/msm_drv.c +@@ -1083,8 +1083,6 @@ static const struct drm_driver msm_driver = { + .lastclose = drm_fb_helper_lastclose, + .dumb_create = msm_gem_dumb_create, + .dumb_map_offset = msm_gem_dumb_map_offset, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = msm_gem_prime_import_sg_table, + .gem_prime_mmap = msm_gem_prime_mmap, + #ifdef CONFIG_DEBUG_FS +diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c +index 28062d682f4367..3b01e86e71a123 100644 +--- a/drivers/gpu/drm/nouveau/nouveau_drm.c ++++ b/drivers/gpu/drm/nouveau/nouveau_drm.c +@@ -1260,8 +1260,6 @@ driver_stub = { + .num_ioctls = ARRAY_SIZE(nouveau_ioctls), + .fops = &nouveau_driver_fops, + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = nouveau_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + +diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c +index 5b6d1668f405c6..99c4c37bc26b6f 100644 +--- a/drivers/gpu/drm/omapdrm/omap_drv.c ++++ b/drivers/gpu/drm/omapdrm/omap_drv.c +@@ -693,8 +693,6 @@ static const struct drm_driver omap_drm_driver = { + #ifdef CONFIG_DEBUG_FS + .debugfs_init = omap_debugfs_init, + #endif +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = omap_gem_prime_import, + .dumb_create = omap_gem_dumb_create, + .dumb_map_offset = omap_gem_dumb_map_offset, +diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c +index 5fbdba35472cb8..bdb2c424640bb9 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_drv.c ++++ b/drivers/gpu/drm/panfrost/panfrost_drv.c +@@ -551,8 +551,6 @@ static const struct drm_driver panfrost_drm_driver = { + .minor = 2, + + .gem_create_object = panfrost_gem_create_object, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + }; +diff --git a/drivers/gpu/drm/pl111/pl111_drv.c b/drivers/gpu/drm/pl111/pl111_drv.c +index eb25eedb5ee008..fbc1352eb63a56 100644 +--- a/drivers/gpu/drm/pl111/pl111_drv.c ++++ b/drivers/gpu/drm/pl111/pl111_drv.c +@@ -224,8 +224,6 @@ static const struct drm_driver pl111_drm_driver = { + .minor = 0, + .patchlevel = 0, + .dumb_create = drm_gem_dma_dumb_create, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = pl111_gem_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + +diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c +index 3044ca948ce22d..c259968a5b1ef3 100644 +--- a/drivers/gpu/drm/qxl/qxl_drv.c ++++ b/drivers/gpu/drm/qxl/qxl_drv.c +@@ -289,8 +289,6 @@ static struct drm_driver qxl_driver = { + #if defined(CONFIG_DEBUG_FS) + .debugfs_init = qxl_debugfs_init, + #endif +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = qxl_gem_prime_import_sg_table, + .fops = &qxl_fops, + .ioctls = qxl_ioctls, +diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c +index 6cbe1ab81abad7..67345bafa3c89b 100644 +--- a/drivers/gpu/drm/radeon/radeon_drv.c ++++ b/drivers/gpu/drm/radeon/radeon_drv.c +@@ -605,8 +605,6 @@ static const struct drm_driver kms_driver = { + .dumb_map_offset = radeon_mode_dumb_mmap, + .fops = &radeon_driver_kms_fops, + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = radeon_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + +diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c +index bd7003d6e0753e..84e474eee59472 100644 +--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c ++++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c +@@ -583,8 +583,6 @@ DEFINE_DRM_GEM_DMA_FOPS(rcar_du_fops); + static const struct drm_driver rcar_du_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .dumb_create = rcar_du_dumb_create, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = rcar_du_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + .fops = &rcar_du_fops, +diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c +index b8d06d221c4a31..a940283870da75 100644 +--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c ++++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c +@@ -1991,8 +1991,6 @@ static struct drm_driver rockchip_drm_driver = { + .lastclose = rockchip_drm_lastclose, + .open = rockchip_drm_open, + .dumb_create = rockchip_gem_dumb_create, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = rockchip_drm_gem_prime_import, + .gem_prime_import_sg_table = rockchip_gem_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, +diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c +index 5fc55b9777cbf4..f98c485243202b 100644 +--- a/drivers/gpu/drm/tegra/drm.c ++++ b/drivers/gpu/drm/tegra/drm.c +@@ -891,8 +891,6 @@ static const struct drm_driver tegra_drm_driver = { + .debugfs_init = tegra_debugfs_init, + #endif + +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import = tegra_gem_prime_import, + + .dumb_create = tegra_bo_dumb_create, +diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c +index e8c975b8158592..9433a6bf6edf14 100644 +--- a/drivers/gpu/drm/v3d/v3d_drv.c ++++ b/drivers/gpu/drm/v3d/v3d_drv.c +@@ -172,8 +172,6 @@ static const struct drm_driver v3d_drm_driver = { + #endif + + .gem_create_object = v3d_create_object, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = v3d_prime_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + +diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c +index 0035affc3e590f..314dbb5e7fbe6f 100644 +--- a/drivers/gpu/drm/virtio/virtgpu_drv.c ++++ b/drivers/gpu/drm/virtio/virtgpu_drv.c +@@ -181,8 +181,6 @@ static const struct drm_driver driver = { + #if defined(CONFIG_DEBUG_FS) + .debugfs_init = virtio_gpu_debugfs_init, + #endif +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_mmap = drm_gem_prime_mmap, + .gem_prime_import = virtgpu_gem_prime_import, + .gem_prime_import_sg_table = virtgpu_gem_prime_import_sg_table, +diff --git a/drivers/gpu/drm/xen/xen_drm_front.c b/drivers/gpu/drm/xen/xen_drm_front.c +index 0d8e6bd1ccbf20..a37224a2da538b 100644 +--- a/drivers/gpu/drm/xen/xen_drm_front.c ++++ b/drivers/gpu/drm/xen/xen_drm_front.c +@@ -474,8 +474,6 @@ DEFINE_DRM_GEM_FOPS(xen_drm_dev_fops); + static const struct drm_driver xen_drm_driver = { + .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC, + .release = xen_drm_drv_release, +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, + .gem_prime_import_sg_table = xen_drm_front_gem_import_sg_table, + .gem_prime_mmap = drm_gem_prime_mmap, + .dumb_create = xen_drm_drv_dumb_create, +diff --git a/include/drm/drm_gem_dma_helper.h b/include/drm/drm_gem_dma_helper.h +index 8a043235dad81f..f0b4e3d3d57ded 100644 +--- a/include/drm/drm_gem_dma_helper.h ++++ b/include/drm/drm_gem_dma_helper.h +@@ -167,8 +167,6 @@ drm_gem_dma_prime_import_sg_table(struct drm_device *dev, + */ + #define DRM_GEM_DMA_DRIVER_OPS_WITH_DUMB_CREATE(dumb_create_func) \ + .dumb_create = (dumb_create_func), \ +- .prime_handle_to_fd = drm_gem_prime_handle_to_fd, \ +- .prime_fd_to_handle = drm_gem_prime_fd_to_handle, \ + .gem_prime_import_sg_table = drm_gem_dma_prime_import_sg_table, \ + .gem_prime_mmap = drm_gem_prime_mmap + +diff --git a/include/drm/drm_gem_vram_helper.h b/include/drm/drm_gem_vram_helper.h +index c083a1d71cf4c2..320a99e5a2bc88 100644 +--- a/include/drm/drm_gem_vram_helper.h ++++ b/include/drm/drm_gem_vram_helper.h +@@ -159,8 +159,7 @@ void drm_gem_vram_simple_display_pipe_cleanup_fb( + #define DRM_GEM_VRAM_DRIVER \ + .debugfs_init = drm_vram_mm_debugfs_init, \ + .dumb_create = drm_gem_vram_driver_dumb_create, \ +- .dumb_map_offset = drm_gem_ttm_dumb_map_offset, \ +- .gem_prime_mmap = drm_gem_prime_mmap ++ .dumb_map_offset = drm_gem_ttm_dumb_map_offset + + /* + * VRAM memory manager + +From 570c505db1373714d201c6a55524fb602408c47e Mon Sep 17 00:00:00 2001 +From: boogie +Date: Sat, 2 Dec 2023 00:38:13 +0100 +Subject: [PATCH 102/150] dts: arm64: rk3588s: migrate gpu to panthor + +--- + arch/arm64/boot/dts/rockchip/rk3588s.dtsi | 239 +++++----------------- + 1 file changed, 51 insertions(+), 188 deletions(-) + +diff --git a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi +index 6b5a512aace40d..b7d5bca2eab63f 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi +@@ -2367,17 +2367,16 @@ + }; + + gpu: gpu@fb000000 { +- compatible = "arm,mali-bifrost"; ++ compatible = "rockchip,rk3588-mali", "arm,mali-valhall-csf"; + reg = <0x0 0xfb000000 0x0 0x200000>; +- interrupts = , +- , +- ; +- interrupt-names = "GPU", "MMU", "JOB"; +- +- clocks = <&scmi_clk SCMI_CLK_GPU>, <&cru CLK_GPU_COREGROUP>, +- <&cru CLK_GPU_STACKS>, <&cru CLK_GPU>; +- clock-names = "clk_mali", "clk_gpu_coregroup", +- "clk_gpu_stacks", "clk_gpu"; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "job", "mmu", "gpu"; ++ ++ clock-names = "core", "coregroup", "stacks"; ++ clocks = <&cru CLK_GPU>, <&cru CLK_GPU_COREGROUP>, ++ <&cru CLK_GPU_STACKS>; + assigned-clocks = <&scmi_clk SCMI_CLK_GPU>; + assigned-clock-rates = <200000000>; + power-domains = <&power RK3588_PD_GPU>; +@@ -2385,35 +2384,22 @@ + #cooling-cells = <2>; + dynamic-power-coefficient = <2982>; + +- upthreshold = <30>; +- downdifferential = <10>; +- + status = "disabled"; + }; + + gpu_opp_table: gpu-opp-table { + compatible = "operating-points-v2"; + +- nvmem-cells = <&gpu_leakage>, <&gpu_opp_info>, <&specification_serial_number>; +- nvmem-cell-names = "leakage", "opp-info", "specification_serial_number"; +- rockchip,supported-hw; ++ nvmem-cells = <&gpu_leakage>; ++ nvmem-cell-names = "leakage"; + +- rockchip,pvtm-hw = <0x04>; +- rockchip,pvtm-voltage-sel-hw = < +- 0 799 0 +- 800 819 1 +- 820 844 2 +- 845 869 3 +- 870 894 4 +- 895 9999 5 +- >; + rockchip,pvtm-voltage-sel = < +- 0 815 0 +- 816 835 1 +- 836 860 2 +- 861 885 3 +- 886 910 4 +- 911 9999 5 ++ 0 815 0 ++ 816 835 1 ++ 836 860 2 ++ 861 885 3 ++ 886 910 4 ++ 911 9999 5 + >; + rockchip,pvtm-pvtpll; + rockchip,pvtm-offset = <0x1c>; +@@ -2424,16 +2410,17 @@ + rockchip,pvtm-temp-prop = <(-135) (-135)>; + rockchip,pvtm-thermal-zone = "gpu-thermal"; + +- rockchip,opp-clocks = <&cru CLK_GPU>; ++ clocks = <&cru CLK_GPU>; ++ clock-names = "clk"; + rockchip,grf = <&gpu_grf>; + volt-mem-read-margin = < +- 855000 1 +- 765000 2 +- 675000 3 +- 495000 4 ++ 855000 1 ++ 765000 2 ++ 675000 3 ++ 495000 4 + >; + low-volt-mem-read-margin = <4>; +- intermediate-threshold-freq = <400000>; /* KHz */ ++ intermediate-threshold-freq = <400000>; /* KHz */ + + rockchip,temp-hysteresis = <5000>; + rockchip,low-temp = <15000>; +@@ -2441,180 +2428,56 @@ + rockchip,high-temp = <85000>; + rockchip,high-temp-max-freq = <800000>; + +- /* RK3588 gpu OPPs */ + opp-300000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <300000000>; +- opp-microvolt = <675000 675000 850000>, +- <675000 675000 850000>; ++ opp-microvolt = <675000 675000 850000>; + }; + opp-400000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <400000000>; +- opp-microvolt = <675000 675000 850000>, +- <675000 675000 850000>; ++ opp-microvolt = <675000 675000 850000>; + }; + opp-500000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <500000000>; +- opp-microvolt = <675000 675000 850000>, +- <675000 675000 850000>; ++ opp-microvolt = <675000 675000 850000>; + }; + opp-600000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <600000000>; +- opp-microvolt = <675000 675000 850000>, +- <675000 675000 850000>; ++ opp-microvolt = <675000 675000 850000>; + }; + opp-700000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <700000000>; +- opp-microvolt = <700000 700000 850000>, +- <700000 700000 850000>; +- opp-microvolt-L2 = <687500 687500 850000>, +- <687500 687500 850000>; +- opp-microvolt-L3 = <675000 675000 850000>, +- <675000 675000 850000>; +- opp-microvolt-L4 = <675000 675000 850000>, +- <675000 675000 850000>; +- opp-microvolt-L5 = <675000 675000 850000>, +- <675000 675000 850000>; ++ opp-microvolt = <700000 700000 850000>; ++ opp-microvolt-L2 = <687500 687500 850000>; ++ opp-microvolt-L3 = <675000 675000 850000>; ++ opp-microvolt-L4 = <675000 675000 850000>; ++ opp-microvolt-L5 = <675000 675000 850000>; + }; + opp-800000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <800000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- opp-microvolt-L1 = <737500 737500 850000>, +- <737500 737500 850000>; +- opp-microvolt-L2 = <725000 725000 850000>, +- <725000 725000 850000>; +- opp-microvolt-L3 = <712500 712500 850000>, +- <712500 712500 850000>; +- opp-microvolt-L4 = <700000 700000 850000>, +- <700000 700000 850000>; +- opp-microvolt-L5 = <700000 700000 850000>, +- <700000 700000 850000>; ++ opp-microvolt = <750000 750000 850000>; ++ opp-microvolt-L1 = <737500 737500 850000>; ++ opp-microvolt-L2 = <725000 725000 850000>; ++ opp-microvolt-L3 = <712500 712500 850000>; ++ opp-microvolt-L4 = <700000 700000 850000>; ++ opp-microvolt-L5 = <700000 700000 850000>; + }; + opp-900000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <900000000>; +- opp-microvolt = <800000 800000 850000>, +- <800000 800000 850000>; +- opp-microvolt-L1 = <787500 787500 850000>, +- <787500 787500 850000>; +- opp-microvolt-L2 = <775000 775000 850000>, +- <775000 775000 850000>; +- opp-microvolt-L3 = <762500 762500 850000>, +- <762500 762500 850000>; +- opp-microvolt-L4 = <750000 750000 850000>, +- <750000 750000 850000>; +- opp-microvolt-L5 = <737500 737500 850000>, +- <737500 737500 850000>; ++ opp-microvolt = <800000 800000 850000>; ++ opp-microvolt-L1 = <787500 787500 850000>; ++ opp-microvolt-L2 = <775000 775000 850000>; ++ opp-microvolt-L3 = <762500 762500 850000>; ++ opp-microvolt-L4 = <750000 750000 850000>; ++ opp-microvolt-L5 = <737500 737500 850000>; + }; + opp-1000000000 { +- opp-supported-hw = <0xf9 0xffff>; + opp-hz = /bits/ 64 <1000000000>; +- opp-microvolt = <850000 850000 850000>, +- <850000 850000 850000>; +- opp-microvolt-L1 = <837500 837500 850000>, +- <837500 837500 850000>; +- opp-microvolt-L2 = <825000 825000 850000>, +- <825000 825000 850000>; +- opp-microvolt-L3 = <812500 812500 850000>, +- <812500 812500 850000>; +- opp-microvolt-L4 = <800000 800000 850000>, +- <800000 800000 850000>; +- opp-microvolt-L5 = <787500 787500 850000>, +- <787500 787500 850000>; +- }; +- +- /* RK3588J/M gpu OPPs */ +- opp-j-m-300000000 { +- opp-supported-hw = <0x06 0xffff>; +- opp-hz = /bits/ 64 <300000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-j-m-400000000 { +- opp-supported-hw = <0x06 0xffff>; +- opp-hz = /bits/ 64 <400000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-j-m-500000000 { +- opp-supported-hw = <0x06 0xffff>; +- opp-hz = /bits/ 64 <500000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-j-m-600000000 { +- opp-supported-hw = <0x06 0xffff>; +- opp-hz = /bits/ 64 <600000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-j-m-700000000 { +- opp-supported-hw = <0x06 0xffff>; +- opp-hz = /bits/ 64 <700000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- /* RK3588J gpu OPPs */ +- opp-j-850000000 { +- opp-supported-hw = <0x04 0xffff>; +- opp-hz = /bits/ 64 <850000000>; +- opp-microvolt = <787500 787500 850000>, +- <787500 787500 850000>; +- opp-microvolt-L1 = <775000 775000 850000>, +- <775000 775000 850000>; +- opp-microvolt-L2 = <762500 762500 850000>, +- <762500 762500 850000>; +- opp-microvolt-L3 = <750000 750000 850000>, +- <750000 750000 850000>; +- opp-microvolt-L4 = <750000 750000 850000>, +- <750000 750000 850000>; +- opp-microvolt-L5 = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- /* RK3588M gpu OPPs */ +- opp-m-800000000 { +- opp-supported-hw = <0x02 0xffff>; +- opp-hz = /bits/ 64 <800000000>; +- opp-microvolt = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-m-900000000 { +- opp-supported-hw = <0x02 0xffff>; +- opp-hz = /bits/ 64 <900000000>; +- opp-microvolt = <800000 800000 850000>, +- <800000 800000 850000>; +- opp-microvolt-L1 = <787500 787500 850000>, +- <787500 787500 850000>; +- opp-microvolt-L2 = <775000 775000 850000>, +- <775000 775000 850000>; +- opp-microvolt-L3 = <762500 762500 850000>, +- <762500 762500 850000>; +- opp-microvolt-L4 = <750000 750000 850000>, +- <750000 750000 850000>; +- opp-microvolt-L5 = <750000 750000 850000>, +- <750000 750000 850000>; +- }; +- opp-m-1000000000 { +- opp-supported-hw = <0x02 0xffff>; +- opp-hz = /bits/ 64 <1000000000>; +- opp-microvolt = <850000 850000 850000>, +- <850000 850000 850000>; +- opp-microvolt-L1 = <837500 837500 850000>, +- <837500 837500 850000>; +- opp-microvolt-L2 = <825000 825000 850000>, +- <825000 825000 850000>; +- opp-microvolt-L3 = <812500 812500 850000>, +- <812500 812500 850000>; +- opp-microvolt-L4 = <800000 800000 850000>, +- <800000 800000 850000>; +- opp-microvolt-L5 = <787500 787500 850000>, +- <787500 787500 850000>; ++ opp-microvolt = <850000 850000 850000>; ++ opp-microvolt-L1 = <837500 837500 850000>; ++ opp-microvolt-L2 = <825000 825000 850000>; ++ opp-microvolt-L3 = <812500 812500 850000>; ++ opp-microvolt-L4 = <800000 800000 850000>; ++ opp-microvolt-L5 = <787500 787500 850000>; + }; + }; + + +From 9677ab8305ce4fd1fa4d3131908d6c9e4dd94a10 Mon Sep 17 00:00:00 2001 +From: boogie +Date: Fri, 22 Mar 2024 21:16:08 +0100 +Subject: [PATCH 103/150] dts: arm64: rk3588s: add seperate mali nodes + +--- + arch/arm64/boot/dts/rockchip/rk3588s.dtsi | 25 +++++++++++++++++++++++ + 1 file changed, 25 insertions(+) + +diff --git a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi +index b7d5bca2eab63f..dfa8bc88406b5c 100644 +--- a/arch/arm64/boot/dts/rockchip/rk3588s.dtsi ++++ b/arch/arm64/boot/dts/rockchip/rk3588s.dtsi +@@ -2387,6 +2387,31 @@ + status = "disabled"; + }; + ++ gpu_mali: gpu-mali@fb000000 { ++ compatible = "arm,mali-bifrost"; ++ reg = <0x0 0xfb000000 0x0 0x200000>; ++ interrupts = , ++ , ++ ; ++ interrupt-names = "GPU", "MMU", "JOB"; ++ ++ clocks = <&scmi_clk SCMI_CLK_GPU>, <&cru CLK_GPU_COREGROUP>, ++ <&cru CLK_GPU_STACKS>, <&cru CLK_GPU>; ++ clock-names = "clk_mali", "clk_gpu_coregroup", ++ "clk_gpu_stacks", "clk_gpu"; ++ assigned-clocks = <&scmi_clk SCMI_CLK_GPU>; ++ assigned-clock-rates = <200000000>; ++ power-domains = <&power RK3588_PD_GPU>; ++ operating-points-v2 = <&gpu_opp_table>; ++ #cooling-cells = <2>; ++ dynamic-power-coefficient = <2982>; ++ ++ upthreshold = <30>; ++ downdifferential = <10>; ++ ++ status = "disabled"; ++ }; ++ + gpu_opp_table: gpu-opp-table { + compatible = "operating-points-v2"; + + +From e4d2808313331dfd3efc3ee0e97e274447072647 Mon Sep 17 00:00:00 2001 +From: boogie +Date: Fri, 22 Mar 2024 22:10:26 +0100 +Subject: [PATCH 104/150] dts: arm64: rockchip: add an overlay to switch mali + from panthor + +--- + arch/arm64/boot/dts/rockchip/overlay/Makefile | 4 ++- + .../dts/rockchip/overlay/rk3588s-mali.dts | 27 +++++++++++++++++++ + 2 files changed, 30 insertions(+), 1 deletion(-) + create mode 100644 arch/arm64/boot/dts/rockchip/overlay/rk3588s-mali.dts + +diff --git a/arch/arm64/boot/dts/rockchip/overlay/Makefile b/arch/arm64/boot/dts/rockchip/overlay/Makefile +index 2d1fe84f3dbbe0..305aa67330961b 100644 +--- a/arch/arm64/boot/dts/rockchip/overlay/Makefile ++++ b/arch/arm64/boot/dts/rockchip/overlay/Makefile +@@ -289,7 +289,9 @@ dtbo-$(CONFIG_ARCH_ROCKCHIP) += \ + rk3568-uart8-m1.dtbo \ + rk3568-uart9-m1.dtbo \ + rk3568-w1-gpio3-a5.dtbo \ +- rk3568-w1-gpio3-d1.dtbo ++ rk3568-w1-gpio3-d1.dtbo \ ++ rk3588s-mali.dtbo ++ + + dtbotxt-$(CONFIG_ARCH_ROCKCHIP) += \ + README.rockchip-overlays +diff --git a/arch/arm64/boot/dts/rockchip/overlay/rk3588s-mali.dts b/arch/arm64/boot/dts/rockchip/overlay/rk3588s-mali.dts +new file mode 100644 +index 00000000000000..d1afe81e433eb1 +--- /dev/null ++++ b/arch/arm64/boot/dts/rockchip/overlay/rk3588s-mali.dts +@@ -0,0 +1,27 @@ ++/dts-v1/; ++/plugin/; ++ ++#include ++#include ++#include ++ ++/ { ++ ++ fragment@0 { ++ target = <&gpu>; ++ ++ __overlay__ { ++ status = "disabled"; ++ }; ++ }; ++ ++ fragment@1 { ++ target = <&gpu_mali>; ++ ++ __overlay__ { ++ status = "okay"; ++ mali-supply = <&vdd_gpu_s0>; ++ sram-supply = <&vdd_gpu_mem_s0>; ++ }; ++ }; ++}; +\ No newline at end of file + +From 48e99654fda7f6a57cc3eae6429ddcfd2fae1b7c Mon Sep 17 00:00:00 2001 +From: boogie +Date: Fri, 1 Dec 2023 22:55:30 +0100 +Subject: [PATCH 105/150] config: arm64: rockchip: enable panthor driver + +--- + debian.rockchip/config/config.common.ubuntu | 6 ++++++ + 1 file changed, 6 insertions(+) + +diff --git a/debian.rockchip/config/config.common.ubuntu b/debian.rockchip/config/config.common.ubuntu +index a7b6ad12e2477b..08e312aa205c37 100644 +--- a/debian.rockchip/config/config.common.ubuntu ++++ b/debian.rockchip/config/config.common.ubuntu +@@ -5592,6 +5592,12 @@ CONFIG_DVB_SP2=m + # Graphics support + # + CONFIG_DRM=y ++CONFIG_DRM_GPUVM=y ++CONFIG_DRM_EXEC=y ++CONFIG_DRM_PANTHOR=m ++CONFIG_ROCKCHIP_REGULATOR_COUPLER=y ++CONFIG_FW_LOADER_USER_HELPER=y ++ + CONFIG_DRM_MIPI_DBI=m + CONFIG_DRM_EDID=y + # CONFIG_DRM_IGNORE_IOTCL_PERMIT is not set + +From 3abcd4f2d9186a3cac3df474be826e83b303d616 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Mon, 4 Mar 2024 10:08:11 +0100 +Subject: [PATCH 106/150] drm/panthor: Explicitly include mm.h for the {virt, + __phys)_to_pfn() defs + +Something on arm[64] must be including , but things fail +to compile on sparc64. Make sure this header is included (through +linux/mm.h) so this driver can be compile-tested on all supported +architectures. + +Reported-by: kernel test robot +Closes: https://lore.kernel.org/oe-kbuild-all/202403031142.Vl4pW7X6-lkp@intel.com/ +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240304090812.3941084-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index 5be34f4821c59e..b299cdfbd22bd9 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -4,6 +4,7 @@ + /* Copyright 2023 Collabora ltd. */ + + #include ++#include + #include + #include + #include + +From e70659218eef31e5dec7e61c760a92948ead09ec Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Mon, 4 Mar 2024 10:08:12 +0100 +Subject: [PATCH 107/150] drm/panthor: Fix undefined + panthor_device_suspend/resume symbol issue + +panthor_device_resume/suspend() are only compiled when CONFIG_PM is +enabled but panthro_drv.c doesn't use the pm_ptr() macro to conditionally +discard resume/suspend assignments, which causes undefined symbol +errors at link time when !PM. + +We could fix that by using pm_ptr(), but supporting the !PM case makes +little sense (the whole point of these embedded GPUs is to be low power, +so proper PM is a basic requirement in that case). So let's just enforce +the presence of CONFIG_PM with a Kconfig dependency instead. + +If someone needs to relax this dependency, it can be done in a follow-up. + +Reported-by: kernel test robot +Closes: https://lore.kernel.org/oe-kbuild-all/202403031944.EOimQ8WK-lkp@intel.com/ +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240304090812.3941084-4-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/Kconfig | 1 + + drivers/gpu/drm/panthor/panthor_device.c | 2 -- + 2 files changed, 1 insertion(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/Kconfig b/drivers/gpu/drm/panthor/Kconfig +index 55b40ad07f3b07..fdce7c1b2310ff 100644 +--- a/drivers/gpu/drm/panthor/Kconfig ++++ b/drivers/gpu/drm/panthor/Kconfig +@@ -6,6 +6,7 @@ config DRM_PANTHOR + depends on ARM || ARM64 || COMPILE_TEST + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE + depends on MMU ++ depends on PM + select DEVFREQ_GOV_SIMPLE_ONDEMAND + select DRM_EXEC + select DRM_GEM_SHMEM_HELPER +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index b299cdfbd22bd9..3326f800eb60c3 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -401,7 +401,6 @@ int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct * + return 0; + } + +-#ifdef CONFIG_PM + int panthor_device_resume(struct device *dev) + { + struct panthor_device *ptdev = dev_get_drvdata(dev); +@@ -546,4 +545,3 @@ int panthor_device_suspend(struct device *dev) + mutex_unlock(&ptdev->pm.mmio_lock); + return ret; + } +-#endif + +From 2995ea90b7516bd4d67a935422fb65e9a95ab77f Mon Sep 17 00:00:00 2001 +From: Jani Nikula +Date: Mon, 11 Mar 2024 13:16:19 +0200 +Subject: [PATCH 108/150] Revert "drm/panthor: Fix undefined + panthor_device_suspend/resume symbol issue" + +This reverts commit 674dc7f61aefea81901c21402946074927e63f1a. + +The commit causes a recursive dependency in kconfig: + +drivers/iommu/Kconfig:14:error: recursive dependency detected! +drivers/iommu/Kconfig:14: symbol IOMMU_SUPPORT is selected by DRM_PANTHOR +drivers/gpu/drm/panthor/Kconfig:3: symbol DRM_PANTHOR depends on PM +kernel/power/Kconfig:183: symbol PM is selected by PM_SLEEP +kernel/power/Kconfig:117: symbol PM_SLEEP depends on HIBERNATE_CALLBACKS +kernel/power/Kconfig:35: symbol HIBERNATE_CALLBACKS is selected by XEN_SAVE_RESTORE +arch/x86/xen/Kconfig:67: symbol XEN_SAVE_RESTORE depends on XEN +arch/x86/xen/Kconfig:6: symbol XEN depends on PARAVIRT +arch/x86/Kconfig:781: symbol PARAVIRT is selected by HYPERV +drivers/hv/Kconfig:5: symbol HYPERV depends on X86_LOCAL_APIC +arch/x86/Kconfig:1106: symbol X86_LOCAL_APIC depends on X86_UP_APIC +arch/x86/Kconfig:1081: symbol X86_UP_APIC prompt is visible depending on PCI_MSI +drivers/pci/Kconfig:39: symbol PCI_MSI is selected by AMD_IOMMU +drivers/iommu/amd/Kconfig:3: symbol AMD_IOMMU depends on IOMMU_SUPPORT +For a resolution refer to Documentation/kbuild/kconfig-language.rst +subsection "Kconfig recursive dependency limitations" + +Fixes: 674dc7f61aef ("drm/panthor: Fix undefined panthor_device_suspend/resume symbol issue") +Cc: Boris Brezillon +Cc: Liviu Dudau +Cc: Steven Price +Acked-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240311111619.249776-1-jani.nikula@intel.com +Signed-off-by: Jani Nikula +--- + drivers/gpu/drm/panthor/Kconfig | 1 - + drivers/gpu/drm/panthor/panthor_device.c | 2 ++ + 2 files changed, 2 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/Kconfig b/drivers/gpu/drm/panthor/Kconfig +index fdce7c1b2310ff..55b40ad07f3b07 100644 +--- a/drivers/gpu/drm/panthor/Kconfig ++++ b/drivers/gpu/drm/panthor/Kconfig +@@ -6,7 +6,6 @@ config DRM_PANTHOR + depends on ARM || ARM64 || COMPILE_TEST + depends on !GENERIC_ATOMIC64 # for IOMMU_IO_PGTABLE_LPAE + depends on MMU +- depends on PM + select DEVFREQ_GOV_SIMPLE_ONDEMAND + select DRM_EXEC + select DRM_GEM_SHMEM_HELPER +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index 3326f800eb60c3..b299cdfbd22bd9 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -401,6 +401,7 @@ int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct * + return 0; + } + ++#ifdef CONFIG_PM + int panthor_device_resume(struct device *dev) + { + struct panthor_device *ptdev = dev_get_drvdata(dev); +@@ -545,3 +546,4 @@ int panthor_device_suspend(struct device *dev) + mutex_unlock(&ptdev->pm.mmio_lock); + return ret; + } ++#endif + +From a45581c0fce3427e5410a7b9c3bb2dade903cf74 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Mon, 18 Mar 2024 16:31:17 +0100 +Subject: [PATCH 109/150] drm/panthor: Fix the CONFIG_PM=n case + +Putting a hard dependency on CONFIG_PM is not possible because of a +circular dependency issue, and it's actually not desirable either. In +order to support this use case, we forcibly resume at init time, and +suspend at unplug time. + +v2: +- Drop the #ifdef CONFIG_PM section around panthor_pm_ops's definition + +Reported-by: kernel test robot +Closes: https://lore.kernel.org/oe-kbuild-all/202403031944.EOimQ8WK-lkp@intel.com/ +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: AngeloGioacchino Del Regno +Link: https://patchwork.freedesktop.org/patch/msgid/20240318153117.1321544-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 13 +++++++++++-- + drivers/gpu/drm/panthor/panthor_drv.c | 2 +- + 2 files changed, 12 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index b299cdfbd22bd9..4b336ddeb9050b 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -87,6 +87,10 @@ void panthor_device_unplug(struct panthor_device *ptdev) + pm_runtime_dont_use_autosuspend(ptdev->base.dev); + pm_runtime_put_sync_suspend(ptdev->base.dev); + ++ /* If PM is disabled, we need to call the suspend handler manually. */ ++ if (!IS_ENABLED(CONFIG_PM)) ++ panthor_device_suspend(ptdev->base.dev); ++ + /* Report the unplug operation as done to unblock concurrent + * panthor_device_unplug() callers. + */ +@@ -218,6 +222,13 @@ int panthor_device_init(struct panthor_device *ptdev) + if (ret) + return ret; + ++ /* If PM is disabled, we need to call panthor_device_resume() manually. */ ++ if (!IS_ENABLED(CONFIG_PM)) { ++ ret = panthor_device_resume(ptdev->base.dev); ++ if (ret) ++ return ret; ++ } ++ + ret = panthor_gpu_init(ptdev); + if (ret) + goto err_rpm_put; +@@ -401,7 +412,6 @@ int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct * + return 0; + } + +-#ifdef CONFIG_PM + int panthor_device_resume(struct device *dev) + { + struct panthor_device *ptdev = dev_get_drvdata(dev); +@@ -546,4 +556,3 @@ int panthor_device_suspend(struct device *dev) + mutex_unlock(&ptdev->pm.mmio_lock); + return ret; + } +-#endif +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index 32fbcba3933b18..b67abe02a3577c 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -1406,7 +1406,7 @@ static struct platform_driver panthor_driver = { + .remove_new = panthor_remove, + .driver = { + .name = "panthor", +- .pm = &panthor_pm_ops, ++ .pm = pm_ptr(&panthor_pm_ops), + .of_match_table = dt_match, + }, + }; + +From df699adec4f457e6f101e686dcfa9d2121b88692 Mon Sep 17 00:00:00 2001 +From: Steven Price +Date: Mon, 18 Mar 2024 14:51:19 +0000 +Subject: [PATCH 110/150] drm/panthor: Don't use virt_to_pfn() + +virt_to_pfn() isn't available on x86 (except to xen) so breaks +COMPILE_TEST builds. Avoid its use completely by instead storing the +struct page pointer allocated in panthor_device_init() and using +page_to_pfn() instead. + +Signed-off-by: Steven Price +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240318145119.368582-1-steven.price@arm.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 10 ++++++---- + drivers/gpu/drm/panthor/panthor_device.h | 2 +- + 2 files changed, 7 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index 4b336ddeb9050b..23953e9cb4f17c 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -151,11 +151,12 @@ static bool panthor_device_is_initialized(struct panthor_device *ptdev) + + static void panthor_device_free_page(struct drm_device *ddev, void *data) + { +- free_page((unsigned long)data); ++ __free_page(data); + } + + int panthor_device_init(struct panthor_device *ptdev) + { ++ u32 *dummy_page_virt; + struct resource *res; + struct page *p; + int ret; +@@ -176,7 +177,8 @@ int panthor_device_init(struct panthor_device *ptdev) + if (!p) + return -ENOMEM; + +- ptdev->pm.dummy_latest_flush = page_address(p); ++ ptdev->pm.dummy_latest_flush = p; ++ dummy_page_virt = page_address(p); + ret = drmm_add_action_or_reset(&ptdev->base, panthor_device_free_page, + ptdev->pm.dummy_latest_flush); + if (ret) +@@ -188,7 +190,7 @@ int panthor_device_init(struct panthor_device *ptdev) + * happens while the dummy page is mapped. Zero cannot be used because + * that means 'always flush'. + */ +- *ptdev->pm.dummy_latest_flush = 1; ++ *dummy_page_virt = 1; + + INIT_WORK(&ptdev->reset.work, panthor_device_reset_work); + ptdev->reset.wq = alloc_ordered_workqueue("panthor-reset-wq", 0); +@@ -364,7 +366,7 @@ static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf) + if (active) + pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID); + else +- pfn = virt_to_pfn(ptdev->pm.dummy_latest_flush); ++ pfn = page_to_pfn(ptdev->pm.dummy_latest_flush); + break; + + default: +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index 51c9d61b67963d..c84c27dcc92c7e 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -160,7 +160,7 @@ struct panthor_device { + * Used to replace the real LATEST_FLUSH page when the GPU + * is suspended. + */ +- u32 *dummy_latest_flush; ++ struct page *dummy_latest_flush; + } pm; + }; + + +From 15b3a33f799807b442b26bc5659da7d60530f743 Mon Sep 17 00:00:00 2001 +From: Colin Ian King +Date: Tue, 26 Mar 2024 10:02:19 +0000 +Subject: [PATCH 111/150] drm/panthor: Fix spelling mistake "readyness" -> + "readiness" + +There is a spelling mistake in a drm_err message. Fix it. + +Signed-off-by: Colin Ian King +Acked-by: Liviu Dudau +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240326100219.43989-1-colin.i.king@gmail.com +--- + drivers/gpu/drm/panthor/panthor_gpu.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c +index 6dbbc4cfbe7e6a..0f7c962440d3e2 100644 +--- a/drivers/gpu/drm/panthor/panthor_gpu.c ++++ b/drivers/gpu/drm/panthor/panthor_gpu.c +@@ -333,7 +333,7 @@ int panthor_gpu_block_power_on(struct panthor_device *ptdev, + val, (mask32 & val) == mask32, + 100, timeout_us); + if (ret) { +- drm_err(&ptdev->base, "timeout waiting on %s:%llx readyness", ++ drm_err(&ptdev->base, "timeout waiting on %s:%llx readiness", + blk_name, mask); + return ret; + } + +From 7f36c56787c0f3aa285b1782afd38241d152ac11 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Tue, 26 Mar 2024 12:12:03 +0100 +Subject: [PATCH 112/150] drm/panthor: Fix IO-page mmap() for 32-bit userspace + on 64-bit kernel +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When mapping an IO region, the pseudo-file offset is dependent on the +userspace architecture. panthor_device_mmio_offset() abstracts that +away for us by turning a userspace MMIO offset into its kernel +equivalent, but we were not updating vm_area_struct::vm_pgoff +accordingly, leading us to attach the MMIO region to the wrong file +offset. + +This has implications when we start mixing 64 bit and 32 bit apps, but +that's only really a problem when we start having more that 2^43 bytes of +memory allocated, which is very unlikely to happen. + +What's more problematic is the fact this turns our +unmap_mapping_range(DRM_PANTHOR_USER_MMIO_OFFSET) calls, which are +supposed to kill the MMIO mapping when entering suspend, into NOPs. +Which means we either keep the dummy flush_id mapping active at all +times, or we risk a BUS_FAULT if the MMIO region was mapped, and the +GPU is suspended after that. + +Solve that by patching vm_pgoff early in panthor_mmap(). With +this in place, we no longer need the panthor_device_mmio_offset() +helper. + +v3: +- No changes + +v2: +- Kill panthor_device_mmio_offset() + +Fixes: 5fe909cae118 ("drm/panthor: Add the device logical block") +Reported-by: Adrián Larumbe +Reported-by: Lukas F. Hartmann +Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/10835 +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240326111205.510019-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 8 ++++---- + drivers/gpu/drm/panthor/panthor_device.h | 24 ------------------------ + drivers/gpu/drm/panthor/panthor_drv.c | 17 ++++++++++++++++- + 3 files changed, 20 insertions(+), 29 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index 23953e9cb4f17c..e8a42e74e693c2 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -348,7 +348,7 @@ static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf) + { + struct vm_area_struct *vma = vmf->vma; + struct panthor_device *ptdev = vma->vm_private_data; +- u64 id = (u64)vma->vm_pgoff << PAGE_SHIFT; ++ u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT; + unsigned long pfn; + pgprot_t pgprot; + vm_fault_t ret; +@@ -361,7 +361,7 @@ static vm_fault_t panthor_mmio_vm_fault(struct vm_fault *vmf) + mutex_lock(&ptdev->pm.mmio_lock); + active = atomic_read(&ptdev->pm.state) == PANTHOR_DEVICE_PM_STATE_ACTIVE; + +- switch (panthor_device_mmio_offset(id)) { ++ switch (offset) { + case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: + if (active) + pfn = __phys_to_pfn(ptdev->phys_addr + CSF_GPU_LATEST_FLUSH_ID); +@@ -392,9 +392,9 @@ static const struct vm_operations_struct panthor_mmio_vm_ops = { + + int panthor_device_mmap_io(struct panthor_device *ptdev, struct vm_area_struct *vma) + { +- u64 id = (u64)vma->vm_pgoff << PAGE_SHIFT; ++ u64 offset = (u64)vma->vm_pgoff << PAGE_SHIFT; + +- switch (panthor_device_mmio_offset(id)) { ++ switch (offset) { + case DRM_PANTHOR_USER_FLUSH_ID_MMIO_OFFSET: + if (vma->vm_end - vma->vm_start != PAGE_SIZE || + (vma->vm_flags & (VM_WRITE | VM_EXEC))) +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index c84c27dcc92c7e..b68e8821d219cc 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -365,30 +365,6 @@ static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ + pirq); \ + } + +-/** +- * panthor_device_mmio_offset() - Turn a user MMIO offset into a kernel one +- * @offset: Offset to convert. +- * +- * With 32-bit systems being limited by the 32-bit representation of mmap2's +- * pgoffset field, we need to make the MMIO offset arch specific. This function +- * converts a user MMIO offset into something the kernel driver understands. +- * +- * If the kernel and userspace architecture match, the offset is unchanged. If +- * the kernel is 64-bit and userspace is 32-bit, the offset is adjusted to match +- * 64-bit offsets. 32-bit kernel with 64-bit userspace is impossible. +- * +- * Return: Adjusted offset. +- */ +-static inline u64 panthor_device_mmio_offset(u64 offset) +-{ +-#ifdef CONFIG_ARM64 +- if (test_tsk_thread_flag(current, TIF_32BIT)) +- offset += DRM_PANTHOR_USER_MMIO_OFFSET_64BIT - DRM_PANTHOR_USER_MMIO_OFFSET_32BIT; +-#endif +- +- return offset; +-} +- + extern struct workqueue_struct *panthor_cleanup_wq; + + #endif +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index b67abe02a3577c..455a3152ce6440 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -1326,7 +1326,22 @@ static int panthor_mmap(struct file *filp, struct vm_area_struct *vma) + if (!drm_dev_enter(file->minor->dev, &cookie)) + return -ENODEV; + +- if (panthor_device_mmio_offset(offset) >= DRM_PANTHOR_USER_MMIO_OFFSET) ++#ifdef CONFIG_ARM64 ++ /* ++ * With 32-bit systems being limited by the 32-bit representation of ++ * mmap2's pgoffset field, we need to make the MMIO offset arch ++ * specific. This converts a user MMIO offset into something the kernel ++ * driver understands. ++ */ ++ if (test_tsk_thread_flag(current, TIF_32BIT) && ++ offset >= DRM_PANTHOR_USER_MMIO_OFFSET_32BIT) { ++ offset += DRM_PANTHOR_USER_MMIO_OFFSET_64BIT - ++ DRM_PANTHOR_USER_MMIO_OFFSET_32BIT; ++ vma->vm_pgoff = offset >> PAGE_SHIFT; ++ } ++#endif ++ ++ if (offset >= DRM_PANTHOR_USER_MMIO_OFFSET) + ret = panthor_device_mmap_io(ptdev, vma); + else + ret = drm_gem_mmap(filp, vma); + +From 9bae68d0c3f0fc129ecb6d8a3d2a889f508cc942 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Tue, 26 Mar 2024 12:12:04 +0100 +Subject: [PATCH 113/150] drm/panthor: Fix ordering in _irq_suspend() + +Make sure we set suspended=true last to avoid generating an irq storm +in the unlikely case where an IRQ happens between the suspended=true +assignment and the _INT_MASK update. + +We also move the mask=0 assignment before writing to the _INT_MASK +register to prevent the thread handler from unmasking the interrupt +behind our back. This means we might lose events if there were some +pending when we get to suspend the IRQ, but that's fine. +The synchronize_irq() we have in the _irq_suspend() path was not +there to make sure all IRQs are processed, just to make sure we don't +have registers accesses coming from the irq handlers after +_irq_suspend() has been called. If there's a need to have all pending +IRQs processed, it should happen before _irq_suspend() is called. + +v3: +- Add Steve's R-b + +v2: +- New patch + +Fixes: 5fe909cae118 ("drm/panthor: Add the device logical block") +Reported-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240326111205.510019-2-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.h | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index b68e8821d219cc..8ced04dad3a8df 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -325,7 +325,7 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) + { \ + int cookie; \ + \ +- atomic_set(&pirq->suspended, true); \ ++ pirq->mask = 0; \ + \ + if (drm_dev_enter(&pirq->ptdev->base, &cookie)) { \ + gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ +@@ -333,7 +333,7 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) + drm_dev_exit(cookie); \ + } \ + \ +- pirq->mask = 0; \ ++ atomic_set(&pirq->suspended, true); \ + } \ + \ + static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask) \ + +From 93ca070cc02b569d886d663f291dca019da0714e Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Tue, 26 Mar 2024 12:12:05 +0100 +Subject: [PATCH 114/150] drm/panthor: Drop the dev_enter/exit() sections in + _irq_suspend/resume() + +There's no reason for _irq_suspend/resume() to be called after the +device has been unplugged, and keeping this dev_enter/exit() +section in _irq_suspend() is turns _irq_suspend() into a NOP +when called from the _unplug() functions, which we don't want. + +v3: +- New patch + +Fixes: 5fe909cae118 ("drm/panthor: Add the device logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240326111205.510019-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.h | 17 ++++------------- + 1 file changed, 4 insertions(+), 13 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index 8ced04dad3a8df..dc3e9c666b5bd4 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -326,13 +326,8 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) + int cookie; \ + \ + pirq->mask = 0; \ +- \ +- if (drm_dev_enter(&pirq->ptdev->base, &cookie)) { \ +- gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ +- synchronize_irq(pirq->irq); \ +- drm_dev_exit(cookie); \ +- } \ +- \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ ++ synchronize_irq(pirq->irq); \ + atomic_set(&pirq->suspended, true); \ + } \ + \ +@@ -342,12 +337,8 @@ static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u + \ + atomic_set(&pirq->suspended, false); \ + pirq->mask = mask; \ +- \ +- if (drm_dev_enter(&pirq->ptdev->base, &cookie)) { \ +- gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \ +- gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask); \ +- drm_dev_exit(cookie); \ +- } \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \ ++ gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, mask); \ + } \ + \ + static int panthor_request_ ## __name ## _irq(struct panthor_device *ptdev, \ + +From c317461f327eca8c621d3d74a51f10a256ef6f48 Mon Sep 17 00:00:00 2001 +From: Nathan Chancellor +Date: Thu, 28 Mar 2024 09:22:07 -0700 +Subject: [PATCH 115/150] drm/panthor: Fix clang -Wunused-but-set-variable in + tick_ctx_apply() + +Clang warns (or errors with CONFIG_WERROR): + + drivers/gpu/drm/panthor/panthor_sched.c:2048:6: error: variable 'csg_mod_mask' set but not used [-Werror,-Wunused-but-set-variable] + 2048 | u32 csg_mod_mask = 0, free_csg_slots = 0; + | ^ + 1 error generated. + +The variable is an artifact left over from refactoring that occurred +during the development of the initial series for this driver. Remove it +to resolve the warning. + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Signed-off-by: Nathan Chancellor +Reviewed-by: Steven Price +Reviewed-by: Justin Stitt +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240328-panthor-drop-csg_mod_mask-v1-1-5a80be3df581@kernel.org +--- + drivers/gpu/drm/panthor/panthor_sched.c | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 5f7803b6fc48c7..e5a710f190d2a8 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2045,7 +2045,7 @@ tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *c + struct panthor_device *ptdev = sched->ptdev; + struct panthor_csg_slot *csg_slot; + int prio, new_csg_prio = MAX_CSG_PRIO, i; +- u32 csg_mod_mask = 0, free_csg_slots = 0; ++ u32 free_csg_slots = 0; + struct panthor_csg_slots_upd_ctx upd_ctx; + int ret; + +@@ -2139,7 +2139,6 @@ tick_ctx_apply(struct panthor_scheduler *sched, struct panthor_sched_tick_ctx *c + + csg_iface = panthor_fw_get_csg_iface(ptdev, csg_id); + csg_slot = &sched->csg_slots[csg_id]; +- csg_mod_mask |= BIT(csg_id); + group_bind_locked(group, csg_id); + csg_slot_prog_locked(ptdev, csg_id, new_csg_prio--); + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, csg_id, + +From cfaa2cca2f68f8694af50a15c447fc63dc283656 Mon Sep 17 00:00:00 2001 +From: Liviu Dudau +Date: Tue, 2 Apr 2024 22:54:22 +0100 +Subject: [PATCH 116/150] drm/panthor: Cleanup unused variable 'cookie' + +Commit 962f88b9c916 ("drm/panthor: Drop the dev_enter/exit() sections in +_irq_suspend/resume()") removed the code that used the 'cookie' variable +but left the declaration in place. Remove it. + +Fixes: 962f88b9c916 ("drm/panthor: Drop the dev_enter/exit() sections in _irq_suspend/resume()") +Cc: Boris Brezillon +Cc: Steven Price +Signed-off-by: Liviu Dudau +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240402215423.360341-1-liviu.dudau@arm.com +--- + drivers/gpu/drm/panthor/panthor_device.h | 4 ---- + 1 file changed, 4 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index dc3e9c666b5bd4..2fdd671b38fd9b 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -323,8 +323,6 @@ static irqreturn_t panthor_ ## __name ## _irq_threaded_handler(int irq, void *da + \ + static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) \ + { \ +- int cookie; \ +- \ + pirq->mask = 0; \ + gpu_write(pirq->ptdev, __reg_prefix ## _INT_MASK, 0); \ + synchronize_irq(pirq->irq); \ +@@ -333,8 +331,6 @@ static inline void panthor_ ## __name ## _irq_suspend(struct panthor_irq *pirq) + \ + static inline void panthor_ ## __name ## _irq_resume(struct panthor_irq *pirq, u32 mask) \ + { \ +- int cookie; \ +- \ + atomic_set(&pirq->suspended, false); \ + pirq->mask = mask; \ + gpu_write(pirq->ptdev, __reg_prefix ## _INT_CLEAR, mask); \ + +From 0a97771c5f5aae3cbd9955768051afd466f80c8a Mon Sep 17 00:00:00 2001 +From: Liviu Dudau +Date: Tue, 2 Apr 2024 22:54:23 +0100 +Subject: [PATCH 117/150] drm/panthor: Fix some kerneldoc warnings + +When compiling with W=1 the build process will flag empty comments, +misnamed documented variables and incorrect tagging of functions. +Fix them in one go. + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Cc: Boris Brezillon +Cc: Steven Price +Signed-off-by: Liviu Dudau +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240402215423.360341-2-liviu.dudau@arm.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 8 +++----- + 1 file changed, 3 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index e5a710f190d2a8..a787f2fea33e98 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -517,7 +517,7 @@ struct panthor_group { + /** @max_compute_cores: Maximum number of shader cores used for compute jobs. */ + u8 max_compute_cores; + +- /** @max_compute_cores: Maximum number of shader cores used for fragment jobs. */ ++ /** @max_fragment_cores: Maximum number of shader cores used for fragment jobs. */ + u8 max_fragment_cores; + + /** @max_tiler_cores: Maximum number of tiler cores used for tiler jobs. */ +@@ -993,7 +993,7 @@ cs_slot_prog_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs_id) + } + + /** +- * @cs_slot_reset_locked() - Reset a queue slot ++ * cs_slot_reset_locked() - Reset a queue slot + * @ptdev: Device. + * @csg_id: Group slot. + * @cs_id: Queue slot. +@@ -1591,7 +1591,7 @@ static void sched_process_idle_event_locked(struct panthor_device *ptdev) + } + + /** +- * panthor_sched_process_global_irq() - Process the scheduling part of a global IRQ ++ * sched_process_global_irq_locked() - Process the scheduling part of a global IRQ + * @ptdev: Device. + */ + static void sched_process_global_irq_locked(struct panthor_device *ptdev) +@@ -1660,8 +1660,6 @@ static const struct dma_fence_ops panthor_queue_fence_ops = { + .get_timeline_name = queue_fence_get_timeline_name, + }; + +-/** +- */ + struct panthor_csg_slots_upd_ctx { + u32 update_mask; + u32 timedout_mask; + +From 8e49026441a03e2c9832b135b83750874ca97609 Mon Sep 17 00:00:00 2001 +From: Dan Carpenter +Date: Tue, 2 Apr 2024 12:58:09 +0300 +Subject: [PATCH 118/150] drm/panthor: Fix a couple -ENOMEM error codes + +These error paths forgot to set the error code to -ENOMEM. + +Fixes: 647810ec2476 ("drm/panthor: Add the MMU/VM logical block") +Signed-off-by: Dan Carpenter +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/cf5bbba5-427e-4940-b91e-925f9fa71f8d@moroto.mountain +--- + drivers/gpu/drm/panthor/panthor_mmu.c | 8 ++++++-- + 1 file changed, 6 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 1ca4e6adc99aa5..1dcd271236dbba 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -1263,8 +1263,10 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, + op_ctx->rsvd_page_tables.pages = kcalloc(pt_count, + sizeof(*op_ctx->rsvd_page_tables.pages), + GFP_KERNEL); +- if (!op_ctx->rsvd_page_tables.pages) ++ if (!op_ctx->rsvd_page_tables.pages) { ++ ret = -ENOMEM; + goto err_cleanup; ++ } + + ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, + op_ctx->rsvd_page_tables.pages); +@@ -1317,8 +1319,10 @@ static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx, + op_ctx->rsvd_page_tables.pages = kcalloc(pt_count, + sizeof(*op_ctx->rsvd_page_tables.pages), + GFP_KERNEL); +- if (!op_ctx->rsvd_page_tables.pages) ++ if (!op_ctx->rsvd_page_tables.pages) { ++ ret = -ENOMEM; + goto err_cleanup; ++ } + + ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count, + op_ctx->rsvd_page_tables.pages); + +From dea4d585f4a5166f176ce567b052ce0cec87608f Mon Sep 17 00:00:00 2001 +From: Dan Carpenter +Date: Tue, 2 Apr 2024 12:56:19 +0300 +Subject: [PATCH 119/150] drm/panthor: Fix error code in panthor_gpu_init() + +This code accidentally returns zero/success on error because of a typo. +It should be "irq" instead of "ret". The other thing is that if +platform_get_irq_byname() were to return zero then the error code would +be cmplicated. Fortunately, it does not so we can just change <= to +< 0. + +Fixes: 5cd894e258c4 ("drm/panthor: Add the GPU logical block") +Signed-off-by: Dan Carpenter +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/d753e684-43ee-45c2-a1fd-86222da204e1@moroto.mountain +--- + drivers/gpu/drm/panthor/panthor_gpu.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c +index 0f7c962440d3e2..5251d8764e7d7a 100644 +--- a/drivers/gpu/drm/panthor/panthor_gpu.c ++++ b/drivers/gpu/drm/panthor/panthor_gpu.c +@@ -211,8 +211,8 @@ int panthor_gpu_init(struct panthor_device *ptdev) + return ret; + + irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "gpu"); +- if (irq <= 0) +- return ret; ++ if (irq < 0) ++ return irq; + + ret = panthor_request_gpu_irq(ptdev, &ptdev->gpu->irq, irq, GPU_INTERRUPTS_MASK); + if (ret) + +From 073236263aaa4abfb277089cff8223b21df782a6 Mon Sep 17 00:00:00 2001 +From: Dan Carpenter +Date: Tue, 2 Apr 2024 12:56:42 +0300 +Subject: [PATCH 120/150] drm/panthor: Fix off by one in + panthor_fw_get_cs_iface() + +The ->iface.streams[csg_slot][] array has MAX_CS_PER_CSG elements so +this > comparison needs to be >= to prevent an out of bounds access. + +Fixes: 2718d91816ee ("drm/panthor: Add the FW logical block") +Signed-off-by: Dan Carpenter +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/62835c16-c85c-483d-a8fe-63be78d49d15@moroto.mountain +--- + drivers/gpu/drm/panthor/panthor_fw.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index 33c87a59834ecf..181395e2859a96 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -308,7 +308,7 @@ panthor_fw_get_csg_iface(struct panthor_device *ptdev, u32 csg_slot) + struct panthor_fw_cs_iface * + panthor_fw_get_cs_iface(struct panthor_device *ptdev, u32 csg_slot, u32 cs_slot) + { +- if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS || cs_slot > MAX_CS_PER_CSG)) ++ if (drm_WARN_ON(&ptdev->base, csg_slot >= MAX_CSGS || cs_slot >= MAX_CS_PER_CSG)) + return NULL; + + return &ptdev->fw->iface.streams[csg_slot][cs_slot]; + +From cc123011bafbe336057316dbe42ad4145b8e5370 Mon Sep 17 00:00:00 2001 +From: Harshit Mogalapalli +Date: Tue, 2 Apr 2024 03:40:40 -0700 +Subject: [PATCH 121/150] drm/panthor: Fix NULL vs IS_ERR() bug in + panthor_probe() + +The devm_drm_dev_alloc() function returns error pointers. +Update the error handling to check for error pointers instead of NULL. + +Fixes: 4bdca1150792 ("drm/panthor: Add the driver frontend block") +Signed-off-by: Harshit Mogalapalli +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240402104041.1689951-1-harshit.m.mogalapalli@oracle.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index 455a3152ce6440..9ceef8536e6668 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -1389,7 +1389,7 @@ static int panthor_probe(struct platform_device *pdev) + + ptdev = devm_drm_dev_alloc(&pdev->dev, &panthor_drm_driver, + struct panthor_device, base); +- if (!ptdev) ++ if (IS_ERR(ptdev)) + return -ENOMEM; + + platform_set_drvdata(pdev, ptdev); + +From 761f17c99a254cbc38cdee7e7c4a941aa08adda8 Mon Sep 17 00:00:00 2001 +From: Harshit Mogalapalli +Date: Tue, 2 Apr 2024 07:14:11 -0700 +Subject: [PATCH 122/150] drm/panthor: Don't return NULL from + panthor_vm_get_heap_pool() + +The kernel doc says this function returns either a valid pointer +or an ERR_PTR(), but in practice this function can return NULL if +create=false. Fix the function to match the doc (return +ERR_PTR(-ENOENT) instead of NULL) and adjust all call-sites +accordingly. + +Fixes: 4bdca1150792 ("drm/panthor: Add the driver frontend block") +Signed-off-by: Harshit Mogalapalli +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240402141412.1707949-1-harshit.m.mogalapalli@oracle.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 4 ++-- + drivers/gpu/drm/panthor/panthor_mmu.c | 2 ++ + drivers/gpu/drm/panthor/panthor_sched.c | 2 +- + 3 files changed, 5 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index 9ceef8536e6668..929fea7977e250 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -1089,8 +1089,8 @@ static int panthor_ioctl_tiler_heap_destroy(struct drm_device *ddev, void *data, + return -EINVAL; + + pool = panthor_vm_get_heap_pool(vm, false); +- if (!pool) { +- ret = -EINVAL; ++ if (IS_ERR(pool)) { ++ ret = PTR_ERR(pool); + goto out_put_vm; + } + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 1dcd271236dbba..15be29ff46c391 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -1896,6 +1896,8 @@ struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool c + vm->heaps.pool = panthor_heap_pool_get(pool); + } else { + pool = panthor_heap_pool_get(vm->heaps.pool); ++ if (!pool) ++ pool = ERR_PTR(-ENOENT); + } + mutex_unlock(&vm->heaps.lock); + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index a787f2fea33e98..d4bc652b34d5df 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -1343,7 +1343,7 @@ static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) + if (unlikely(csg_id < 0)) + return 0; + +- if (!heaps || frag_end > vt_end || vt_end >= vt_start) { ++ if (IS_ERR(heaps) || frag_end > vt_end || vt_end >= vt_start) { + ret = -EINVAL; + } else { + /* We do the allocation without holding the scheduler lock to avoid + +From 7484a3cf2a82299177f5fb6e8837363422fd32d3 Mon Sep 17 00:00:00 2001 +From: Dan Carpenter +Date: Mon, 8 Apr 2024 10:36:35 +0300 +Subject: [PATCH 123/150] drm/panthor: clean up some types in + panthor_sched_suspend() + +These variables should be u32 instead of u64 because they're only +storing u32 values. Also static checkers complain when we do: + + suspended_slots &= ~upd_ctx.timedout_mask; + +In this code "suspended_slots" is a u64 and "upd_ctx.timedout_mask". The +mask clears out the top 32 bits which would likely be a bug if anything +were stored there. + +Signed-off-by: Dan Carpenter +Reviewed-by: Steven Price +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/85356b15-4840-4e64-8c75-922cdd6a5fef@moroto.mountain +--- + drivers/gpu/drm/panthor/panthor_sched.c | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index d4bc652b34d5df..b3a51a6de5235e 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2546,7 +2546,7 @@ void panthor_sched_suspend(struct panthor_device *ptdev) + { + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slots_upd_ctx upd_ctx; +- u64 suspended_slots, faulty_slots; ++ u32 suspended_slots, faulty_slots; + struct panthor_group *group; + u32 i; + + +From 98f47fd7ad0e6063c8117b73ac982f3d331f9f4c Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 25 Apr 2024 12:39:20 +0200 +Subject: [PATCH 124/150] drm/panthor: Kill the faulty_slots variable in + panthor_sched_suspend() + +We can use upd_ctx.timedout_mask directly, and the faulty_slots update +in the flush_caches_failed situation is never used. + +Suggested-by: Suggested-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240425103920.826458-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 10 +++------- + 1 file changed, 3 insertions(+), 7 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index b3a51a6de5235e..8ec66dd47db3de 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2546,8 +2546,8 @@ void panthor_sched_suspend(struct panthor_device *ptdev) + { + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_csg_slots_upd_ctx upd_ctx; +- u32 suspended_slots, faulty_slots; + struct panthor_group *group; ++ u32 suspended_slots; + u32 i; + + mutex_lock(&sched->lock); +@@ -2566,10 +2566,9 @@ void panthor_sched_suspend(struct panthor_device *ptdev) + + csgs_upd_ctx_apply_locked(ptdev, &upd_ctx); + suspended_slots &= ~upd_ctx.timedout_mask; +- faulty_slots = upd_ctx.timedout_mask; + +- if (faulty_slots) { +- u32 slot_mask = faulty_slots; ++ if (upd_ctx.timedout_mask) { ++ u32 slot_mask = upd_ctx.timedout_mask; + + drm_err(&ptdev->base, "CSG suspend failed, escalating to termination"); + csgs_upd_ctx_init(&upd_ctx); +@@ -2620,9 +2619,6 @@ void panthor_sched_suspend(struct panthor_device *ptdev) + + slot_mask &= ~BIT(csg_id); + } +- +- if (flush_caches_failed) +- faulty_slots |= suspended_slots; + } + + for (i = 0; i < sched->csg_slot_count; i++) { + +From f8d68e5721e8e8ce552307fc952dd2d699ed4d87 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 17:52:48 +0200 +Subject: [PATCH 125/150] drm/panthor: Make sure we handle 'unknown group + state' case properly + +When we check for state values returned by the FW, we only cover part of +the 0:7 range. Make sure we catch FW inconsistencies by adding a default +to the switch statement, and flagging the group state as unknown in that +case. + +When an unknown state is detected, we trigger a reset, and consider the +group as unusable after that point, to prevent the potential corruption +from creeping in other places if we continue executing stuff on this +context. + +v2: +- Add Steve's R-b +- Fix commit message + +Reported-by: Dan Carpenter +Closes: https://lore.kernel.org/dri-devel/3b7fd2f2-679e-440c-81cd-42fc2573b515@moroto.mountain/T/#u +Suggested-by: Steven Price +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502155248.1430582-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 37 +++++++++++++++++++++++-- + 1 file changed, 35 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 8ec66dd47db3de..7f16a4a14e9af4 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -490,6 +490,18 @@ enum panthor_group_state { + * Can no longer be scheduled. The only allowed action is a destruction. + */ + PANTHOR_CS_GROUP_TERMINATED, ++ ++ /** ++ * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state. ++ * ++ * The FW returned an inconsistent state. The group is flagged unusable ++ * and can no longer be scheduled. The only allowed action is a ++ * destruction. ++ * ++ * When that happens, we also schedule a FW reset, to start from a fresh ++ * state. ++ */ ++ PANTHOR_CS_GROUP_UNKNOWN_STATE, + }; + + /** +@@ -1127,6 +1139,7 @@ csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) + struct panthor_fw_csg_iface *csg_iface; + struct panthor_group *group; + enum panthor_group_state new_state, old_state; ++ u32 csg_state; + + lockdep_assert_held(&ptdev->scheduler->lock); + +@@ -1137,7 +1150,8 @@ csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) + return; + + old_state = group->state; +- switch (csg_iface->output->ack & CSG_STATE_MASK) { ++ csg_state = csg_iface->output->ack & CSG_STATE_MASK; ++ switch (csg_state) { + case CSG_STATE_START: + case CSG_STATE_RESUME: + new_state = PANTHOR_CS_GROUP_ACTIVE; +@@ -1148,11 +1162,28 @@ csg_slot_sync_state_locked(struct panthor_device *ptdev, u32 csg_id) + case CSG_STATE_SUSPEND: + new_state = PANTHOR_CS_GROUP_SUSPENDED; + break; ++ default: ++ /* The unknown state might be caused by a FW state corruption, ++ * which means the group metadata can't be trusted anymore, and ++ * the SUSPEND operation might propagate the corruption to the ++ * suspend buffers. Flag the group state as unknown to make ++ * sure it's unusable after that point. ++ */ ++ drm_err(&ptdev->base, "Invalid state on CSG %d (state=%d)", ++ csg_id, csg_state); ++ new_state = PANTHOR_CS_GROUP_UNKNOWN_STATE; ++ break; + } + + if (old_state == new_state) + return; + ++ /* The unknown state might be caused by a FW issue, reset the FW to ++ * take a fresh start. ++ */ ++ if (new_state == PANTHOR_CS_GROUP_UNKNOWN_STATE) ++ panthor_device_schedule_reset(ptdev); ++ + if (new_state == PANTHOR_CS_GROUP_SUSPENDED) + csg_slot_sync_queues_state_locked(ptdev, csg_id); + +@@ -1783,6 +1814,7 @@ static bool + group_can_run(struct panthor_group *group) + { + return group->state != PANTHOR_CS_GROUP_TERMINATED && ++ group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && + !group->destroyed && group->fatal_queues == 0 && + !group->timedout; + } +@@ -2557,7 +2589,8 @@ void panthor_sched_suspend(struct panthor_device *ptdev) + + if (csg_slot->group) { + csgs_upd_ctx_queue_reqs(ptdev, &upd_ctx, i, +- CSG_STATE_SUSPEND, ++ group_can_run(csg_slot->group) ? ++ CSG_STATE_SUSPEND : CSG_STATE_TERMINATE, + CSG_STATE_MASK); + } + } + +From 4cac02edd09edda23fc86691b9e9afe2e02f99b6 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Tue, 30 Apr 2024 13:37:27 +0200 +Subject: [PATCH 126/150] drm/panthor: Fix the FW reset logic + +In the post_reset function, if the fast reset didn't succeed, we +are not clearing the fast_reset flag, which prevents firmware +sections from being reloaded. While at it, use panthor_fw_stop() +instead of manually writing DISABLE to the MCU_CONTROL register. + +Fixes: 2718d91816ee ("drm/panthor: Add the FW logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240430113727.493155-1-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 7 ++++--- + 1 file changed, 4 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index 181395e2859a96..fedf9627453fb1 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -1083,10 +1083,11 @@ int panthor_fw_post_reset(struct panthor_device *ptdev) + if (!ret) + goto out; + +- /* Force a disable, so we get a fresh boot on the next +- * panthor_fw_start() call. ++ /* Forcibly reset the MCU and force a slow reset, so we get a ++ * fresh boot on the next panthor_fw_start() call. + */ +- gpu_write(ptdev, MCU_CONTROL, MCU_CONTROL_DISABLE); ++ panthor_fw_stop(ptdev); ++ ptdev->fw->fast_reset = false; + drm_err(&ptdev->base, "FW fast reset failed, trying a slow reset"); + } + + +From 67d31a24afb77606b928737be6cc37140a33988c Mon Sep 17 00:00:00 2001 +From: Antonino Maniscalco +Date: Thu, 2 May 2024 18:51:54 +0200 +Subject: [PATCH 127/150] drm/panthor: Fix tiler OOM handling to allow + incremental rendering + +If the kernel couldn't allocate memory because we reached the maximum +number of chunks but no render passes are in flight +(panthor_heap_grow() returning -ENOMEM), we should defer the OOM +handling to the FW by returning a NULL chunk. The FW will then call +the tiler OOM exception handler, which is supposed to implement +incremental rendering (execute an intermediate fragment job to flush +the pending primitives, release the tiler memory that was used to +store those primitives, and start over from where it stopped). + +Instead of checking for both ENOMEM and EBUSY, make panthor_heap_grow() +return ENOMEM no matter the reason of this allocation failure, the FW +doesn't care anyway. + +v3: +- Add R-bs + +v2: +- Make panthor_heap_grow() return -ENOMEM for all kind of allocation + failures +- Document the panthor_heap_grow() semantics + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Signed-off-by: Antonino Maniscalco +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240502165158.1458959-2-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_heap.c | 12 ++++++++---- + drivers/gpu/drm/panthor/panthor_sched.c | 7 ++++++- + 2 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +index 143fa35f2e746c..c3c0ba744937da 100644 +--- a/drivers/gpu/drm/panthor/panthor_heap.c ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -410,6 +410,13 @@ int panthor_heap_return_chunk(struct panthor_heap_pool *pool, + * @renderpasses_in_flight: Number of render passes currently in-flight. + * @pending_frag_count: Number of fragment jobs waiting for execution/completion. + * @new_chunk_gpu_va: Pointer used to return the chunk VA. ++ * ++ * Return: ++ * - 0 if a new heap was allocated ++ * - -ENOMEM if the tiler context reached the maximum number of chunks ++ * or if too many render passes are in-flight ++ * or if the allocation failed ++ * - -EINVAL if any of the arguments passed to panthor_heap_grow() is invalid + */ + int panthor_heap_grow(struct panthor_heap_pool *pool, + u64 heap_gpu_va, +@@ -439,10 +446,7 @@ int panthor_heap_grow(struct panthor_heap_pool *pool, + * handler provided by the userspace driver, if any). + */ + if (renderpasses_in_flight > heap->target_in_flight || +- (pending_frag_count > 0 && heap->chunk_count >= heap->max_chunks)) { +- ret = -EBUSY; +- goto out_unlock; +- } else if (heap->chunk_count >= heap->max_chunks) { ++ heap->chunk_count >= heap->max_chunks) { + ret = -ENOMEM; + goto out_unlock; + } +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 7f16a4a14e9af4..c126251c5ba7e1 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -1385,7 +1385,12 @@ static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) + pending_frag_count, &new_chunk_va); + } + +- if (ret && ret != -EBUSY) { ++ /* If the heap context doesn't have memory for us, we want to let the ++ * FW try to reclaim memory by waiting for fragment jobs to land or by ++ * executing the tiler OOM exception handler, which is supposed to ++ * implement incremental rendering. ++ */ ++ if (ret && ret != -ENOMEM) { + drm_warn(&ptdev->base, "Failed to extend the tiler heap\n"); + group->fatal_queues |= BIT(cs_id); + sched_queue_delayed_work(sched, tick, 0); + +From a58cdbe71a86ae1faa77434763ab259bb58bfe57 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 18:51:55 +0200 +Subject: [PATCH 128/150] drm/panthor: Make sure the tiler initial/max chunks + are consistent + +It doesn't make sense to have a maximum number of chunks smaller than +the initial number of chunks attached to the context. + +Fix the uAPI header to reflect the new constraint, and mention the +undocumented "initial_chunk_count > 0" constraint while at it. + +v3: +- Add R-b + +v2: +- Fix the check + +Fixes: 9cca48fa4f89 ("drm/panthor: Add the heap logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240502165158.1458959-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_heap.c | 3 +++ + include/uapi/drm/panthor_drm.h | 8 ++++++-- + 2 files changed, 9 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +index c3c0ba744937da..3be86ec383d6b6 100644 +--- a/drivers/gpu/drm/panthor/panthor_heap.c ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -281,6 +281,9 @@ int panthor_heap_create(struct panthor_heap_pool *pool, + if (initial_chunk_count == 0) + return -EINVAL; + ++ if (initial_chunk_count > max_chunks) ++ return -EINVAL; ++ + if (hweight32(chunk_size) != 1 || + chunk_size < SZ_256K || chunk_size > SZ_2M) + return -EINVAL; +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 373df80f41edf6..9d874ace4fc3c3 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -895,13 +895,17 @@ struct drm_panthor_tiler_heap_create { + /** @vm_id: VM ID the tiler heap should be mapped to */ + __u32 vm_id; + +- /** @initial_chunk_count: Initial number of chunks to allocate. */ ++ /** @initial_chunk_count: Initial number of chunks to allocate. Must be at least one. */ + __u32 initial_chunk_count; + + /** @chunk_size: Chunk size. Must be a power of two at least 256KB large. */ + __u32 chunk_size; + +- /** @max_chunks: Maximum number of chunks that can be allocated. */ ++ /** ++ * @max_chunks: Maximum number of chunks that can be allocated. ++ * ++ * Must be at least @initial_chunk_count. ++ */ + __u32 max_chunks; + + /** + +From 180bb4b16099248e7a4dd919b14e8a0391bbdb44 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 18:51:56 +0200 +Subject: [PATCH 129/150] drm/panthor: Relax the constraints on the tiler chunk + size + +The field used to store the chunk size if 12 bits wide, and the encoding +is chunk_size = chunk_header.chunk_size << 12, which gives us a +theoretical [4k:8M] range. This range is further limited by +implementation constraints, and all known implementations seem to +impose a [128k:8M] range, so do the same here. + +We also relax the power-of-two constraint, which doesn't seem to +exist on v10. This will allow userspace to fine-tune initial/max +tiler memory on memory-constrained devices. + +v4: +- Actually fix the range in the kerneldoc + +v3: +- Add R-bs +- Fix valid range in the kerneldoc + +v2: +- Turn the power-of-two constraint into a page-aligned constraint to allow + fine-tune of the initial/max heap memory size +- Fix the panthor_heap_create() kerneldoc + +Fixes: 9cca48fa4f89 ("drm/panthor: Add the heap logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240502165158.1458959-4-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_heap.c | 8 ++++---- + include/uapi/drm/panthor_drm.h | 6 +++++- + 2 files changed, 9 insertions(+), 5 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +index 3be86ec383d6b6..b0fc5b9ee8479d 100644 +--- a/drivers/gpu/drm/panthor/panthor_heap.c ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -253,8 +253,8 @@ int panthor_heap_destroy(struct panthor_heap_pool *pool, u32 handle) + * @pool: Pool to instantiate the heap context from. + * @initial_chunk_count: Number of chunk allocated at initialization time. + * Must be at least 1. +- * @chunk_size: The size of each chunk. Must be a power of two between 256k +- * and 2M. ++ * @chunk_size: The size of each chunk. Must be page-aligned and lie in the ++ * [128k:8M] range. + * @max_chunks: Maximum number of chunks that can be allocated. + * @target_in_flight: Maximum number of in-flight render passes. + * @heap_ctx_gpu_va: Pointer holding the GPU address of the allocated heap +@@ -284,8 +284,8 @@ int panthor_heap_create(struct panthor_heap_pool *pool, + if (initial_chunk_count > max_chunks) + return -EINVAL; + +- if (hweight32(chunk_size) != 1 || +- chunk_size < SZ_256K || chunk_size > SZ_2M) ++ if (!IS_ALIGNED(chunk_size, PAGE_SIZE) || ++ chunk_size < SZ_128K || chunk_size > SZ_8M) + return -EINVAL; + + down_read(&pool->lock); +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 9d874ace4fc3c3..4e1fd1212454df 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -898,7 +898,11 @@ struct drm_panthor_tiler_heap_create { + /** @initial_chunk_count: Initial number of chunks to allocate. Must be at least one. */ + __u32 initial_chunk_count; + +- /** @chunk_size: Chunk size. Must be a power of two at least 256KB large. */ ++ /** ++ * @chunk_size: Chunk size. ++ * ++ * Must be page-aligned and lie in the [128k:8M] range. ++ */ + __u32 chunk_size; + + /** + +From 1ecc20aef60d4e5baa92ce451578e40573c04015 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 18:51:57 +0200 +Subject: [PATCH 130/150] drm/panthor: Fix an off-by-one in the heap context + retrieval logic + +The heap ID is used to index the heap context pool, and allocating +in the [1:MAX_HEAPS_PER_POOL] leads to an off-by-one. This was +originally to avoid returning a zero heap handle, but given the handle +is formed with (vm_id << 16) | heap_id, with vm_id > 0, we already can't +end up with a valid heap handle that's zero. + +v4: +- s/XA_FLAGS_ALLOC1/XA_FLAGS_ALLOC/ + +v3: +- Allocate in the [0:MAX_HEAPS_PER_POOL-1] range + +v2: +- New patch + +Fixes: 9cca48fa4f89 ("drm/panthor: Add the heap logical block") +Reported-by: Eric Smith +Signed-off-by: Boris Brezillon +Tested-by: Eric Smith +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502165158.1458959-5-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_heap.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +index b0fc5b9ee8479d..95a1c6c9f35e03 100644 +--- a/drivers/gpu/drm/panthor/panthor_heap.c ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -323,7 +323,8 @@ int panthor_heap_create(struct panthor_heap_pool *pool, + if (!pool->vm) { + ret = -EINVAL; + } else { +- ret = xa_alloc(&pool->xa, &id, heap, XA_LIMIT(1, MAX_HEAPS_PER_POOL), GFP_KERNEL); ++ ret = xa_alloc(&pool->xa, &id, heap, ++ XA_LIMIT(0, MAX_HEAPS_PER_POOL - 1), GFP_KERNEL); + if (!ret) { + void *gpu_ctx = panthor_get_heap_ctx(pool, id); + +@@ -543,7 +544,7 @@ panthor_heap_pool_create(struct panthor_device *ptdev, struct panthor_vm *vm) + pool->vm = vm; + pool->ptdev = ptdev; + init_rwsem(&pool->lock); +- xa_init_flags(&pool->xa, XA_FLAGS_ALLOC1); ++ xa_init_flags(&pool->xa, XA_FLAGS_ALLOC); + kref_init(&pool->refcount); + + pool->gpu_contexts = panthor_kernel_bo_create(ptdev, vm, bosize, + +From 06071646a57879558ee2cb4247d2e24fe58e8a3e Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 20:38:09 +0200 +Subject: [PATCH 131/150] drm/panthor: Force an immediate reset on + unrecoverable faults + +If the FW reports an unrecoverable fault, we need to reset the GPU +before we can start re-using it again. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502183813.1612017-2-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 1 + + drivers/gpu/drm/panthor/panthor_device.h | 1 + + drivers/gpu/drm/panthor/panthor_sched.c | 11 ++++++++++- + 3 files changed, 12 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index e8a42e74e693c2..03aa42119c9f80 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -293,6 +293,7 @@ static const struct panthor_exception_info panthor_exception_infos[] = { + PANTHOR_EXCEPTION(ACTIVE), + PANTHOR_EXCEPTION(CS_RES_TERM), + PANTHOR_EXCEPTION(CS_CONFIG_FAULT), ++ PANTHOR_EXCEPTION(CS_UNRECOVERABLE), + PANTHOR_EXCEPTION(CS_ENDPOINT_FAULT), + PANTHOR_EXCEPTION(CS_BUS_FAULT), + PANTHOR_EXCEPTION(CS_INSTR_INVALID), +diff --git a/drivers/gpu/drm/panthor/panthor_device.h b/drivers/gpu/drm/panthor/panthor_device.h +index 2fdd671b38fd9b..e388c0472ba783 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.h ++++ b/drivers/gpu/drm/panthor/panthor_device.h +@@ -216,6 +216,7 @@ enum drm_panthor_exception_type { + DRM_PANTHOR_EXCEPTION_CS_RES_TERM = 0x0f, + DRM_PANTHOR_EXCEPTION_MAX_NON_FAULT = 0x3f, + DRM_PANTHOR_EXCEPTION_CS_CONFIG_FAULT = 0x40, ++ DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE = 0x41, + DRM_PANTHOR_EXCEPTION_CS_ENDPOINT_FAULT = 0x44, + DRM_PANTHOR_EXCEPTION_CS_BUS_FAULT = 0x48, + DRM_PANTHOR_EXCEPTION_CS_INSTR_INVALID = 0x49, +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index c126251c5ba7e1..e455e8445582a1 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -1281,7 +1281,16 @@ cs_slot_process_fatal_event_locked(struct panthor_device *ptdev, + if (group) + group->fatal_queues |= BIT(cs_id); + +- sched_queue_delayed_work(sched, tick, 0); ++ if (CS_EXCEPTION_TYPE(fatal) == DRM_PANTHOR_EXCEPTION_CS_UNRECOVERABLE) { ++ /* If this exception is unrecoverable, queue a reset, and make ++ * sure we stop scheduling groups until the reset has happened. ++ */ ++ panthor_device_schedule_reset(ptdev); ++ cancel_delayed_work(&sched->tick_work); ++ } else { ++ sched_queue_delayed_work(sched, tick, 0); ++ } ++ + drm_warn(&ptdev->base, + "CSG slot %d CS slot: %d\n" + "CS_FATAL.EXCEPTION_TYPE: 0x%x (%s)\n" + +From cb242e5098aa083d99dea46c3c32e968042d7eb6 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 20:38:10 +0200 +Subject: [PATCH 132/150] drm/panthor: Keep a ref to the VM at the + panthor_kernel_bo level + +Avoids use-after-free situations when panthor_fw_unplug() is called +and the kernel BO was mapped to the FW VM. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502183813.1612017-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 4 ++-- + drivers/gpu/drm/panthor/panthor_gem.c | 8 +++++--- + drivers/gpu/drm/panthor/panthor_gem.h | 8 ++++++-- + drivers/gpu/drm/panthor/panthor_heap.c | 8 ++++---- + drivers/gpu/drm/panthor/panthor_sched.c | 11 +++++------ + 5 files changed, 22 insertions(+), 17 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index fedf9627453fb1..394e00bd75bb7a 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -453,7 +453,7 @@ panthor_fw_alloc_queue_iface_mem(struct panthor_device *ptdev, + + ret = panthor_kernel_bo_vmap(mem); + if (ret) { +- panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), mem); ++ panthor_kernel_bo_destroy(mem); + return ERR_PTR(ret); + } + +@@ -1134,7 +1134,7 @@ void panthor_fw_unplug(struct panthor_device *ptdev) + panthor_fw_stop(ptdev); + + list_for_each_entry(section, &ptdev->fw->sections, node) +- panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), section->mem); ++ panthor_kernel_bo_destroy(section->mem); + + /* We intentionally don't call panthor_vm_idle() and let + * panthor_mmu_unplug() release the AS we acquired with +diff --git a/drivers/gpu/drm/panthor/panthor_gem.c b/drivers/gpu/drm/panthor/panthor_gem.c +index d6483266d0c202..38f560864879c5 100644 +--- a/drivers/gpu/drm/panthor/panthor_gem.c ++++ b/drivers/gpu/drm/panthor/panthor_gem.c +@@ -26,18 +26,18 @@ static void panthor_gem_free_object(struct drm_gem_object *obj) + + /** + * panthor_kernel_bo_destroy() - Destroy a kernel buffer object +- * @vm: The VM this BO was mapped to. + * @bo: Kernel buffer object to destroy. If NULL or an ERR_PTR(), the destruction + * is skipped. + */ +-void panthor_kernel_bo_destroy(struct panthor_vm *vm, +- struct panthor_kernel_bo *bo) ++void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo) + { ++ struct panthor_vm *vm; + int ret; + + if (IS_ERR_OR_NULL(bo)) + return; + ++ vm = bo->vm; + panthor_kernel_bo_vunmap(bo); + + if (drm_WARN_ON(bo->obj->dev, +@@ -53,6 +53,7 @@ void panthor_kernel_bo_destroy(struct panthor_vm *vm, + drm_gem_object_put(bo->obj); + + out_free_bo: ++ panthor_vm_put(vm); + kfree(bo); + } + +@@ -106,6 +107,7 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, + if (ret) + goto err_free_va; + ++ kbo->vm = panthor_vm_get(vm); + bo->exclusive_vm_root_gem = panthor_vm_root_gem(vm); + drm_gem_object_get(bo->exclusive_vm_root_gem); + bo->base.base.resv = bo->exclusive_vm_root_gem->resv; +diff --git a/drivers/gpu/drm/panthor/panthor_gem.h b/drivers/gpu/drm/panthor/panthor_gem.h +index 3bccba394d00a4..e43021cf6d4587 100644 +--- a/drivers/gpu/drm/panthor/panthor_gem.h ++++ b/drivers/gpu/drm/panthor/panthor_gem.h +@@ -61,6 +61,11 @@ struct panthor_kernel_bo { + */ + struct drm_gem_object *obj; + ++ /** ++ * @vm: VM this private buffer is attached to. ++ */ ++ struct panthor_vm *vm; ++ + /** + * @va_node: VA space allocated to this GEM. + */ +@@ -136,7 +141,6 @@ panthor_kernel_bo_create(struct panthor_device *ptdev, struct panthor_vm *vm, + size_t size, u32 bo_flags, u32 vm_map_flags, + u64 gpu_va); + +-void panthor_kernel_bo_destroy(struct panthor_vm *vm, +- struct panthor_kernel_bo *bo); ++void panthor_kernel_bo_destroy(struct panthor_kernel_bo *bo); + + #endif /* __PANTHOR_GEM_H__ */ +diff --git a/drivers/gpu/drm/panthor/panthor_heap.c b/drivers/gpu/drm/panthor/panthor_heap.c +index 95a1c6c9f35e03..3796a9eb22af2f 100644 +--- a/drivers/gpu/drm/panthor/panthor_heap.c ++++ b/drivers/gpu/drm/panthor/panthor_heap.c +@@ -127,7 +127,7 @@ static void panthor_free_heap_chunk(struct panthor_vm *vm, + heap->chunk_count--; + mutex_unlock(&heap->lock); + +- panthor_kernel_bo_destroy(vm, chunk->bo); ++ panthor_kernel_bo_destroy(chunk->bo); + kfree(chunk); + } + +@@ -183,7 +183,7 @@ static int panthor_alloc_heap_chunk(struct panthor_device *ptdev, + return 0; + + err_destroy_bo: +- panthor_kernel_bo_destroy(vm, chunk->bo); ++ panthor_kernel_bo_destroy(chunk->bo); + + err_free_chunk: + kfree(chunk); +@@ -395,7 +395,7 @@ int panthor_heap_return_chunk(struct panthor_heap_pool *pool, + mutex_unlock(&heap->lock); + + if (removed) { +- panthor_kernel_bo_destroy(pool->vm, chunk->bo); ++ panthor_kernel_bo_destroy(chunk->bo); + kfree(chunk); + ret = 0; + } else { +@@ -595,7 +595,7 @@ void panthor_heap_pool_destroy(struct panthor_heap_pool *pool) + drm_WARN_ON(&pool->ptdev->base, panthor_heap_destroy_locked(pool, i)); + + if (!IS_ERR_OR_NULL(pool->gpu_contexts)) +- panthor_kernel_bo_destroy(pool->vm, pool->gpu_contexts); ++ panthor_kernel_bo_destroy(pool->gpu_contexts); + + /* Reflects the fact the pool has been destroyed. */ + pool->vm = NULL; +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index e455e8445582a1..9308596e081226 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -826,8 +826,8 @@ static void group_free_queue(struct panthor_group *group, struct panthor_queue * + + panthor_queue_put_syncwait_obj(queue); + +- panthor_kernel_bo_destroy(group->vm, queue->ringbuf); +- panthor_kernel_bo_destroy(panthor_fw_vm(group->ptdev), queue->iface.mem); ++ panthor_kernel_bo_destroy(queue->ringbuf); ++ panthor_kernel_bo_destroy(queue->iface.mem); + + kfree(queue); + } +@@ -837,15 +837,14 @@ static void group_release_work(struct work_struct *work) + struct panthor_group *group = container_of(work, + struct panthor_group, + release_work); +- struct panthor_device *ptdev = group->ptdev; + u32 i; + + for (i = 0; i < group->queue_count; i++) + group_free_queue(group, group->queues[i]); + +- panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->suspend_buf); +- panthor_kernel_bo_destroy(panthor_fw_vm(ptdev), group->protm_suspend_buf); +- panthor_kernel_bo_destroy(group->vm, group->syncobjs); ++ panthor_kernel_bo_destroy(group->suspend_buf); ++ panthor_kernel_bo_destroy(group->protm_suspend_buf); ++ panthor_kernel_bo_destroy(group->syncobjs); + + panthor_vm_put(group->vm); + kfree(group); + +From b7aad7b3ecdb9575858470d7d32a5a1da07b32ad Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 20:38:11 +0200 +Subject: [PATCH 133/150] drm/panthor: Reset the FW VM to NULL on unplug + +This way get NULL derefs instead of use-after-free if the FW VM is +referenced after the device has been unplugged. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Acked-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502183813.1612017-4-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index 394e00bd75bb7a..857f3f11258aad 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -1142,6 +1142,7 @@ void panthor_fw_unplug(struct panthor_device *ptdev) + * state to keep the active_refcnt balanced. + */ + panthor_vm_put(ptdev->fw->vm); ++ ptdev->fw->vm = NULL; + + panthor_gpu_power_off(ptdev, L2, ptdev->gpu_info.l2_present, 20000); + } + +From e6bae0d04ef7a68ac622c1e416d1c216c29da87a Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 2 May 2024 20:38:12 +0200 +Subject: [PATCH 134/150] drm/panthor: Call panthor_sched_post_reset() even if + the reset failed + +We need to undo what was done in panthor_sched_pre_reset() even if the +reset failed. We just flag all previously running groups as terminated +when that happens to unblock things. + +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240502183813.1612017-5-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_device.c | 7 +------ + drivers/gpu/drm/panthor/panthor_sched.c | 19 ++++++++++++++----- + drivers/gpu/drm/panthor/panthor_sched.h | 2 +- + 3 files changed, 16 insertions(+), 12 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_device.c b/drivers/gpu/drm/panthor/panthor_device.c +index 03aa42119c9f80..e142b8388b7bd1 100644 +--- a/drivers/gpu/drm/panthor/panthor_device.c ++++ b/drivers/gpu/drm/panthor/panthor_device.c +@@ -129,13 +129,8 @@ static void panthor_device_reset_work(struct work_struct *work) + panthor_gpu_l2_power_on(ptdev); + panthor_mmu_post_reset(ptdev); + ret = panthor_fw_post_reset(ptdev); +- if (ret) +- goto out_dev_exit; +- + atomic_set(&ptdev->reset.pending, 0); +- panthor_sched_post_reset(ptdev); +- +-out_dev_exit: ++ panthor_sched_post_reset(ptdev, ret != 0); + drm_dev_exit(cookie); + + if (ret) { +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 9308596e081226..79ffcbc41d78e5 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2733,15 +2733,22 @@ void panthor_sched_pre_reset(struct panthor_device *ptdev) + mutex_unlock(&sched->reset.lock); + } + +-void panthor_sched_post_reset(struct panthor_device *ptdev) ++void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed) + { + struct panthor_scheduler *sched = ptdev->scheduler; + struct panthor_group *group, *group_tmp; + + mutex_lock(&sched->reset.lock); + +- list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) ++ list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) { ++ /* Consider all previously running group as terminated if the ++ * reset failed. ++ */ ++ if (reset_failed) ++ group->state = PANTHOR_CS_GROUP_TERMINATED; ++ + panthor_group_start(group); ++ } + + /* We're done resetting the GPU, clear the reset.in_progress bit so we can + * kick the scheduler. +@@ -2749,9 +2756,11 @@ void panthor_sched_post_reset(struct panthor_device *ptdev) + atomic_set(&sched->reset.in_progress, false); + mutex_unlock(&sched->reset.lock); + +- sched_queue_delayed_work(sched, tick, 0); +- +- sched_queue_work(sched, sync_upd); ++ /* No need to queue a tick and update syncs if the reset failed. */ ++ if (!reset_failed) { ++ sched_queue_delayed_work(sched, tick, 0); ++ sched_queue_work(sched, sync_upd); ++ } + } + + static void group_sync_upd_work(struct work_struct *work) +diff --git a/drivers/gpu/drm/panthor/panthor_sched.h b/drivers/gpu/drm/panthor/panthor_sched.h +index 66438b1f331f69..3a30d2328b308d 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.h ++++ b/drivers/gpu/drm/panthor/panthor_sched.h +@@ -40,7 +40,7 @@ void panthor_group_pool_destroy(struct panthor_file *pfile); + int panthor_sched_init(struct panthor_device *ptdev); + void panthor_sched_unplug(struct panthor_device *ptdev); + void panthor_sched_pre_reset(struct panthor_device *ptdev); +-void panthor_sched_post_reset(struct panthor_device *ptdev); ++void panthor_sched_post_reset(struct panthor_device *ptdev, bool reset_failed); + void panthor_sched_suspend(struct panthor_device *ptdev); + void panthor_sched_resume(struct panthor_device *ptdev); + + +From 1ac35425b0c62724e928667abe0e94df7e1056a1 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Wed, 3 Jul 2024 09:16:39 +0200 +Subject: [PATCH 135/150] drm/panthor: Don't check the array stride on empty + uobj arrays + +The user is likely to leave all the drm_panthor_obj_array fields +to zero when the array is empty, which will cause an EINVAL failure. + +v2: +- Added R-bs + +Fixes: 4bdca1150792 ("drm/panthor: Add the driver frontend block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240703071640.231278-2-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index 929fea7977e250..e6c50c97b9c7d8 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -85,15 +85,15 @@ panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride, + int ret = 0; + void *out_alloc; + ++ if (!in->count) ++ return NULL; ++ + /* User stride must be at least the minimum object size, otherwise it might + * lack useful information. + */ + if (in->stride < min_stride) + return ERR_PTR(-EINVAL); + +- if (!in->count) +- return NULL; +- + out_alloc = kvmalloc_array(in->count, obj_size, GFP_KERNEL); + if (!out_alloc) + return ERR_PTR(-ENOMEM); + +From 674efc93d34a4786c5a1bf66fd0db6dc60485ade Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Wed, 3 Jul 2024 09:16:40 +0200 +Subject: [PATCH 136/150] drm/panthor: Fix sync-only jobs + +A sync-only job is meant to provide a synchronization point on a +queue, so we can't return a NULL fence there, we have to add a signal +operation to the command stream which executes after all other +previously submitted jobs are done. + +v2: +- Fixed a UAF bug +- Added R-bs + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240703071640.231278-3-boris.brezillon@collabora.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 44 ++++++++++++++++++------- + include/uapi/drm/panthor_drm.h | 5 +++ + 2 files changed, 38 insertions(+), 11 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 79ffcbc41d78e5..9a0ff48f7061d7 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -458,6 +458,16 @@ struct panthor_queue { + /** @seqno: Sequence number of the last initialized fence. */ + atomic64_t seqno; + ++ /** ++ * @last_fence: Fence of the last submitted job. ++ * ++ * We return this fence when we get an empty command stream. ++ * This way, we are guaranteed that all earlier jobs have completed ++ * when drm_sched_job::s_fence::finished without having to feed ++ * the CS ring buffer with a dummy job that only signals the fence. ++ */ ++ struct dma_fence *last_fence; ++ + /** + * @in_flight_jobs: List containing all in-flight jobs. + * +@@ -829,6 +839,9 @@ static void group_free_queue(struct panthor_group *group, struct panthor_queue * + panthor_kernel_bo_destroy(queue->ringbuf); + panthor_kernel_bo_destroy(queue->iface.mem); + ++ /* Release the last_fence we were holding, if any. */ ++ dma_fence_put(queue->fence_ctx.last_fence); ++ + kfree(queue); + } + +@@ -2784,9 +2797,6 @@ static void group_sync_upd_work(struct work_struct *work) + + spin_lock(&queue->fence_ctx.lock); + list_for_each_entry_safe(job, job_tmp, &queue->fence_ctx.in_flight_jobs, node) { +- if (!job->call_info.size) +- continue; +- + if (syncobj->seqno < job->done_fence->seqno) + break; + +@@ -2865,11 +2875,14 @@ queue_run_job(struct drm_sched_job *sched_job) + static_assert(sizeof(call_instrs) % 64 == 0, + "call_instrs is not aligned on a cacheline"); + +- /* Stream size is zero, nothing to do => return a NULL fence and let +- * drm_sched signal the parent. ++ /* Stream size is zero, nothing to do except making sure all previously ++ * submitted jobs are done before we signal the ++ * drm_sched_job::s_fence::finished fence. + */ +- if (!job->call_info.size) +- return NULL; ++ if (!job->call_info.size) { ++ job->done_fence = dma_fence_get(queue->fence_ctx.last_fence); ++ return dma_fence_get(job->done_fence); ++ } + + ret = pm_runtime_resume_and_get(ptdev->base.dev); + if (drm_WARN_ON(&ptdev->base, ret)) +@@ -2928,6 +2941,10 @@ queue_run_job(struct drm_sched_job *sched_job) + } + } + ++ /* Update the last fence. */ ++ dma_fence_put(queue->fence_ctx.last_fence); ++ queue->fence_ctx.last_fence = dma_fence_get(job->done_fence); ++ + done_fence = dma_fence_get(job->done_fence); + + out_unlock: +@@ -3378,10 +3395,15 @@ panthor_job_create(struct panthor_file *pfile, + goto err_put_job; + } + +- job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL); +- if (!job->done_fence) { +- ret = -ENOMEM; +- goto err_put_job; ++ /* Empty command streams don't need a fence, they'll pick the one from ++ * the previously submitted job. ++ */ ++ if (job->call_info.size) { ++ job->done_fence = kzalloc(sizeof(*job->done_fence), GFP_KERNEL); ++ if (!job->done_fence) { ++ ret = -ENOMEM; ++ goto err_put_job; ++ } + } + + ret = drm_sched_job_init(&job->base, +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 4e1fd1212454df..2d84c5920da040 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -802,6 +802,9 @@ struct drm_panthor_queue_submit { + * Must be 64-bit/8-byte aligned (the size of a CS instruction) + * + * Can be zero if stream_addr is zero too. ++ * ++ * When the stream size is zero, the queue submit serves as a ++ * synchronization point. + */ + __u32 stream_size; + +@@ -822,6 +825,8 @@ struct drm_panthor_queue_submit { + * ensure the GPU doesn't get garbage when reading the indirect command + * stream buffers. If you want the cache flush to happen + * unconditionally, pass a zero here. ++ * ++ * Ignored when stream_size is zero. + */ + __u32 latest_flush; + + +From 8cf1e59db353401e63776b43a4e76740c435851f Mon Sep 17 00:00:00 2001 +From: Steven Price +Date: Wed, 3 Jul 2024 16:56:46 +0100 +Subject: [PATCH 137/150] drm/panthor: Record devfreq busy as soon as a job is + started + +If a queue is already assigned to the hardware, then a newly submitted +job can start straight away without waiting for the tick. However in +this case the devfreq infrastructure isn't notified that the GPU is +busy. By the time the tick happens the job might well have finished and +no time will be accounted for the GPU being busy. + +Fix this by recording the GPU as busy directly in queue_run_job() in the +case where there is a CSG assigned and therefore we just ring the +doorbell. + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Signed-off-by: Steven Price +Reviewed-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240703155646.80928-1-steven.price@arm.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 9a0ff48f7061d7..463bcd3cf00f35 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2939,6 +2939,7 @@ queue_run_job(struct drm_sched_job *sched_job) + pm_runtime_get(ptdev->base.dev); + sched->pm.has_ref = true; + } ++ panthor_devfreq_record_busy(sched->ptdev); + } + + /* Update the last fence. */ + +From 03552a9d2d903048125b1ec02b15b49fad0cb6cd Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Fri, 19 Jul 2024 17:24:14 +0200 +Subject: [PATCH 138/150] drm/scheduler: remove full_recover from + drm_sched_start +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +This was basically just another one of amdgpus hacks. The parameter +allowed to restart the scheduler without turning fence signaling on +again. + +That this is absolutely not a good idea should be obvious by now since +the fences will then just sit there and never signal. + +While at it cleanup the code a bit. + +Signed-off-by: Christian König +Reviewed-by: Matthew Brost +Link: https://patchwork.freedesktop.org/patch/msgid/20240722083816.99685-1-christian.koenig@amd.com +--- + drivers/gpu/drm/etnaviv/etnaviv_sched.c | 4 ++-- + drivers/gpu/drm/lima/lima_sched.c | 2 +- + drivers/gpu/drm/panfrost/panfrost_job.c | 2 +- + drivers/gpu/drm/panthor/panthor_mmu.c | 2 +- + drivers/gpu/drm/panthor/panthor_sched.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 25 ++++++++----------------- + drivers/gpu/drm/v3d/v3d_sched.c | 2 +- + include/drm/gpu_scheduler.h | 2 +- + 8 files changed, 16 insertions(+), 25 deletions(-) + +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +index 8d8ef85cbad57a..752fc5a21cb38b 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +@@ -71,12 +71,12 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job + + drm_sched_resubmit_jobs(&gpu->sched); + +- drm_sched_start(&gpu->sched, true); ++ drm_sched_start(&gpu->sched); + return DRM_GPU_SCHED_STAT_NOMINAL; + + out_no_timeout: + /* restart scheduler after GPU is usable again */ +- drm_sched_start(&gpu->sched, true); ++ drm_sched_start(&gpu->sched); + return DRM_GPU_SCHED_STAT_NOMINAL; + } + +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index b59269db0ffa65..8247abee957e30 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -430,7 +430,7 @@ static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job + lima_pm_idle(ldev); + + drm_sched_resubmit_jobs(&pipe->base); +- drm_sched_start(&pipe->base, true); ++ drm_sched_start(&pipe->base); + + return DRM_GPU_SCHED_STAT_NOMINAL; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c +index 045524a7695ba5..b812e0b6f2dad6 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_job.c ++++ b/drivers/gpu/drm/panfrost/panfrost_job.c +@@ -696,7 +696,7 @@ panfrost_reset(struct panfrost_device *pfdev, + + /* Restart the schedulers */ + for (i = 0; i < NUM_JOB_SLOTS; i++) +- drm_sched_start(&pfdev->js->queue[i].sched, true); ++ drm_sched_start(&pfdev->js->queue[i].sched); + + /* Re-enable job interrupts now that everything has been restarted. */ + job_write(pfdev, JOB_INT_MASK, +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 15be29ff46c391..4a1219610c58f1 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -826,7 +826,7 @@ static void panthor_vm_stop(struct panthor_vm *vm) + + static void panthor_vm_start(struct panthor_vm *vm) + { +- drm_sched_start(&vm->sched, true); ++ drm_sched_start(&vm->sched); + } + + /** +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 463bcd3cf00f35..c426a392b08117 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2538,7 +2538,7 @@ static void queue_start(struct panthor_queue *queue) + list_for_each_entry(job, &queue->scheduler.pending_list, base.list) + job->base.s_fence->parent = dma_fence_get(job->done_fence); + +- drm_sched_start(&queue->scheduler, true); ++ drm_sched_start(&queue->scheduler); + } + + static void panthor_group_stop(struct panthor_group *group) +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index 8acbef7ae53d11..e75ccf6ef90741 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -674,13 +674,11 @@ EXPORT_SYMBOL(drm_sched_stop); + * drm_sched_start - recover jobs after a reset + * + * @sched: scheduler instance +- * @full_recovery: proceed with complete sched restart + * + */ +-void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) ++void drm_sched_start(struct drm_gpu_scheduler *sched) + { + struct drm_sched_job *s_job, *tmp; +- int r; + + /* + * Locking the list is not required here as the sched thread is parked +@@ -692,24 +690,17 @@ void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery) + + atomic_add(s_job->credits, &sched->credit_count); + +- if (!full_recovery) ++ if (!fence) { ++ drm_sched_job_done(s_job, -ECANCELED); + continue; ++ } + +- if (fence) { +- r = dma_fence_add_callback(fence, &s_job->cb, +- drm_sched_job_done_cb); +- if (r == -ENOENT) +- drm_sched_job_done(s_job, fence->error); +- else if (r) +- DRM_DEV_ERROR(sched->dev, "fence add callback failed (%d)\n", +- r); +- } else +- drm_sched_job_done(s_job, -ECANCELED); ++ if (dma_fence_add_callback(fence, &s_job->cb, ++ drm_sched_job_done_cb)) ++ drm_sched_job_done(s_job, fence->error); + } + +- if (full_recovery) +- drm_sched_start_timeout_unlocked(sched); +- ++ drm_sched_start_timeout_unlocked(sched); + drm_sched_wqueue_start(sched); + } + EXPORT_SYMBOL(drm_sched_start); +diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c +index 0b6696b0d882fa..4c7be3a2cf4a3e 100644 +--- a/drivers/gpu/drm/v3d/v3d_sched.c ++++ b/drivers/gpu/drm/v3d/v3d_sched.c +@@ -274,7 +274,7 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job) + + /* Unblock schedulers and restart their jobs. */ + for (q = 0; q < V3D_MAX_QUEUES; q++) { +- drm_sched_start(&v3d->queue[q].sched, true); ++ drm_sched_start(&v3d->queue[q].sched); + } + + mutex_unlock(&v3d->reset_lock); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 28cd4a9f4aad8a..81e9d1568f2a33 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -579,7 +579,7 @@ bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched); + void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad); +-void drm_sched_start(struct drm_gpu_scheduler *sched, bool full_recovery); ++void drm_sched_start(struct drm_gpu_scheduler *sched); + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); + void drm_sched_increase_karma(struct drm_sched_job *bad); + void drm_sched_fault(struct drm_gpu_scheduler *sched); + +From d915429f96e15434f3442aeec4dfe97dc84f24e3 Mon Sep 17 00:00:00 2001 +From: Jinjie Ruan +Date: Mon, 2 Sep 2024 17:44:04 +0800 +Subject: [PATCH 139/150] drm/panthor: Use the BITS_PER_LONG macro + +sizeof(unsigned long) * 8 is the number of bits in an unsigned long +variable, replace it with BITS_PER_LONG macro to make them simpler. + +And fix the warning: + WARNING: Comparisons should place the constant on the right side of the test + #23: FILE: drivers/gpu/drm/panthor/panthor_mmu.c:2696: + + if (BITS_PER_LONG < va_bits) { + +Signed-off-by: Jinjie Ruan +Reviewed-by: Steven Price +Signed-off-by: Steven Price +Link: https://patchwork.freedesktop.org/patch/msgid/20240902094404.1943710-1-ruanjinjie@huawei.com +--- + drivers/gpu/drm/panthor/panthor_mmu.c | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 4a1219610c58f1..4d8b7ab7b06bf8 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -2692,9 +2692,9 @@ int panthor_mmu_init(struct panthor_device *ptdev) + * which passes iova as an unsigned long. Patch the mmu_features to reflect this + * limitation. + */ +- if (sizeof(unsigned long) * 8 < va_bits) { ++ if (va_bits > BITS_PER_LONG) { + ptdev->gpu_info.mmu_features &= ~GENMASK(7, 0); +- ptdev->gpu_info.mmu_features |= sizeof(unsigned long) * 8; ++ ptdev->gpu_info.mmu_features |= BITS_PER_LONG; + } + + return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq); + +From ec36cd28147f7ef6ae07bff44227066e129f44f7 Mon Sep 17 00:00:00 2001 +From: Mary Guillemard +Date: Tue, 3 Sep 2024 16:49:55 +0200 +Subject: [PATCH 140/150] drm/panthor: Restrict high priorities on group_create + +We were allowing any users to create a high priority group without any +permission checks. As a result, this was allowing possible denial of +service. + +We now only allow the DRM master or users with the CAP_SYS_NICE +capability to set higher priorities than PANTHOR_GROUP_PRIORITY_MEDIUM. + +As the sole user of that uAPI lives in Mesa and hardcode a value of +MEDIUM [1], this should be safe to do. + +Additionally, as those checks are performed at the ioctl level, +panthor_group_create now only check for priority level validity. + +[1]https://gitlab.freedesktop.org/mesa/mesa/-/blob/f390835074bdf162a63deb0311d1a6de527f9f89/src/gallium/drivers/panfrost/pan_csf.c#L1038 + +Signed-off-by: Mary Guillemard +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Cc: stable@vger.kernel.org +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240903144955.144278-2-mary.guillemard@collabora.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 23 +++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_sched.c | 2 +- + include/uapi/drm/panthor_drm.h | 6 +++++- + 3 files changed, 29 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index e6c50c97b9c7d8..61f9d6b2348b9c 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -10,6 +10,7 @@ + #include + #include + ++#include + #include + #include + #include +@@ -995,6 +996,24 @@ static int panthor_ioctl_group_destroy(struct drm_device *ddev, void *data, + return panthor_group_destroy(pfile, args->group_handle); + } + ++static int group_priority_permit(struct drm_file *file, ++ u8 priority) ++{ ++ /* Ensure that priority is valid */ ++ if (priority > PANTHOR_GROUP_PRIORITY_HIGH) ++ return -EINVAL; ++ ++ /* Medium priority and below are always allowed */ ++ if (priority <= PANTHOR_GROUP_PRIORITY_MEDIUM) ++ return 0; ++ ++ /* Higher priorities require CAP_SYS_NICE or DRM_MASTER */ ++ if (capable(CAP_SYS_NICE) || drm_is_current_master(file)) ++ return 0; ++ ++ return -EACCES; ++} ++ + static int panthor_ioctl_group_create(struct drm_device *ddev, void *data, + struct drm_file *file) + { +@@ -1010,6 +1029,10 @@ static int panthor_ioctl_group_create(struct drm_device *ddev, void *data, + if (ret) + return ret; + ++ ret = group_priority_permit(file, args->priority); ++ if (ret) ++ return ret; ++ + ret = panthor_group_create(pfile, args, queue_args); + if (ret >= 0) { + args->group_handle = ret; +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index c426a392b08117..91a31b70c03752 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -3092,7 +3092,7 @@ int panthor_group_create(struct panthor_file *pfile, + if (group_args->pad) + return -EINVAL; + +- if (group_args->priority > PANTHOR_CSG_PRIORITY_HIGH) ++ if (group_args->priority >= PANTHOR_CSG_PRIORITY_COUNT) + return -EINVAL; + + if ((group_args->compute_core_mask & ~ptdev->gpu_info.shader_present) || +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 2d84c5920da040..a68c56612e82d0 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -692,7 +692,11 @@ enum drm_panthor_group_priority { + /** @PANTHOR_GROUP_PRIORITY_MEDIUM: Medium priority group. */ + PANTHOR_GROUP_PRIORITY_MEDIUM, + +- /** @PANTHOR_GROUP_PRIORITY_HIGH: High priority group. */ ++ /** ++ * @PANTHOR_GROUP_PRIORITY_HIGH: High priority group. ++ * ++ * Requires CAP_SYS_NICE or DRM_MASTER. ++ */ + PANTHOR_GROUP_PRIORITY_HIGH, + }; + + +From 101bd55a9db3db129039b685c75a7143cd13d813 Mon Sep 17 00:00:00 2001 +From: Mary Guillemard +Date: Fri, 30 Aug 2024 10:03:50 +0200 +Subject: [PATCH 141/150] drm/panthor: Add DEV_QUERY_TIMESTAMP_INFO dev query + +Expose timestamp information supported by the GPU with a new device +query. + +Mali uses an external timer as GPU system time. On ARM, this is wired to +the generic arch timer so we wire cntfrq_el0 as device frequency. + +This new uAPI will be used in Mesa to implement timestamp queries and +VK_KHR_calibrated_timestamps. + +Since this extends the uAPI and because userland needs a way to advertise +those features conditionally, this also bumps the driver minor version. + +v2: +- Rewrote to use GPU timestamp register +- Added timestamp_offset to drm_panthor_timestamp_info +- Add missing include for arch_timer_get_cntfrq +- Rework commit message + +v3: +- Add panthor_gpu_read_64bit_counter +- Change panthor_gpu_read_timestamp to use + panthor_gpu_read_64bit_counter + +v4: +- Fix multiple typos in uAPI documentation +- Mention behavior when the timestamp frequency is unknown +- Use u64 instead of unsigned long long + for panthor_gpu_read_timestamp +- Apply r-b from Mihail + +Signed-off-by: Mary Guillemard +Reviewed-by: Mihail Atanassov +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240830080349.24736-2-mary.guillemard@collabora.com +--- + drivers/gpu/drm/panthor/panthor_drv.c | 43 +++++++++++++++++++++++- + drivers/gpu/drm/panthor/panthor_gpu.c | 47 +++++++++++++++++++++++++++ + drivers/gpu/drm/panthor/panthor_gpu.h | 4 +++ + include/uapi/drm/panthor_drm.h | 22 +++++++++++++ + 4 files changed, 115 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index 61f9d6b2348b9c..fab2cf475fdfa0 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -3,6 +3,10 @@ + /* Copyright 2019 Linaro, Ltd., Rob Herring */ + /* Copyright 2019 Collabora ltd. */ + ++#ifdef CONFIG_ARM_ARCH_TIMER ++#include ++#endif ++ + #include + #include + #include +@@ -164,6 +168,7 @@ panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride, + _Generic(_obj_name, \ + PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_timestamp_info, current_timestamp), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \ +@@ -750,10 +755,33 @@ static void panthor_submit_ctx_cleanup(struct panthor_submit_ctx *ctx, + kvfree(ctx->jobs); + } + ++static int panthor_query_timestamp_info(struct panthor_device *ptdev, ++ struct drm_panthor_timestamp_info *arg) ++{ ++ int ret; ++ ++ ret = pm_runtime_resume_and_get(ptdev->base.dev); ++ if (ret) ++ return ret; ++ ++#ifdef CONFIG_ARM_ARCH_TIMER ++ arg->timestamp_frequency = arch_timer_get_cntfrq(); ++#else ++ arg->timestamp_frequency = 0; ++#endif ++ arg->current_timestamp = panthor_gpu_read_timestamp(ptdev); ++ arg->timestamp_offset = panthor_gpu_read_timestamp_offset(ptdev); ++ ++ pm_runtime_put(ptdev->base.dev); ++ return 0; ++} ++ + static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct drm_file *file) + { + struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); + struct drm_panthor_dev_query *args = data; ++ struct drm_panthor_timestamp_info timestamp_info; ++ int ret; + + if (!args->pointer) { + switch (args->type) { +@@ -765,6 +793,10 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d + args->size = sizeof(ptdev->csif_info); + return 0; + ++ case DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO: ++ args->size = sizeof(timestamp_info); ++ return 0; ++ + default: + return -EINVAL; + } +@@ -777,6 +809,14 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d + case DRM_PANTHOR_DEV_QUERY_CSIF_INFO: + return PANTHOR_UOBJ_SET(args->pointer, args->size, ptdev->csif_info); + ++ case DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO: ++ ret = panthor_query_timestamp_info(ptdev, ×tamp_info); ++ ++ if (ret) ++ return ret; ++ ++ return PANTHOR_UOBJ_SET(args->pointer, args->size, timestamp_info); ++ + default: + return -EINVAL; + } +@@ -1387,6 +1427,7 @@ static const struct file_operations panthor_drm_driver_fops = { + /* + * PanCSF driver version: + * - 1.0 - initial interface ++ * - 1.1 - adds DEV_QUERY_TIMESTAMP_INFO query + */ + static const struct drm_driver panthor_drm_driver = { + .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ | +@@ -1400,7 +1441,7 @@ static const struct drm_driver panthor_drm_driver = { + .desc = "Panthor DRM driver", + .date = "20230801", + .major = 1, +- .minor = 0, ++ .minor = 1, + + .gem_create_object = panthor_gem_create_object, + .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.c b/drivers/gpu/drm/panthor/panthor_gpu.c +index 5251d8764e7d7a..2d3529a0b156c1 100644 +--- a/drivers/gpu/drm/panthor/panthor_gpu.c ++++ b/drivers/gpu/drm/panthor/panthor_gpu.c +@@ -480,3 +480,50 @@ void panthor_gpu_resume(struct panthor_device *ptdev) + panthor_gpu_irq_resume(&ptdev->gpu->irq, GPU_INTERRUPTS_MASK); + panthor_gpu_l2_power_on(ptdev); + } ++ ++/** ++ * panthor_gpu_read_64bit_counter() - Read a 64-bit counter at a given offset. ++ * @ptdev: Device. ++ * @reg: The offset of the register to read. ++ * ++ * Return: The counter value. ++ */ ++static u64 ++panthor_gpu_read_64bit_counter(struct panthor_device *ptdev, u32 reg) ++{ ++ u32 hi, lo; ++ ++ do { ++ hi = gpu_read(ptdev, reg + 0x4); ++ lo = gpu_read(ptdev, reg); ++ } while (hi != gpu_read(ptdev, reg + 0x4)); ++ ++ return ((u64)hi << 32) | lo; ++} ++ ++/** ++ * panthor_gpu_read_timestamp() - Read the timestamp register. ++ * @ptdev: Device. ++ * ++ * Return: The GPU timestamp value. ++ */ ++u64 panthor_gpu_read_timestamp(struct panthor_device *ptdev) ++{ ++ return panthor_gpu_read_64bit_counter(ptdev, GPU_TIMESTAMP_LO); ++} ++ ++/** ++ * panthor_gpu_read_timestamp_offset() - Read the timestamp offset register. ++ * @ptdev: Device. ++ * ++ * Return: The GPU timestamp offset value. ++ */ ++u64 panthor_gpu_read_timestamp_offset(struct panthor_device *ptdev) ++{ ++ u32 hi, lo; ++ ++ hi = gpu_read(ptdev, GPU_TIMESTAMP_OFFSET_HI); ++ lo = gpu_read(ptdev, GPU_TIMESTAMP_OFFSET_LO); ++ ++ return ((u64)hi << 32) | lo; ++} +diff --git a/drivers/gpu/drm/panthor/panthor_gpu.h b/drivers/gpu/drm/panthor/panthor_gpu.h +index bba7555dd3c649..7f6133a6612743 100644 +--- a/drivers/gpu/drm/panthor/panthor_gpu.h ++++ b/drivers/gpu/drm/panthor/panthor_gpu.h +@@ -5,6 +5,8 @@ + #ifndef __PANTHOR_GPU_H__ + #define __PANTHOR_GPU_H__ + ++#include ++ + struct panthor_device; + + int panthor_gpu_init(struct panthor_device *ptdev); +@@ -48,5 +50,7 @@ int panthor_gpu_l2_power_on(struct panthor_device *ptdev); + int panthor_gpu_flush_caches(struct panthor_device *ptdev, + u32 l2, u32 lsc, u32 other); + int panthor_gpu_soft_reset(struct panthor_device *ptdev); ++u64 panthor_gpu_read_timestamp(struct panthor_device *ptdev); ++u64 panthor_gpu_read_timestamp_offset(struct panthor_device *ptdev); + + #endif +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index a68c56612e82d0..5281a0c11ae7a9 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -260,6 +260,9 @@ enum drm_panthor_dev_query_type { + + /** @DRM_PANTHOR_DEV_QUERY_CSIF_INFO: Query command-stream interface information. */ + DRM_PANTHOR_DEV_QUERY_CSIF_INFO, ++ ++ /** @DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO: Query timestamp information. */ ++ DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO, + }; + + /** +@@ -377,6 +380,25 @@ struct drm_panthor_csif_info { + __u32 pad; + }; + ++/** ++ * struct drm_panthor_timestamp_info - Timestamp information ++ * ++ * Structure grouping all queryable information relating to the GPU timestamp. ++ */ ++struct drm_panthor_timestamp_info { ++ /** ++ * @timestamp_frequency: The frequency of the timestamp timer or 0 if ++ * unknown. ++ */ ++ __u64 timestamp_frequency; ++ ++ /** @current_timestamp: The current timestamp. */ ++ __u64 current_timestamp; ++ ++ /** @timestamp_offset: The offset of the timestamp timer. */ ++ __u64 timestamp_offset; ++}; ++ + /** + * struct drm_panthor_dev_query - Arguments passed to DRM_PANTHOR_IOCTL_DEV_QUERY + */ + +From 7e12cd71c5156f5a0468051bf722097855ab2886 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Adri=C3=A1n=20Larumbe?= +Date: Mon, 2 Sep 2024 14:02:35 +0100 +Subject: [PATCH 142/150] drm/panthor: flush FW AS caches in slow reset path +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +In the off-chance that waiting for the firmware to signal its booted status +timed out in the fast reset path, one must flush the cache lines for the +entire FW VM address space before reloading the regions, otherwise stale +values eventually lead to a scheduler job timeout. + +Fixes: 647810ec2476 ("drm/panthor: Add the MMU/VM logical block") +Cc: stable@vger.kernel.org +Signed-off-by: Adrián Larumbe +Acked-by: Liviu Dudau +Reviewed-by: Steven Price +Reviewed-by: Boris Brezillon +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240902130237.3440720-1-adrian.larumbe@collabora.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 8 +++++++- + drivers/gpu/drm/panthor/panthor_mmu.c | 21 ++++++++++++++++++--- + drivers/gpu/drm/panthor/panthor_mmu.h | 1 + + 3 files changed, 26 insertions(+), 4 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index 857f3f11258aad..ef232c0c204932 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -1089,6 +1089,12 @@ int panthor_fw_post_reset(struct panthor_device *ptdev) + panthor_fw_stop(ptdev); + ptdev->fw->fast_reset = false; + drm_err(&ptdev->base, "FW fast reset failed, trying a slow reset"); ++ ++ ret = panthor_vm_flush_all(ptdev->fw->vm); ++ if (ret) { ++ drm_err(&ptdev->base, "FW slow reset failed (couldn't flush FW's AS l2cache)"); ++ return ret; ++ } + } + + /* Reload all sections, including RO ones. We're not supposed +@@ -1099,7 +1105,7 @@ int panthor_fw_post_reset(struct panthor_device *ptdev) + + ret = panthor_fw_start(ptdev); + if (ret) { +- drm_err(&ptdev->base, "FW slow reset failed"); ++ drm_err(&ptdev->base, "FW slow reset failed (couldn't start the FW )"); + return ret; + } + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 4d8b7ab7b06bf8..df12c857990359 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -575,6 +575,12 @@ static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr, + if (as_nr < 0) + return 0; + ++ /* ++ * If the AS number is greater than zero, then we can be sure ++ * the device is up and running, so we don't need to explicitly ++ * power it up ++ */ ++ + if (op != AS_COMMAND_UNLOCK) + lock_region(ptdev, as_nr, iova, size); + +@@ -873,14 +879,23 @@ static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size) + if (!drm_dev_enter(&ptdev->base, &cookie)) + return 0; + +- /* Flush the PTs only if we're already awake */ +- if (pm_runtime_active(ptdev->base.dev)) +- ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT); ++ ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT); + + drm_dev_exit(cookie); + return ret; + } + ++/** ++ * panthor_vm_flush_all() - Flush L2 caches for the entirety of a VM's AS ++ * @vm: VM whose cache to flush ++ * ++ * Return: 0 on success, a negative error code if flush failed. ++ */ ++int panthor_vm_flush_all(struct panthor_vm *vm) ++{ ++ return panthor_vm_flush_range(vm, vm->base.mm_start, vm->base.mm_range); ++} ++ + static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, u64 size) + { + struct panthor_device *ptdev = vm->ptdev; +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h +index e6a2198b009b76..10635cc8a94dfa 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.h ++++ b/drivers/gpu/drm/panthor/panthor_mmu.h +@@ -31,6 +31,7 @@ panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset); + int panthor_vm_active(struct panthor_vm *vm); + void panthor_vm_idle(struct panthor_vm *vm); + int panthor_vm_as(struct panthor_vm *vm); ++int panthor_vm_flush_all(struct panthor_vm *vm); + + struct panthor_heap_pool * + panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create); + +From c351db0bed6a73eb357d080f1f9d48e893ca94ea Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Christian=20K=C3=B6nig?= +Date: Mon, 26 Aug 2024 14:25:38 +0200 +Subject: [PATCH 143/150] drm/sched: add optional errno to drm_sched_start() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +The current implementation of drm_sched_start uses a hardcoded +-ECANCELED to dispose of a job when the parent/hw fence is NULL. +This results in drm_sched_job_done being called with -ECANCELED for +each job with a NULL parent in the pending list, making it difficult +to distinguish between recovery methods, whether a queue reset or a +full GPU reset was used. + +To improve this, we first try a soft recovery for timeout jobs and +use the error code -ENODATA. If soft recovery fails, we proceed with +a queue reset, where the error code remains -ENODATA for the job. +Finally, for a full GPU reset, we use error codes -ECANCELED or +-ETIME. This patch adds an error code parameter to drm_sched_start, +allowing us to differentiate between queue reset and GPU reset +failures. This enables user mode and test applications to validate +the expected correctness of the requested operation. After a +successful queue reset, the only way to continue normal operation is +to call drm_sched_job_done with the specific error code -ENODATA. + +v1: Initial implementation by Jesse utilized +amdgpu_device_lock_reset_domain + and amdgpu_device_unlock_reset_domain to allow user mode to track + the queue reset status and distinguish between queue reset and + GPU reset. +v2: Christian suggested using the error codes -ENODATA for queue reset + and -ECANCELED or -ETIME for GPU reset, returned to + amdgpu_cs_wait_ioctl. +v3: To meet the requirements, we introduce a new function + drm_sched_start_ex with an additional parameter to set + dma_fence_set_error, allowing us to handle the specific error + codes appropriately and dispose of bad jobs with the selected + error code depending on whether it was a queue reset or GPU reset. +v4: Alex suggested using a new name, +drm_sched_start_with_recovery_error, + which more accurately describes the function's purpose. + Additionally, it was recommended to add documentation details + about the new method. +v5: Fixed declaration of new function +drm_sched_start_with_recovery_error.(Alex) +v6 (chk): rebase on upstream changes, cleanup the commit message, + drop the new function again and update all callers, + apply the errno also to scheduler fences with hw fences +v7 (chk): rebased + +Signed-off-by: Jesse Zhang +Signed-off-by: Vitaly Prosyak +Signed-off-by: Christian König +Acked-by: Daniel Vetter +Reviewed-by: Alex Deucher +Link: https://patchwork.freedesktop.org/patch/msgid/20240826122541.85663-1-christian.koenig@amd.com +--- + drivers/gpu/drm/etnaviv/etnaviv_sched.c | 2 +- + drivers/gpu/drm/lima/lima_sched.c | 2 +- + drivers/gpu/drm/panfrost/panfrost_job.c | 2 +- + drivers/gpu/drm/panthor/panthor_mmu.c | 2 +- + drivers/gpu/drm/panthor/panthor_sched.c | 2 +- + drivers/gpu/drm/scheduler/sched_main.c | 7 ++++--- + drivers/gpu/drm/v3d/v3d_sched.c | 2 +- + include/drm/gpu_scheduler.h | 2 +- + 8 files changed, 11 insertions(+), 10 deletions(-) + +diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +index 752fc5a21cb38b..25216240899e9e 100644 +--- a/drivers/gpu/drm/etnaviv/etnaviv_sched.c ++++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c +@@ -71,7 +71,7 @@ static enum drm_gpu_sched_stat etnaviv_sched_timedout_job(struct drm_sched_job + + drm_sched_resubmit_jobs(&gpu->sched); + +- drm_sched_start(&gpu->sched); ++ drm_sched_start(&gpu->sched, 0); + return DRM_GPU_SCHED_STAT_NOMINAL; + + out_no_timeout: +diff --git a/drivers/gpu/drm/lima/lima_sched.c b/drivers/gpu/drm/lima/lima_sched.c +index 8247abee957e30..849709645af423 100644 +--- a/drivers/gpu/drm/lima/lima_sched.c ++++ b/drivers/gpu/drm/lima/lima_sched.c +@@ -430,7 +430,7 @@ static enum drm_gpu_sched_stat lima_sched_timedout_job(struct drm_sched_job *job + lima_pm_idle(ldev); + + drm_sched_resubmit_jobs(&pipe->base); +- drm_sched_start(&pipe->base); ++ drm_sched_start(&pipe->base, 0); + + return DRM_GPU_SCHED_STAT_NOMINAL; + } +diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c +index b812e0b6f2dad6..81ded687fe205d 100644 +--- a/drivers/gpu/drm/panfrost/panfrost_job.c ++++ b/drivers/gpu/drm/panfrost/panfrost_job.c +@@ -696,7 +696,7 @@ panfrost_reset(struct panfrost_device *pfdev, + + /* Restart the schedulers */ + for (i = 0; i < NUM_JOB_SLOTS; i++) +- drm_sched_start(&pfdev->js->queue[i].sched); ++ drm_sched_start(&pfdev->js->queue[i].sched, 0); + + /* Re-enable job interrupts now that everything has been restarted. */ + job_write(pfdev, JOB_INT_MASK, +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index df12c857990359..1eee95abcda726 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -832,7 +832,7 @@ static void panthor_vm_stop(struct panthor_vm *vm) + + static void panthor_vm_start(struct panthor_vm *vm) + { +- drm_sched_start(&vm->sched); ++ drm_sched_start(&vm->sched, 0); + } + + /** +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 91a31b70c03752..42afdf0ddb7ece 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -2538,7 +2538,7 @@ static void queue_start(struct panthor_queue *queue) + list_for_each_entry(job, &queue->scheduler.pending_list, base.list) + job->base.s_fence->parent = dma_fence_get(job->done_fence); + +- drm_sched_start(&queue->scheduler); ++ drm_sched_start(&queue->scheduler, 0); + } + + static void panthor_group_stop(struct panthor_group *group) +diff --git a/drivers/gpu/drm/scheduler/sched_main.c b/drivers/gpu/drm/scheduler/sched_main.c +index e75ccf6ef90741..8d969074086430 100644 +--- a/drivers/gpu/drm/scheduler/sched_main.c ++++ b/drivers/gpu/drm/scheduler/sched_main.c +@@ -674,9 +674,10 @@ EXPORT_SYMBOL(drm_sched_stop); + * drm_sched_start - recover jobs after a reset + * + * @sched: scheduler instance ++ * @errno: error to set on the pending fences + * + */ +-void drm_sched_start(struct drm_gpu_scheduler *sched) ++void drm_sched_start(struct drm_gpu_scheduler *sched, int errno) + { + struct drm_sched_job *s_job, *tmp; + +@@ -691,13 +692,13 @@ void drm_sched_start(struct drm_gpu_scheduler *sched) + atomic_add(s_job->credits, &sched->credit_count); + + if (!fence) { +- drm_sched_job_done(s_job, -ECANCELED); ++ drm_sched_job_done(s_job, errno ?: -ECANCELED); + continue; + } + + if (dma_fence_add_callback(fence, &s_job->cb, + drm_sched_job_done_cb)) +- drm_sched_job_done(s_job, fence->error); ++ drm_sched_job_done(s_job, fence->error ?: errno); + } + + drm_sched_start_timeout_unlocked(sched); +diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c +index 4c7be3a2cf4a3e..07dfea037cad91 100644 +--- a/drivers/gpu/drm/v3d/v3d_sched.c ++++ b/drivers/gpu/drm/v3d/v3d_sched.c +@@ -274,7 +274,7 @@ v3d_gpu_reset_for_timeout(struct v3d_dev *v3d, struct drm_sched_job *sched_job) + + /* Unblock schedulers and restart their jobs. */ + for (q = 0; q < V3D_MAX_QUEUES; q++) { +- drm_sched_start(&v3d->queue[q].sched); ++ drm_sched_start(&v3d->queue[q].sched, 0); + } + + mutex_unlock(&v3d->reset_lock); +diff --git a/include/drm/gpu_scheduler.h b/include/drm/gpu_scheduler.h +index 81e9d1568f2a33..bdd3845e02c5bc 100644 +--- a/include/drm/gpu_scheduler.h ++++ b/include/drm/gpu_scheduler.h +@@ -579,7 +579,7 @@ bool drm_sched_wqueue_ready(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_stop(struct drm_gpu_scheduler *sched); + void drm_sched_wqueue_start(struct drm_gpu_scheduler *sched); + void drm_sched_stop(struct drm_gpu_scheduler *sched, struct drm_sched_job *bad); +-void drm_sched_start(struct drm_gpu_scheduler *sched); ++void drm_sched_start(struct drm_gpu_scheduler *sched, int errno); + void drm_sched_resubmit_jobs(struct drm_gpu_scheduler *sched); + void drm_sched_increase_karma(struct drm_sched_job *bad); + void drm_sched_fault(struct drm_gpu_scheduler *sched); + +From 9b8ea8421555ddf283186894eac1d783d580085e Mon Sep 17 00:00:00 2001 +From: Steven Price +Date: Fri, 6 Sep 2024 10:40:25 +0100 +Subject: [PATCH 144/150] drm/panthor: Display FW version information + +The version number output when loading the firmware is actually the +interface version not the version of the firmware itself. Update the +message to make this clearer. + +However, the firmware binary has a git SHA embedded into it which can be +used to identify which firmware binary is being loaded. So output this +as a drm_info() so that it's obvious from a dmesg log which firmware +binary is being used. + +Reviewed-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Signed-off-by: Steven Price +Signed-off-by: Boris Brezillon +Link: https://patchwork.freedesktop.org/patch/msgid/20240906094025.638173-1-steven.price@arm.com +--- + drivers/gpu/drm/panthor/panthor_fw.c | 57 +++++++++++++++++++++++++++- + 1 file changed, 56 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_fw.c b/drivers/gpu/drm/panthor/panthor_fw.c +index ef232c0c204932..631f639b8b865a 100644 +--- a/drivers/gpu/drm/panthor/panthor_fw.c ++++ b/drivers/gpu/drm/panthor/panthor_fw.c +@@ -78,6 +78,12 @@ enum panthor_fw_binary_entry_type { + + /** @CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA: Timeline metadata interface. */ + CSF_FW_BINARY_ENTRY_TYPE_TIMELINE_METADATA = 4, ++ ++ /** ++ * @CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA: Metadata about how ++ * the FW binary was built. ++ */ ++ CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA = 6 + }; + + #define CSF_FW_BINARY_ENTRY_TYPE(ehdr) ((ehdr) & 0xff) +@@ -132,6 +138,13 @@ struct panthor_fw_binary_section_entry_hdr { + } data; + }; + ++struct panthor_fw_build_info_hdr { ++ /** @meta_start: Offset of the build info data in the FW binary */ ++ u32 meta_start; ++ /** @meta_size: Size of the build info data in the FW binary */ ++ u32 meta_size; ++}; ++ + /** + * struct panthor_fw_binary_iter - Firmware binary iterator + * +@@ -628,6 +641,46 @@ static int panthor_fw_load_section_entry(struct panthor_device *ptdev, + return 0; + } + ++static int panthor_fw_read_build_info(struct panthor_device *ptdev, ++ const struct firmware *fw, ++ struct panthor_fw_binary_iter *iter, ++ u32 ehdr) ++{ ++ struct panthor_fw_build_info_hdr hdr; ++ char header[9]; ++ const char git_sha_header[sizeof(header)] = "git_sha: "; ++ int ret; ++ ++ ret = panthor_fw_binary_iter_read(ptdev, iter, &hdr, sizeof(hdr)); ++ if (ret) ++ return ret; ++ ++ if (hdr.meta_start > fw->size || ++ hdr.meta_start + hdr.meta_size > fw->size) { ++ drm_err(&ptdev->base, "Firmware build info corrupt\n"); ++ /* We don't need the build info, so continue */ ++ return 0; ++ } ++ ++ if (memcmp(git_sha_header, fw->data + hdr.meta_start, ++ sizeof(git_sha_header))) { ++ /* Not the expected header, this isn't metadata we understand */ ++ return 0; ++ } ++ ++ /* Check that the git SHA is NULL terminated as expected */ ++ if (fw->data[hdr.meta_start + hdr.meta_size - 1] != '\0') { ++ drm_warn(&ptdev->base, "Firmware's git sha is not NULL terminated\n"); ++ /* Don't treat as fatal */ ++ return 0; ++ } ++ ++ drm_info(&ptdev->base, "Firmware git sha: %s\n", ++ fw->data + hdr.meta_start + sizeof(git_sha_header)); ++ ++ return 0; ++} ++ + static void + panthor_reload_fw_sections(struct panthor_device *ptdev, bool full_reload) + { +@@ -672,6 +725,8 @@ static int panthor_fw_load_entry(struct panthor_device *ptdev, + switch (CSF_FW_BINARY_ENTRY_TYPE(ehdr)) { + case CSF_FW_BINARY_ENTRY_TYPE_IFACE: + return panthor_fw_load_section_entry(ptdev, fw, &eiter, ehdr); ++ case CSF_FW_BINARY_ENTRY_TYPE_BUILD_INFO_METADATA: ++ return panthor_fw_read_build_info(ptdev, fw, &eiter, ehdr); + + /* FIXME: handle those entry types? */ + case CSF_FW_BINARY_ENTRY_TYPE_CONFIG: +@@ -921,7 +976,7 @@ static int panthor_fw_init_ifaces(struct panthor_device *ptdev) + return ret; + } + +- drm_info(&ptdev->base, "CSF FW v%d.%d.%d, Features %#x Instrumentation features %#x", ++ drm_info(&ptdev->base, "CSF FW using interface v%d.%d.%d, Features %#x Instrumentation features %#x", + CSF_IFACE_VERSION_MAJOR(glb_iface->control->version), + CSF_IFACE_VERSION_MINOR(glb_iface->control->version), + CSF_IFACE_VERSION_PATCH(glb_iface->control->version), + +From 95ff25714f2e897ce4f9634fbda00474e2aa0ead Mon Sep 17 00:00:00 2001 +From: Steven Price +Date: Mon, 23 Sep 2024 11:34:06 +0100 +Subject: [PATCH 145/150] drm/panthor: Fix race when converting group handle to + group object + +XArray provides it's own internal lock which protects the internal array +when entries are being simultaneously added and removed. However there +is still a race between retrieving the pointer from the XArray and +incrementing the reference count. + +To avoid this race simply hold the internal XArray lock when +incrementing the reference count, this ensures there cannot be a racing +call to xa_erase(). + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Signed-off-by: Steven Price +Reviewed-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Link: https://patchwork.freedesktop.org/patch/msgid/20240923103406.2509906-1-steven.price@arm.com +--- + drivers/gpu/drm/panthor/panthor_sched.c | 16 ++++++++++++++-- + 1 file changed, 14 insertions(+), 2 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 42afdf0ddb7ece..0dbeebcf23b4f5 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -3242,6 +3242,18 @@ int panthor_group_destroy(struct panthor_file *pfile, u32 group_handle) + return 0; + } + ++static struct panthor_group *group_from_handle(struct panthor_group_pool *pool, ++ u32 group_handle) ++{ ++ struct panthor_group *group; ++ ++ xa_lock(&pool->xa); ++ group = group_get(xa_load(&pool->xa, group_handle)); ++ xa_unlock(&pool->xa); ++ ++ return group; ++} ++ + int panthor_group_get_state(struct panthor_file *pfile, + struct drm_panthor_group_get_state *get_state) + { +@@ -3253,7 +3265,7 @@ int panthor_group_get_state(struct panthor_file *pfile, + if (get_state->pad) + return -EINVAL; + +- group = group_get(xa_load(&gpool->xa, get_state->group_handle)); ++ group = group_from_handle(gpool, get_state->group_handle); + if (!group) + return -EINVAL; + +@@ -3384,7 +3396,7 @@ panthor_job_create(struct panthor_file *pfile, + job->call_info.latest_flush = qsubmit->latest_flush; + INIT_LIST_HEAD(&job->node); + +- job->group = group_get(xa_load(&gpool->xa, group_handle)); ++ job->group = group_from_handle(gpool, group_handle); + if (!job->group) { + ret = -EINVAL; + goto err_put_job; + +From e5c8af4a2cb05eb24af3cc9ea3db1b86cfd7ca65 Mon Sep 17 00:00:00 2001 +From: Mary Guillemard +Date: Mon, 9 Sep 2024 08:48:20 +0200 +Subject: [PATCH 146/150] drm/panthor: Add PANTHOR_GROUP_PRIORITY_REALTIME + group priority + +This adds a new value to drm_panthor_group_priority exposing the +realtime priority to userspace. + +This is required to implement NV_context_priority_realtime in Mesa. + +v2: +- Add Steven Price r-b + +v3: +- Add Boris Brezillon r-b + +Signed-off-by: Mary Guillemard +Reviewed-by: Steven Price +Reviewed-by: Boris Brezillon +--- + drivers/gpu/drm/panthor/panthor_drv.c | 2 +- + drivers/gpu/drm/panthor/panthor_sched.c | 2 -- + include/uapi/drm/panthor_drm.h | 7 +++++++ + 3 files changed, 8 insertions(+), 3 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index fab2cf475fdfa0..f0a259978c136c 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -1040,7 +1040,7 @@ static int group_priority_permit(struct drm_file *file, + u8 priority) + { + /* Ensure that priority is valid */ +- if (priority > PANTHOR_GROUP_PRIORITY_HIGH) ++ if (priority > PANTHOR_GROUP_PRIORITY_REALTIME) + return -EINVAL; + + /* Medium priority and below are always allowed */ +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 0dbeebcf23b4f5..03a1febbb9b94a 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -137,8 +137,6 @@ enum panthor_csg_priority { + * non-real-time groups. When such a group becomes executable, + * it will evict the group with the lowest non-rt priority if + * there's no free group slot available. +- * +- * Currently not exposed to userspace. + */ + PANTHOR_CSG_PRIORITY_RT, + +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 5281a0c11ae7a9..2922d0d7f9bb60 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -720,6 +720,13 @@ enum drm_panthor_group_priority { + * Requires CAP_SYS_NICE or DRM_MASTER. + */ + PANTHOR_GROUP_PRIORITY_HIGH, ++ ++ /** ++ * @PANTHOR_GROUP_PRIORITY_REALTIME: Realtime priority group. ++ * ++ * Requires CAP_SYS_NICE or DRM_MASTER. ++ */ ++ PANTHOR_GROUP_PRIORITY_REALTIME, + }; + + /** + +From d15e51532e6770de1976ae467a9b4ff6f7a55036 Mon Sep 17 00:00:00 2001 +From: Mary Guillemard +Date: Mon, 9 Sep 2024 08:48:21 +0200 +Subject: [PATCH 147/150] drm/panthor: Add DEV_QUERY_GROUP_PRIORITIES_INFO dev + query + +Expose allowed group priorities with a new device query. + +This new uAPI will be used in Mesa to properly report what priorities a +user can use for EGL_IMG_context_priority. + +Since this extends the uAPI and because userland needs a way to +advertise priorities accordingly, this also bumps the driver minor +version. + +v2: +- Remove drm_panthor_group_allow_priority_flags definition +- Document that allowed_mask is a bitmask of drm_panthor_group_priority + +v3: +- Use BIT macro in panthor_query_group_priorities_info +- Add r-b from Steven Price and Boris Brezillon + +Signed-off-by: Mary Guillemard +Reviewed-by: Steven Price +Reviewed-by: Boris Brezillon +--- + drivers/gpu/drm/panthor/panthor_drv.c | 61 ++++++++++++++++++--------- + include/uapi/drm/panthor_drm.h | 22 ++++++++++ + 2 files changed, 64 insertions(+), 19 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_drv.c b/drivers/gpu/drm/panthor/panthor_drv.c +index f0a259978c136c..10bd86e028d320 100644 +--- a/drivers/gpu/drm/panthor/panthor_drv.c ++++ b/drivers/gpu/drm/panthor/panthor_drv.c +@@ -169,6 +169,7 @@ panthor_get_uobj_array(const struct drm_panthor_obj_array *in, u32 min_stride, + PANTHOR_UOBJ_DECL(struct drm_panthor_gpu_info, tiler_present), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_csif_info, pad), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_timestamp_info, current_timestamp), \ ++ PANTHOR_UOBJ_DECL(struct drm_panthor_group_priorities_info, pad), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_sync_op, timeline_value), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_queue_submit, syncs), \ + PANTHOR_UOBJ_DECL(struct drm_panthor_queue_create, ringbuf_size), \ +@@ -776,11 +777,41 @@ static int panthor_query_timestamp_info(struct panthor_device *ptdev, + return 0; + } + ++static int group_priority_permit(struct drm_file *file, ++ u8 priority) ++{ ++ /* Ensure that priority is valid */ ++ if (priority > PANTHOR_GROUP_PRIORITY_REALTIME) ++ return -EINVAL; ++ ++ /* Medium priority and below are always allowed */ ++ if (priority <= PANTHOR_GROUP_PRIORITY_MEDIUM) ++ return 0; ++ ++ /* Higher priorities require CAP_SYS_NICE or DRM_MASTER */ ++ if (capable(CAP_SYS_NICE) || drm_is_current_master(file)) ++ return 0; ++ ++ return -EACCES; ++} ++ ++static void panthor_query_group_priorities_info(struct drm_file *file, ++ struct drm_panthor_group_priorities_info *arg) ++{ ++ int prio; ++ ++ for (prio = PANTHOR_GROUP_PRIORITY_REALTIME; prio >= 0; prio--) { ++ if (!group_priority_permit(file, prio)) ++ arg->allowed_mask |= BIT(prio); ++ } ++} ++ + static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct drm_file *file) + { + struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base); + struct drm_panthor_dev_query *args = data; + struct drm_panthor_timestamp_info timestamp_info; ++ struct drm_panthor_group_priorities_info priorities_info; + int ret; + + if (!args->pointer) { +@@ -797,6 +828,10 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d + args->size = sizeof(timestamp_info); + return 0; + ++ case DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO: ++ args->size = sizeof(priorities_info); ++ return 0; ++ + default: + return -EINVAL; + } +@@ -817,6 +852,10 @@ static int panthor_ioctl_dev_query(struct drm_device *ddev, void *data, struct d + + return PANTHOR_UOBJ_SET(args->pointer, args->size, timestamp_info); + ++ case DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO: ++ panthor_query_group_priorities_info(file, &priorities_info); ++ return PANTHOR_UOBJ_SET(args->pointer, args->size, priorities_info); ++ + default: + return -EINVAL; + } +@@ -1036,24 +1075,6 @@ static int panthor_ioctl_group_destroy(struct drm_device *ddev, void *data, + return panthor_group_destroy(pfile, args->group_handle); + } + +-static int group_priority_permit(struct drm_file *file, +- u8 priority) +-{ +- /* Ensure that priority is valid */ +- if (priority > PANTHOR_GROUP_PRIORITY_REALTIME) +- return -EINVAL; +- +- /* Medium priority and below are always allowed */ +- if (priority <= PANTHOR_GROUP_PRIORITY_MEDIUM) +- return 0; +- +- /* Higher priorities require CAP_SYS_NICE or DRM_MASTER */ +- if (capable(CAP_SYS_NICE) || drm_is_current_master(file)) +- return 0; +- +- return -EACCES; +-} +- + static int panthor_ioctl_group_create(struct drm_device *ddev, void *data, + struct drm_file *file) + { +@@ -1428,6 +1449,8 @@ static const struct file_operations panthor_drm_driver_fops = { + * PanCSF driver version: + * - 1.0 - initial interface + * - 1.1 - adds DEV_QUERY_TIMESTAMP_INFO query ++ * - 1.2 - adds DEV_QUERY_GROUP_PRIORITIES_INFO query ++ * - adds PANTHOR_GROUP_PRIORITY_REALTIME priority + */ + static const struct drm_driver panthor_drm_driver = { + .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ | +@@ -1441,7 +1464,7 @@ static const struct drm_driver panthor_drm_driver = { + .desc = "Panthor DRM driver", + .date = "20230801", + .major = 1, +- .minor = 1, ++ .minor = 2, + + .gem_create_object = panthor_gem_create_object, + .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table, +diff --git a/include/uapi/drm/panthor_drm.h b/include/uapi/drm/panthor_drm.h +index 2922d0d7f9bb60..0199e788bb7184 100644 +--- a/include/uapi/drm/panthor_drm.h ++++ b/include/uapi/drm/panthor_drm.h +@@ -263,6 +263,11 @@ enum drm_panthor_dev_query_type { + + /** @DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO: Query timestamp information. */ + DRM_PANTHOR_DEV_QUERY_TIMESTAMP_INFO, ++ ++ /** ++ * @DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO: Query allowed group priorities information. ++ */ ++ DRM_PANTHOR_DEV_QUERY_GROUP_PRIORITIES_INFO, + }; + + /** +@@ -399,6 +404,23 @@ struct drm_panthor_timestamp_info { + __u64 timestamp_offset; + }; + ++/** ++ * struct drm_panthor_group_priorities_info - Group priorities information ++ * ++ * Structure grouping all queryable information relating to the allowed group priorities. ++ */ ++struct drm_panthor_group_priorities_info { ++ /** ++ * @allowed_mask: Bitmask of the allowed group priorities. ++ * ++ * Each bit represents a variant of the enum drm_panthor_group_priority. ++ */ ++ __u8 allowed_mask; ++ ++ /** @pad: Padding fields, MBZ. */ ++ __u8 pad[3]; ++}; ++ + /** + * struct drm_panthor_dev_query - Arguments passed to DRM_PANTHOR_IOCTL_DEV_QUERY + */ + +From feae5f46353b1ce509f27fceb7169514ee34357d Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Fri, 13 Sep 2024 13:27:22 +0200 +Subject: [PATCH 148/150] drm/panthor: Lock the VM resv before calling + drm_gpuvm_bo_obtain_prealloc() + +drm_gpuvm_bo_obtain_prealloc() will call drm_gpuvm_bo_put() on our +pre-allocated BO if the association exists. Given we +only have one ref on preallocated_vm_bo, drm_gpuvm_bo_destroy() will +be called immediately, and we have to hold the VM resv lock when +calling this function. + +Fixes: 647810ec2476 ("drm/panthor: Add the MMU/VM logical block") +Signed-off-by: Boris Brezillon +Reviewed-by: Liviu Dudau +Reviewed-by: Steven Price +--- + drivers/gpu/drm/panthor/panthor_mmu.c | 8 ++++++++ + 1 file changed, 8 insertions(+) + +diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c +index 1eee95abcda726..0b38db5f9fac73 100644 +--- a/drivers/gpu/drm/panthor/panthor_mmu.c ++++ b/drivers/gpu/drm/panthor/panthor_mmu.c +@@ -1250,9 +1250,17 @@ static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx, + goto err_cleanup; + } + ++ /* drm_gpuvm_bo_obtain_prealloc() will call drm_gpuvm_bo_put() on our ++ * pre-allocated BO if the association exists. Given we ++ * only have one ref on preallocated_vm_bo, drm_gpuvm_bo_destroy() will ++ * be called immediately, and we have to hold the VM resv lock when ++ * calling this function. ++ */ ++ dma_resv_lock(panthor_vm_resv(vm), NULL); + mutex_lock(&bo->gpuva_list_lock); + op_ctx->map.vm_bo = drm_gpuvm_bo_obtain_prealloc(preallocated_vm_bo); + mutex_unlock(&bo->gpuva_list_lock); ++ dma_resv_unlock(panthor_vm_resv(vm)); + + /* If the a vm_bo for this combination exists, it already + * retains a pin ref, and we can release the one we took earlier. + +From 642b33c75b6071c6d7e11139b45c1f88c406a4e4 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 5 Sep 2024 09:01:54 +0200 +Subject: [PATCH 149/150] drm/panthor: Don't add write fences to the shared BOs + +The only user (the mesa gallium driver) is already assuming explicit +synchronization and doing the export/import dance on shared BOs. The +only reason we were registering ourselves as writers on external BOs +is because Xe, which was the reference back when we developed Panthor, +was doing so. Turns out Xe was wrong, and we really want bookkeep on +all registered fences, so userspace can explicitly upgrade those to +read/write when needed. + +Fixes: 4bdca1150792 ("drm/panthor: Add the driver frontend block") +Cc: Matthew Brost +Cc: Simona Vetter +Cc: +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +--- + drivers/gpu/drm/panthor/panthor_sched.c | 7 +------ + 1 file changed, 1 insertion(+), 6 deletions(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 03a1febbb9b94a..42138b69c70422 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -3434,13 +3434,8 @@ void panthor_job_update_resvs(struct drm_exec *exec, struct drm_sched_job *sched + { + struct panthor_job *job = container_of(sched_job, struct panthor_job, base); + +- /* Still not sure why we want USAGE_WRITE for external objects, since I +- * was assuming this would be handled through explicit syncs being imported +- * to external BOs with DMA_BUF_IOCTL_IMPORT_SYNC_FILE, but other drivers +- * seem to pass DMA_RESV_USAGE_WRITE, so there must be a good reason. +- */ + panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, +- DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_WRITE); ++ DMA_RESV_USAGE_BOOKKEEP, DMA_RESV_USAGE_BOOKKEEP); + } + + void panthor_sched_unplug(struct panthor_device *ptdev) + +From c81ebd8e12b64a42a6efd68cc0ed018b57d14e91 Mon Sep 17 00:00:00 2001 +From: Boris Brezillon +Date: Thu, 5 Sep 2024 09:19:14 +0200 +Subject: [PATCH 150/150] drm/panthor: Don't declare a queue blocked if + deferred operations are pending + +If deferred operations are pending, we want to wait for those to +land before declaring the queue blocked on a SYNC_WAIT. We need +this to deal with the case where the sync object is signalled through +a deferred SYNC_{ADD,SET} from the same queue. If we don't do that +and the group gets scheduled out before the deferred SYNC_{SET,ADD} +is executed, we'll end up with a timeout, because no external +SYNC_{SET,ADD} will make the scheduler reconsider the group for +execution. + +Fixes: de8548813824 ("drm/panthor: Add the scheduler logical block") +Cc: +Signed-off-by: Boris Brezillon +Reviewed-by: Steven Price +Reviewed-by: Liviu Dudau +--- + drivers/gpu/drm/panthor/panthor_sched.c | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/drivers/gpu/drm/panthor/panthor_sched.c b/drivers/gpu/drm/panthor/panthor_sched.c +index 42138b69c70422..37d8a2d97c4773 100644 +--- a/drivers/gpu/drm/panthor/panthor_sched.c ++++ b/drivers/gpu/drm/panthor/panthor_sched.c +@@ -1101,7 +1101,13 @@ cs_slot_sync_queue_state_locked(struct panthor_device *ptdev, u32 csg_id, u32 cs + list_move_tail(&group->wait_node, + &group->ptdev->scheduler->groups.waiting); + } +- group->blocked_queues |= BIT(cs_id); ++ ++ /* The queue is only blocked if there's no deferred operation ++ * pending, which can be checked through the scoreboard status. ++ */ ++ if (!cs_iface->output->status_scoreboards) ++ group->blocked_queues |= BIT(cs_id); ++ + queue->syncwait.gpu_va = cs_iface->output->status_wait_sync_ptr; + queue->syncwait.ref = cs_iface->output->status_wait_sync_value; + status_wait_cond = cs_iface->output->status_wait & CS_STATUS_WAIT_SYNC_COND_MASK; diff --git a/pkgs/kernel/vendor.nix b/pkgs/kernel/vendor.nix index b3e3e51..e367106 100644 --- a/pkgs/kernel/vendor.nix +++ b/pkgs/kernel/vendor.nix @@ -15,9 +15,6 @@ }: let modDirVersion = "6.1.75"; - - panthor-base = "aa54fa4e0712616d44f2c2f312ecc35c0827833d"; - panthor-head = "c81ebd8e12b64a42a6efd68cc0ed018b57d14e91"; in (linuxManualConfig { inherit modDirVersion; @@ -36,12 +33,9 @@ in # allows usage of mainline mesa kernelPatches = [{ name = "hbiyik-panthor.patch"; - # NOTE: This needs to be `fetchurl` instead of `fetchpatch`, because `fetchpatch` - # reorders the patches, and the order matters since they're generated from commits. - patch = fetchurl { - url = "https://github.com/hbiyik/linux/compare/${panthor-base}...${panthor-head}.patch"; - hash = "sha256-/5SvlGsgHbn1i68+GASOeNZmxoZiIt280L6iUFz3MFU="; - }; + # Generate using this command: + # curl -o hbiyik-panthor.patch -L https://github.com/hbiyik/linux/compare/aa54fa4e0712616d44f2c2f312ecc35c0827833d...c81ebd8e12b64a42a6efd68cc0ed018b57d14e91.patch + patch = ./hbiyik-panthor.patch; extraConfig = { }; }];