1 | /* |
2 | * Copyright 2016-2018 Advanced Micro Devices, Inc. |
3 | * |
4 | * Permission is hereby granted, free of charge, to any person obtaining a |
5 | * copy of this software and associated documentation files (the "Software"), |
6 | * to deal in the Software without restriction, including without limitation |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
8 | * and/or sell copies of the Software, and to permit persons to whom the |
9 | * Software is furnished to do so, subject to the following conditions: |
10 | * |
11 | * The above copyright notice and this permission notice shall be included in |
12 | * all copies or substantial portions of the Software. |
13 | * |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR |
20 | * OTHER DEALINGS IN THE SOFTWARE. |
21 | */ |
22 | |
23 | #include <linux/dma-fence.h> |
24 | #include <linux/spinlock.h> |
25 | #include <linux/atomic.h> |
26 | #include <linux/stacktrace.h> |
27 | #include <linux/sched.h> |
28 | #include <linux/slab.h> |
29 | #include <linux/sched/mm.h> |
30 | #include "amdgpu_amdkfd.h" |
31 | #include "kfd_svm.h" |
32 | |
33 | static const struct dma_fence_ops amdkfd_fence_ops; |
34 | static atomic_t fence_seq = ATOMIC_INIT(0); |
35 | |
36 | /* Eviction Fence |
37 | * Fence helper functions to deal with KFD memory eviction. |
38 | * Big Idea - Since KFD submissions are done by user queues, a BO cannot be |
39 | * evicted unless all the user queues for that process are evicted. |
40 | * |
41 | * All the BOs in a process share an eviction fence. When process X wants |
42 | * to map VRAM memory but TTM can't find enough space, TTM will attempt to |
43 | * evict BOs from its LRU list. TTM checks if the BO is valuable to evict |
44 | * by calling ttm_device_funcs->eviction_valuable(). |
45 | * |
46 | * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs |
47 | * to process X. Otherwise, it will return true to indicate BO can be |
48 | * evicted by TTM. |
49 | * |
50 | * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue |
51 | * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move |
52 | * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler. |
53 | * |
54 | * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to |
55 | * nofity when the BO is free to move. fence_add_callback --> enable_signaling |
56 | * --> amdgpu_amdkfd_fence.enable_signaling |
57 | * |
58 | * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce |
59 | * user queues and signal fence. The work item will also start another delayed |
60 | * work item to restore BOs |
61 | */ |
62 | |
63 | struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, |
64 | struct mm_struct *mm, |
65 | struct svm_range_bo *svm_bo) |
66 | { |
67 | struct amdgpu_amdkfd_fence *fence; |
68 | |
69 | fence = kzalloc(size: sizeof(*fence), GFP_KERNEL); |
70 | if (fence == NULL) |
71 | return NULL; |
72 | |
73 | /* This reference gets released in amdkfd_fence_release */ |
74 | mmgrab(mm); |
75 | fence->mm = mm; |
76 | get_task_comm(fence->timeline_name, current); |
77 | spin_lock_init(&fence->lock); |
78 | fence->svm_bo = svm_bo; |
79 | dma_fence_init(fence: &fence->base, ops: &amdkfd_fence_ops, lock: &fence->lock, |
80 | context, seqno: atomic_inc_return(v: &fence_seq)); |
81 | |
82 | return fence; |
83 | } |
84 | |
85 | struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) |
86 | { |
87 | struct amdgpu_amdkfd_fence *fence; |
88 | |
89 | if (!f) |
90 | return NULL; |
91 | |
92 | fence = container_of(f, struct amdgpu_amdkfd_fence, base); |
93 | if (fence && f->ops == &amdkfd_fence_ops) |
94 | return fence; |
95 | |
96 | return NULL; |
97 | } |
98 | |
99 | static const char *amdkfd_fence_get_driver_name(struct dma_fence *f) |
100 | { |
101 | return "amdgpu_amdkfd_fence" ; |
102 | } |
103 | |
104 | static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f) |
105 | { |
106 | struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); |
107 | |
108 | return fence->timeline_name; |
109 | } |
110 | |
111 | /** |
112 | * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict |
113 | * a KFD BO and schedules a job to move the BO. |
114 | * If fence is already signaled return true. |
115 | * If fence is not signaled schedule a evict KFD process work item. |
116 | * |
117 | * @f: dma_fence |
118 | */ |
119 | static bool amdkfd_fence_enable_signaling(struct dma_fence *f) |
120 | { |
121 | struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); |
122 | |
123 | if (!fence) |
124 | return false; |
125 | |
126 | if (dma_fence_is_signaled(fence: f)) |
127 | return true; |
128 | |
129 | if (!fence->svm_bo) { |
130 | if (!kgd2kfd_schedule_evict_and_restore_process(mm: fence->mm, fence: f)) |
131 | return true; |
132 | } else { |
133 | if (!svm_range_schedule_evict_svm_bo(fence)) |
134 | return true; |
135 | } |
136 | return false; |
137 | } |
138 | |
139 | /** |
140 | * amdkfd_fence_release - callback that fence can be freed |
141 | * |
142 | * @f: dma_fence |
143 | * |
144 | * This function is called when the reference count becomes zero. |
145 | * Drops the mm_struct reference and RCU schedules freeing up the fence. |
146 | */ |
147 | static void amdkfd_fence_release(struct dma_fence *f) |
148 | { |
149 | struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); |
150 | |
151 | /* Unconditionally signal the fence. The process is getting |
152 | * terminated. |
153 | */ |
154 | if (WARN_ON(!fence)) |
155 | return; /* Not an amdgpu_amdkfd_fence */ |
156 | |
157 | mmdrop(mm: fence->mm); |
158 | kfree_rcu(f, rcu); |
159 | } |
160 | |
161 | /** |
162 | * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm |
163 | * |
164 | * @f: [IN] fence |
165 | * @mm: [IN] mm that needs to be verified |
166 | * |
167 | * Check if @mm is same as that of the fence @f, if same return TRUE else |
168 | * return FALSE. |
169 | * For svm bo, which support vram overcommitment, always return FALSE. |
170 | */ |
171 | bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) |
172 | { |
173 | struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); |
174 | |
175 | if (!fence) |
176 | return false; |
177 | else if (fence->mm == mm && !fence->svm_bo) |
178 | return true; |
179 | |
180 | return false; |
181 | } |
182 | |
183 | static const struct dma_fence_ops amdkfd_fence_ops = { |
184 | .get_driver_name = amdkfd_fence_get_driver_name, |
185 | .get_timeline_name = amdkfd_fence_get_timeline_name, |
186 | .enable_signaling = amdkfd_fence_enable_signaling, |
187 | .release = amdkfd_fence_release, |
188 | }; |
189 | |