1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Generic ASID allocator. |
4 | * |
5 | * Based on arch/arm/mm/context.c |
6 | * |
7 | * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved. |
8 | * Copyright (C) 2012 ARM Ltd. |
9 | */ |
10 | |
11 | #include <linux/slab.h> |
12 | #include <linux/mm_types.h> |
13 | |
14 | #include <asm/asid.h> |
15 | |
16 | #define reserved_asid(info, cpu) *per_cpu_ptr((info)->reserved, cpu) |
17 | |
18 | #define ASID_MASK(info) (~GENMASK((info)->bits - 1, 0)) |
19 | #define ASID_FIRST_VERSION(info) (1UL << ((info)->bits)) |
20 | |
21 | #define asid2idx(info, asid) (((asid) & ~ASID_MASK(info)) >> (info)->ctxt_shift) |
22 | #define idx2asid(info, idx) (((idx) << (info)->ctxt_shift) & ~ASID_MASK(info)) |
23 | |
24 | static void flush_context(struct asid_info *info) |
25 | { |
26 | int i; |
27 | u64 asid; |
28 | |
29 | /* Update the list of reserved ASIDs and the ASID bitmap. */ |
30 | bitmap_zero(dst: info->map, nbits: NUM_CTXT_ASIDS(info)); |
31 | |
32 | for_each_possible_cpu(i) { |
33 | asid = atomic64_xchg_relaxed(&active_asid(info, i)v: , new: 0); |
34 | /* |
35 | * If this CPU has already been through a |
36 | * rollover, but hasn't run another task in |
37 | * the meantime, we must preserve its reserved |
38 | * ASID, as this is the only trace we have of |
39 | * the process it is still running. |
40 | */ |
41 | if (asid == 0) |
42 | asid = reserved_asid(info, i); |
43 | __set_bit(asid2idx(info, asid), info->map); |
44 | reserved_asid(info, i) = asid; |
45 | } |
46 | |
47 | /* |
48 | * Queue a TLB invalidation for each CPU to perform on next |
49 | * context-switch |
50 | */ |
51 | cpumask_setall(dstp: &info->flush_pending); |
52 | } |
53 | |
54 | static bool check_update_reserved_asid(struct asid_info *info, u64 asid, |
55 | u64 newasid) |
56 | { |
57 | int cpu; |
58 | bool hit = false; |
59 | |
60 | /* |
61 | * Iterate over the set of reserved ASIDs looking for a match. |
62 | * If we find one, then we can update our mm to use newasid |
63 | * (i.e. the same ASID in the current generation) but we can't |
64 | * exit the loop early, since we need to ensure that all copies |
65 | * of the old ASID are updated to reflect the mm. Failure to do |
66 | * so could result in us missing the reserved ASID in a future |
67 | * generation. |
68 | */ |
69 | for_each_possible_cpu(cpu) { |
70 | if (reserved_asid(info, cpu) == asid) { |
71 | hit = true; |
72 | reserved_asid(info, cpu) = newasid; |
73 | } |
74 | } |
75 | |
76 | return hit; |
77 | } |
78 | |
79 | static u64 new_context(struct asid_info *info, atomic64_t *pasid, |
80 | struct mm_struct *mm) |
81 | { |
82 | static u32 cur_idx = 1; |
83 | u64 asid = atomic64_read(v: pasid); |
84 | u64 generation = atomic64_read(v: &info->generation); |
85 | |
86 | if (asid != 0) { |
87 | u64 newasid = generation | (asid & ~ASID_MASK(info)); |
88 | |
89 | /* |
90 | * If our current ASID was active during a rollover, we |
91 | * can continue to use it and this was just a false alarm. |
92 | */ |
93 | if (check_update_reserved_asid(info, asid, newasid)) |
94 | return newasid; |
95 | |
96 | /* |
97 | * We had a valid ASID in a previous life, so try to re-use |
98 | * it if possible. |
99 | */ |
100 | if (!__test_and_set_bit(asid2idx(info, asid), info->map)) |
101 | return newasid; |
102 | } |
103 | |
104 | /* |
105 | * Allocate a free ASID. If we can't find one, take a note of the |
106 | * currently active ASIDs and mark the TLBs as requiring flushes. We |
107 | * always count from ASID #2 (index 1), as we use ASID #0 when setting |
108 | * a reserved TTBR0 for the init_mm and we allocate ASIDs in even/odd |
109 | * pairs. |
110 | */ |
111 | asid = find_next_zero_bit(addr: info->map, size: NUM_CTXT_ASIDS(info), offset: cur_idx); |
112 | if (asid != NUM_CTXT_ASIDS(info)) |
113 | goto set_asid; |
114 | |
115 | /* We're out of ASIDs, so increment the global generation count */ |
116 | generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION(info), |
117 | v: &info->generation); |
118 | flush_context(info); |
119 | |
120 | /* We have more ASIDs than CPUs, so this will always succeed */ |
121 | asid = find_next_zero_bit(addr: info->map, size: NUM_CTXT_ASIDS(info), offset: 1); |
122 | |
123 | set_asid: |
124 | __set_bit(asid, info->map); |
125 | cur_idx = asid; |
126 | cpumask_clear(dstp: mm_cpumask(mm)); |
127 | return idx2asid(info, asid) | generation; |
128 | } |
129 | |
130 | /* |
131 | * Generate a new ASID for the context. |
132 | * |
133 | * @pasid: Pointer to the current ASID batch allocated. It will be updated |
134 | * with the new ASID batch. |
135 | * @cpu: current CPU ID. Must have been acquired through get_cpu() |
136 | */ |
137 | void asid_new_context(struct asid_info *info, atomic64_t *pasid, |
138 | unsigned int cpu, struct mm_struct *mm) |
139 | { |
140 | unsigned long flags; |
141 | u64 asid; |
142 | |
143 | raw_spin_lock_irqsave(&info->lock, flags); |
144 | /* Check that our ASID belongs to the current generation. */ |
145 | asid = atomic64_read(v: pasid); |
146 | if ((asid ^ atomic64_read(v: &info->generation)) >> info->bits) { |
147 | asid = new_context(info, pasid, mm); |
148 | atomic64_set(v: pasid, i: asid); |
149 | } |
150 | |
151 | if (cpumask_test_and_clear_cpu(cpu, cpumask: &info->flush_pending)) |
152 | info->flush_cpu_ctxt_cb(); |
153 | |
154 | atomic64_set(&active_asid(info, cpu)v: , i: asid); |
155 | cpumask_set_cpu(cpu, dstp: mm_cpumask(mm)); |
156 | raw_spin_unlock_irqrestore(&info->lock, flags); |
157 | } |
158 | |
159 | /* |
160 | * Initialize the ASID allocator |
161 | * |
162 | * @info: Pointer to the asid allocator structure |
163 | * @bits: Number of ASIDs available |
164 | * @asid_per_ctxt: Number of ASIDs to allocate per-context. ASIDs are |
165 | * allocated contiguously for a given context. This value should be a power of |
166 | * 2. |
167 | */ |
168 | int asid_allocator_init(struct asid_info *info, |
169 | u32 bits, unsigned int asid_per_ctxt, |
170 | void (*flush_cpu_ctxt_cb)(void)) |
171 | { |
172 | info->bits = bits; |
173 | info->ctxt_shift = ilog2(asid_per_ctxt); |
174 | info->flush_cpu_ctxt_cb = flush_cpu_ctxt_cb; |
175 | /* |
176 | * Expect allocation after rollover to fail if we don't have at least |
177 | * one more ASID than CPUs. ASID #0 is always reserved. |
178 | */ |
179 | WARN_ON(NUM_CTXT_ASIDS(info) - 1 <= num_possible_cpus()); |
180 | atomic64_set(v: &info->generation, ASID_FIRST_VERSION(info)); |
181 | info->map = bitmap_zalloc(nbits: NUM_CTXT_ASIDS(info), GFP_KERNEL); |
182 | if (!info->map) |
183 | return -ENOMEM; |
184 | |
185 | raw_spin_lock_init(&info->lock); |
186 | |
187 | return 0; |
188 | } |
189 | |