1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Implement the default iomap interfaces |
4 | * |
5 | * (C) Copyright 2004 Linus Torvalds |
6 | */ |
7 | #include <linux/pci.h> |
8 | #include <linux/io.h> |
9 | #include <linux/kmsan-checks.h> |
10 | |
11 | #include <linux/export.h> |
12 | |
13 | /* |
14 | * Read/write from/to an (offsettable) iomem cookie. It might be a PIO |
15 | * access or a MMIO access, these functions don't care. The info is |
16 | * encoded in the hardware mapping set up by the mapping functions |
17 | * (or the cookie itself, depending on implementation and hw). |
18 | * |
19 | * The generic routines don't assume any hardware mappings, and just |
20 | * encode the PIO/MMIO as part of the cookie. They coldly assume that |
21 | * the MMIO IO mappings are not in the low address range. |
22 | * |
23 | * Architectures for which this is not true can't use this generic |
24 | * implementation and should do their own copy. |
25 | */ |
26 | |
27 | #ifndef HAVE_ARCH_PIO_SIZE |
28 | /* |
29 | * We encode the physical PIO addresses (0-0xffff) into the |
30 | * pointer by offsetting them with a constant (0x10000) and |
31 | * assuming that all the low addresses are always PIO. That means |
32 | * we can do some sanity checks on the low bits, and don't |
33 | * need to just take things for granted. |
34 | */ |
35 | #define PIO_OFFSET 0x10000UL |
36 | #define PIO_MASK 0x0ffffUL |
37 | #define PIO_RESERVED 0x40000UL |
38 | #endif |
39 | |
40 | static void bad_io_access(unsigned long port, const char *access) |
41 | { |
42 | static int count = 10; |
43 | if (count) { |
44 | count--; |
45 | WARN(1, KERN_ERR "Bad IO access at port %#lx (%s)\n" , port, access); |
46 | } |
47 | } |
48 | |
49 | /* |
50 | * Ugly macros are a way of life. |
51 | */ |
52 | #define IO_COND(addr, is_pio, is_mmio) do { \ |
53 | unsigned long port = (unsigned long __force)addr; \ |
54 | if (port >= PIO_RESERVED) { \ |
55 | is_mmio; \ |
56 | } else if (port > PIO_OFFSET) { \ |
57 | port &= PIO_MASK; \ |
58 | is_pio; \ |
59 | } else \ |
60 | bad_io_access(port, #is_pio ); \ |
61 | } while (0) |
62 | |
63 | #ifndef pio_read16be |
64 | #define pio_read16be(port) swab16(inw(port)) |
65 | #define pio_read32be(port) swab32(inl(port)) |
66 | #endif |
67 | |
68 | #ifndef mmio_read16be |
69 | #define mmio_read16be(addr) swab16(readw(addr)) |
70 | #define mmio_read32be(addr) swab32(readl(addr)) |
71 | #define mmio_read64be(addr) swab64(readq(addr)) |
72 | #endif |
73 | |
74 | /* |
75 | * Here and below, we apply __no_kmsan_checks to functions reading data from |
76 | * hardware, to ensure that KMSAN marks their return values as initialized. |
77 | */ |
78 | __no_kmsan_checks |
79 | unsigned int ioread8(const void __iomem *addr) |
80 | { |
81 | IO_COND(addr, return inb(port), return readb(addr)); |
82 | return 0xff; |
83 | } |
84 | __no_kmsan_checks |
85 | unsigned int ioread16(const void __iomem *addr) |
86 | { |
87 | IO_COND(addr, return inw(port), return readw(addr)); |
88 | return 0xffff; |
89 | } |
90 | __no_kmsan_checks |
91 | unsigned int ioread16be(const void __iomem *addr) |
92 | { |
93 | IO_COND(addr, return pio_read16be(port), return mmio_read16be(addr)); |
94 | return 0xffff; |
95 | } |
96 | __no_kmsan_checks |
97 | unsigned int ioread32(const void __iomem *addr) |
98 | { |
99 | IO_COND(addr, return inl(port), return readl(addr)); |
100 | return 0xffffffff; |
101 | } |
102 | __no_kmsan_checks |
103 | unsigned int ioread32be(const void __iomem *addr) |
104 | { |
105 | IO_COND(addr, return pio_read32be(port), return mmio_read32be(addr)); |
106 | return 0xffffffff; |
107 | } |
108 | EXPORT_SYMBOL(ioread8); |
109 | EXPORT_SYMBOL(ioread16); |
110 | EXPORT_SYMBOL(ioread16be); |
111 | EXPORT_SYMBOL(ioread32); |
112 | EXPORT_SYMBOL(ioread32be); |
113 | |
114 | #ifdef readq |
115 | static u64 pio_read64_lo_hi(unsigned long port) |
116 | { |
117 | u64 lo, hi; |
118 | |
119 | lo = inl(port); |
120 | hi = inl(port: port + sizeof(u32)); |
121 | |
122 | return lo | (hi << 32); |
123 | } |
124 | |
125 | static u64 pio_read64_hi_lo(unsigned long port) |
126 | { |
127 | u64 lo, hi; |
128 | |
129 | hi = inl(port: port + sizeof(u32)); |
130 | lo = inl(port); |
131 | |
132 | return lo | (hi << 32); |
133 | } |
134 | |
135 | static u64 pio_read64be_lo_hi(unsigned long port) |
136 | { |
137 | u64 lo, hi; |
138 | |
139 | lo = pio_read32be(port + sizeof(u32)); |
140 | hi = pio_read32be(port); |
141 | |
142 | return lo | (hi << 32); |
143 | } |
144 | |
145 | static u64 pio_read64be_hi_lo(unsigned long port) |
146 | { |
147 | u64 lo, hi; |
148 | |
149 | hi = pio_read32be(port); |
150 | lo = pio_read32be(port + sizeof(u32)); |
151 | |
152 | return lo | (hi << 32); |
153 | } |
154 | |
155 | __no_kmsan_checks |
156 | u64 ioread64_lo_hi(const void __iomem *addr) |
157 | { |
158 | IO_COND(addr, return pio_read64_lo_hi(port), return readq(addr)); |
159 | return 0xffffffffffffffffULL; |
160 | } |
161 | |
162 | __no_kmsan_checks |
163 | u64 ioread64_hi_lo(const void __iomem *addr) |
164 | { |
165 | IO_COND(addr, return pio_read64_hi_lo(port), return readq(addr)); |
166 | return 0xffffffffffffffffULL; |
167 | } |
168 | |
169 | __no_kmsan_checks |
170 | u64 ioread64be_lo_hi(const void __iomem *addr) |
171 | { |
172 | IO_COND(addr, return pio_read64be_lo_hi(port), |
173 | return mmio_read64be(addr)); |
174 | return 0xffffffffffffffffULL; |
175 | } |
176 | |
177 | __no_kmsan_checks |
178 | u64 ioread64be_hi_lo(const void __iomem *addr) |
179 | { |
180 | IO_COND(addr, return pio_read64be_hi_lo(port), |
181 | return mmio_read64be(addr)); |
182 | return 0xffffffffffffffffULL; |
183 | } |
184 | |
185 | EXPORT_SYMBOL(ioread64_lo_hi); |
186 | EXPORT_SYMBOL(ioread64_hi_lo); |
187 | EXPORT_SYMBOL(ioread64be_lo_hi); |
188 | EXPORT_SYMBOL(ioread64be_hi_lo); |
189 | |
190 | #endif /* readq */ |
191 | |
192 | #ifndef pio_write16be |
193 | #define pio_write16be(val,port) outw(swab16(val),port) |
194 | #define pio_write32be(val,port) outl(swab32(val),port) |
195 | #endif |
196 | |
197 | #ifndef mmio_write16be |
198 | #define mmio_write16be(val,port) writew(swab16(val),port) |
199 | #define mmio_write32be(val,port) writel(swab32(val),port) |
200 | #define mmio_write64be(val,port) writeq(swab64(val),port) |
201 | #endif |
202 | |
203 | void iowrite8(u8 val, void __iomem *addr) |
204 | { |
205 | /* Make sure uninitialized memory isn't copied to devices. */ |
206 | kmsan_check_memory(address: &val, size: sizeof(val)); |
207 | IO_COND(addr, outb(val,port), writeb(val, addr)); |
208 | } |
209 | void iowrite16(u16 val, void __iomem *addr) |
210 | { |
211 | /* Make sure uninitialized memory isn't copied to devices. */ |
212 | kmsan_check_memory(address: &val, size: sizeof(val)); |
213 | IO_COND(addr, outw(val,port), writew(val, addr)); |
214 | } |
215 | void iowrite16be(u16 val, void __iomem *addr) |
216 | { |
217 | /* Make sure uninitialized memory isn't copied to devices. */ |
218 | kmsan_check_memory(address: &val, size: sizeof(val)); |
219 | IO_COND(addr, pio_write16be(val,port), mmio_write16be(val, addr)); |
220 | } |
221 | void iowrite32(u32 val, void __iomem *addr) |
222 | { |
223 | /* Make sure uninitialized memory isn't copied to devices. */ |
224 | kmsan_check_memory(address: &val, size: sizeof(val)); |
225 | IO_COND(addr, outl(val,port), writel(val, addr)); |
226 | } |
227 | void iowrite32be(u32 val, void __iomem *addr) |
228 | { |
229 | /* Make sure uninitialized memory isn't copied to devices. */ |
230 | kmsan_check_memory(address: &val, size: sizeof(val)); |
231 | IO_COND(addr, pio_write32be(val,port), mmio_write32be(val, addr)); |
232 | } |
233 | EXPORT_SYMBOL(iowrite8); |
234 | EXPORT_SYMBOL(iowrite16); |
235 | EXPORT_SYMBOL(iowrite16be); |
236 | EXPORT_SYMBOL(iowrite32); |
237 | EXPORT_SYMBOL(iowrite32be); |
238 | |
239 | #ifdef writeq |
240 | static void pio_write64_lo_hi(u64 val, unsigned long port) |
241 | { |
242 | outl(value: val, port); |
243 | outl(value: val >> 32, port: port + sizeof(u32)); |
244 | } |
245 | |
246 | static void pio_write64_hi_lo(u64 val, unsigned long port) |
247 | { |
248 | outl(value: val >> 32, port: port + sizeof(u32)); |
249 | outl(value: val, port); |
250 | } |
251 | |
252 | static void pio_write64be_lo_hi(u64 val, unsigned long port) |
253 | { |
254 | pio_write32be(val, port + sizeof(u32)); |
255 | pio_write32be(val >> 32, port); |
256 | } |
257 | |
258 | static void pio_write64be_hi_lo(u64 val, unsigned long port) |
259 | { |
260 | pio_write32be(val >> 32, port); |
261 | pio_write32be(val, port + sizeof(u32)); |
262 | } |
263 | |
264 | void iowrite64_lo_hi(u64 val, void __iomem *addr) |
265 | { |
266 | /* Make sure uninitialized memory isn't copied to devices. */ |
267 | kmsan_check_memory(address: &val, size: sizeof(val)); |
268 | IO_COND(addr, pio_write64_lo_hi(val, port), |
269 | writeq(val, addr)); |
270 | } |
271 | |
272 | void iowrite64_hi_lo(u64 val, void __iomem *addr) |
273 | { |
274 | /* Make sure uninitialized memory isn't copied to devices. */ |
275 | kmsan_check_memory(address: &val, size: sizeof(val)); |
276 | IO_COND(addr, pio_write64_hi_lo(val, port), |
277 | writeq(val, addr)); |
278 | } |
279 | |
280 | void iowrite64be_lo_hi(u64 val, void __iomem *addr) |
281 | { |
282 | /* Make sure uninitialized memory isn't copied to devices. */ |
283 | kmsan_check_memory(address: &val, size: sizeof(val)); |
284 | IO_COND(addr, pio_write64be_lo_hi(val, port), |
285 | mmio_write64be(val, addr)); |
286 | } |
287 | |
288 | void iowrite64be_hi_lo(u64 val, void __iomem *addr) |
289 | { |
290 | /* Make sure uninitialized memory isn't copied to devices. */ |
291 | kmsan_check_memory(address: &val, size: sizeof(val)); |
292 | IO_COND(addr, pio_write64be_hi_lo(val, port), |
293 | mmio_write64be(val, addr)); |
294 | } |
295 | |
296 | EXPORT_SYMBOL(iowrite64_lo_hi); |
297 | EXPORT_SYMBOL(iowrite64_hi_lo); |
298 | EXPORT_SYMBOL(iowrite64be_lo_hi); |
299 | EXPORT_SYMBOL(iowrite64be_hi_lo); |
300 | |
301 | #endif /* readq */ |
302 | |
303 | /* |
304 | * These are the "repeat MMIO read/write" functions. |
305 | * Note the "__raw" accesses, since we don't want to |
306 | * convert to CPU byte order. We write in "IO byte |
307 | * order" (we also don't have IO barriers). |
308 | */ |
309 | #ifndef mmio_insb |
310 | static inline void mmio_insb(const void __iomem *addr, u8 *dst, int count) |
311 | { |
312 | while (--count >= 0) { |
313 | u8 data = __raw_readb(addr); |
314 | *dst = data; |
315 | dst++; |
316 | } |
317 | } |
318 | static inline void mmio_insw(const void __iomem *addr, u16 *dst, int count) |
319 | { |
320 | while (--count >= 0) { |
321 | u16 data = __raw_readw(addr); |
322 | *dst = data; |
323 | dst++; |
324 | } |
325 | } |
326 | static inline void mmio_insl(const void __iomem *addr, u32 *dst, int count) |
327 | { |
328 | while (--count >= 0) { |
329 | u32 data = __raw_readl(addr); |
330 | *dst = data; |
331 | dst++; |
332 | } |
333 | } |
334 | #endif |
335 | |
336 | #ifndef mmio_outsb |
337 | static inline void mmio_outsb(void __iomem *addr, const u8 *src, int count) |
338 | { |
339 | while (--count >= 0) { |
340 | __raw_writeb(val: *src, addr); |
341 | src++; |
342 | } |
343 | } |
344 | static inline void mmio_outsw(void __iomem *addr, const u16 *src, int count) |
345 | { |
346 | while (--count >= 0) { |
347 | __raw_writew(val: *src, addr); |
348 | src++; |
349 | } |
350 | } |
351 | static inline void mmio_outsl(void __iomem *addr, const u32 *src, int count) |
352 | { |
353 | while (--count >= 0) { |
354 | __raw_writel(val: *src, addr); |
355 | src++; |
356 | } |
357 | } |
358 | #endif |
359 | |
360 | void ioread8_rep(const void __iomem *addr, void *dst, unsigned long count) |
361 | { |
362 | IO_COND(addr, insb(port,dst,count), mmio_insb(addr, dst, count)); |
363 | /* KMSAN must treat values read from devices as initialized. */ |
364 | kmsan_unpoison_memory(address: dst, size: count); |
365 | } |
366 | void ioread16_rep(const void __iomem *addr, void *dst, unsigned long count) |
367 | { |
368 | IO_COND(addr, insw(port,dst,count), mmio_insw(addr, dst, count)); |
369 | /* KMSAN must treat values read from devices as initialized. */ |
370 | kmsan_unpoison_memory(address: dst, size: count * 2); |
371 | } |
372 | void ioread32_rep(const void __iomem *addr, void *dst, unsigned long count) |
373 | { |
374 | IO_COND(addr, insl(port,dst,count), mmio_insl(addr, dst, count)); |
375 | /* KMSAN must treat values read from devices as initialized. */ |
376 | kmsan_unpoison_memory(address: dst, size: count * 4); |
377 | } |
378 | EXPORT_SYMBOL(ioread8_rep); |
379 | EXPORT_SYMBOL(ioread16_rep); |
380 | EXPORT_SYMBOL(ioread32_rep); |
381 | |
382 | void iowrite8_rep(void __iomem *addr, const void *src, unsigned long count) |
383 | { |
384 | /* Make sure uninitialized memory isn't copied to devices. */ |
385 | kmsan_check_memory(address: src, size: count); |
386 | IO_COND(addr, outsb(port, src, count), mmio_outsb(addr, src, count)); |
387 | } |
388 | void iowrite16_rep(void __iomem *addr, const void *src, unsigned long count) |
389 | { |
390 | /* Make sure uninitialized memory isn't copied to devices. */ |
391 | kmsan_check_memory(address: src, size: count * 2); |
392 | IO_COND(addr, outsw(port, src, count), mmio_outsw(addr, src, count)); |
393 | } |
394 | void iowrite32_rep(void __iomem *addr, const void *src, unsigned long count) |
395 | { |
396 | /* Make sure uninitialized memory isn't copied to devices. */ |
397 | kmsan_check_memory(address: src, size: count * 4); |
398 | IO_COND(addr, outsl(port, src,count), mmio_outsl(addr, src, count)); |
399 | } |
400 | EXPORT_SYMBOL(iowrite8_rep); |
401 | EXPORT_SYMBOL(iowrite16_rep); |
402 | EXPORT_SYMBOL(iowrite32_rep); |
403 | |
404 | #ifdef CONFIG_HAS_IOPORT_MAP |
405 | /* Create a virtual mapping cookie for an IO port range */ |
406 | void __iomem *ioport_map(unsigned long port, unsigned int nr) |
407 | { |
408 | if (port > PIO_MASK) |
409 | return NULL; |
410 | return (void __iomem *) (unsigned long) (port + PIO_OFFSET); |
411 | } |
412 | |
413 | void ioport_unmap(void __iomem *addr) |
414 | { |
415 | /* Nothing to do */ |
416 | } |
417 | EXPORT_SYMBOL(ioport_map); |
418 | EXPORT_SYMBOL(ioport_unmap); |
419 | #endif /* CONFIG_HAS_IOPORT_MAP */ |
420 | |
421 | #ifdef CONFIG_PCI |
422 | /* Hide the details if this is a MMIO or PIO address space and just do what |
423 | * you expect in the correct way. */ |
424 | void pci_iounmap(struct pci_dev *dev, void __iomem * addr) |
425 | { |
426 | IO_COND(addr, /* nothing */, iounmap(addr)); |
427 | } |
428 | EXPORT_SYMBOL(pci_iounmap); |
429 | #endif /* CONFIG_PCI */ |
430 | |