1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2015, Sony Mobile Communications AB. |
4 | * Copyright (c) 2012-2013, The Linux Foundation. All rights reserved. |
5 | */ |
6 | |
7 | #include <linux/hwspinlock.h> |
8 | #include <linux/io.h> |
9 | #include <linux/module.h> |
10 | #include <linux/of.h> |
11 | #include <linux/of_address.h> |
12 | #include <linux/of_reserved_mem.h> |
13 | #include <linux/platform_device.h> |
14 | #include <linux/sizes.h> |
15 | #include <linux/slab.h> |
16 | #include <linux/soc/qcom/smem.h> |
17 | #include <linux/soc/qcom/socinfo.h> |
18 | |
19 | /* |
20 | * The Qualcomm shared memory system is a allocate only heap structure that |
21 | * consists of one of more memory areas that can be accessed by the processors |
22 | * in the SoC. |
23 | * |
24 | * All systems contains a global heap, accessible by all processors in the SoC, |
25 | * with a table of contents data structure (@smem_header) at the beginning of |
26 | * the main shared memory block. |
27 | * |
28 | * The global header contains meta data for allocations as well as a fixed list |
29 | * of 512 entries (@smem_global_entry) that can be initialized to reference |
30 | * parts of the shared memory space. |
31 | * |
32 | * |
33 | * In addition to this global heap a set of "private" heaps can be set up at |
34 | * boot time with access restrictions so that only certain processor pairs can |
35 | * access the data. |
36 | * |
37 | * These partitions are referenced from an optional partition table |
38 | * (@smem_ptable), that is found 4kB from the end of the main smem region. The |
39 | * partition table entries (@smem_ptable_entry) lists the involved processors |
40 | * (or hosts) and their location in the main shared memory region. |
41 | * |
42 | * Each partition starts with a header (@smem_partition_header) that identifies |
43 | * the partition and holds properties for the two internal memory regions. The |
44 | * two regions are cached and non-cached memory respectively. Each region |
45 | * contain a link list of allocation headers (@smem_private_entry) followed by |
46 | * their data. |
47 | * |
48 | * Items in the non-cached region are allocated from the start of the partition |
49 | * while items in the cached region are allocated from the end. The free area |
50 | * is hence the region between the cached and non-cached offsets. The header of |
51 | * cached items comes after the data. |
52 | * |
53 | * Version 12 (SMEM_GLOBAL_PART_VERSION) changes the item alloc/get procedure |
54 | * for the global heap. A new global partition is created from the global heap |
55 | * region with partition type (SMEM_GLOBAL_HOST) and the max smem item count is |
56 | * set by the bootloader. |
57 | * |
58 | * To synchronize allocations in the shared memory heaps a remote spinlock must |
59 | * be held - currently lock number 3 of the sfpb or tcsr is used for this on all |
60 | * platforms. |
61 | * |
62 | */ |
63 | |
64 | /* |
65 | * The version member of the smem header contains an array of versions for the |
66 | * various software components in the SoC. We verify that the boot loader |
67 | * version is a valid version as a sanity check. |
68 | */ |
69 | #define SMEM_MASTER_SBL_VERSION_INDEX 7 |
70 | #define SMEM_GLOBAL_HEAP_VERSION 11 |
71 | #define SMEM_GLOBAL_PART_VERSION 12 |
72 | |
73 | /* |
74 | * The first 8 items are only to be allocated by the boot loader while |
75 | * initializing the heap. |
76 | */ |
77 | #define SMEM_ITEM_LAST_FIXED 8 |
78 | |
79 | /* Highest accepted item number, for both global and private heaps */ |
80 | #define SMEM_ITEM_COUNT 512 |
81 | |
82 | /* Processor/host identifier for the application processor */ |
83 | #define SMEM_HOST_APPS 0 |
84 | |
85 | /* Processor/host identifier for the global partition */ |
86 | #define SMEM_GLOBAL_HOST 0xfffe |
87 | |
88 | /* Max number of processors/hosts in a system */ |
89 | #define SMEM_HOST_COUNT 20 |
90 | |
91 | /** |
92 | * struct smem_proc_comm - proc_comm communication struct (legacy) |
93 | * @command: current command to be executed |
94 | * @status: status of the currently requested command |
95 | * @params: parameters to the command |
96 | */ |
97 | struct smem_proc_comm { |
98 | __le32 command; |
99 | __le32 status; |
100 | __le32 params[2]; |
101 | }; |
102 | |
103 | /** |
104 | * struct smem_global_entry - entry to reference smem items on the heap |
105 | * @allocated: boolean to indicate if this entry is used |
106 | * @offset: offset to the allocated space |
107 | * @size: size of the allocated space, 8 byte aligned |
108 | * @aux_base: base address for the memory region used by this unit, or 0 for |
109 | * the default region. bits 0,1 are reserved |
110 | */ |
111 | struct smem_global_entry { |
112 | __le32 allocated; |
113 | __le32 offset; |
114 | __le32 size; |
115 | __le32 aux_base; /* bits 1:0 reserved */ |
116 | }; |
117 | #define AUX_BASE_MASK 0xfffffffc |
118 | |
119 | /** |
120 | * struct smem_header - header found in beginning of primary smem region |
121 | * @proc_comm: proc_comm communication interface (legacy) |
122 | * @version: array of versions for the various subsystems |
123 | * @initialized: boolean to indicate that smem is initialized |
124 | * @free_offset: index of the first unallocated byte in smem |
125 | * @available: number of bytes available for allocation |
126 | * @reserved: reserved field, must be 0 |
127 | * @toc: array of references to items |
128 | */ |
129 | struct { |
130 | struct smem_proc_comm [4]; |
131 | __le32 [32]; |
132 | __le32 ; |
133 | __le32 ; |
134 | __le32 ; |
135 | __le32 ; |
136 | struct smem_global_entry [SMEM_ITEM_COUNT]; |
137 | }; |
138 | |
139 | /** |
140 | * struct smem_ptable_entry - one entry in the @smem_ptable list |
141 | * @offset: offset, within the main shared memory region, of the partition |
142 | * @size: size of the partition |
143 | * @flags: flags for the partition (currently unused) |
144 | * @host0: first processor/host with access to this partition |
145 | * @host1: second processor/host with access to this partition |
146 | * @cacheline: alignment for "cached" entries |
147 | * @reserved: reserved entries for later use |
148 | */ |
149 | struct smem_ptable_entry { |
150 | __le32 offset; |
151 | __le32 size; |
152 | __le32 flags; |
153 | __le16 host0; |
154 | __le16 host1; |
155 | __le32 cacheline; |
156 | __le32 reserved[7]; |
157 | }; |
158 | |
159 | /** |
160 | * struct smem_ptable - partition table for the private partitions |
161 | * @magic: magic number, must be SMEM_PTABLE_MAGIC |
162 | * @version: version of the partition table |
163 | * @num_entries: number of partitions in the table |
164 | * @reserved: for now reserved entries |
165 | * @entry: list of @smem_ptable_entry for the @num_entries partitions |
166 | */ |
167 | struct smem_ptable { |
168 | u8 magic[4]; |
169 | __le32 version; |
170 | __le32 num_entries; |
171 | __le32 reserved[5]; |
172 | struct smem_ptable_entry entry[]; |
173 | }; |
174 | |
175 | static const u8 SMEM_PTABLE_MAGIC[] = { 0x24, 0x54, 0x4f, 0x43 }; /* "$TOC" */ |
176 | |
177 | /** |
178 | * struct smem_partition_header - header of the partitions |
179 | * @magic: magic number, must be SMEM_PART_MAGIC |
180 | * @host0: first processor/host with access to this partition |
181 | * @host1: second processor/host with access to this partition |
182 | * @size: size of the partition |
183 | * @offset_free_uncached: offset to the first free byte of uncached memory in |
184 | * this partition |
185 | * @offset_free_cached: offset to the first free byte of cached memory in this |
186 | * partition |
187 | * @reserved: for now reserved entries |
188 | */ |
189 | struct { |
190 | u8 [4]; |
191 | __le16 ; |
192 | __le16 ; |
193 | __le32 ; |
194 | __le32 ; |
195 | __le32 ; |
196 | __le32 [3]; |
197 | }; |
198 | |
199 | /** |
200 | * struct smem_partition - describes smem partition |
201 | * @virt_base: starting virtual address of partition |
202 | * @phys_base: starting physical address of partition |
203 | * @cacheline: alignment for "cached" entries |
204 | * @size: size of partition |
205 | */ |
206 | struct smem_partition { |
207 | void __iomem *virt_base; |
208 | phys_addr_t phys_base; |
209 | size_t cacheline; |
210 | size_t size; |
211 | }; |
212 | |
213 | static const u8 SMEM_PART_MAGIC[] = { 0x24, 0x50, 0x52, 0x54 }; |
214 | |
215 | /** |
216 | * struct smem_private_entry - header of each item in the private partition |
217 | * @canary: magic number, must be SMEM_PRIVATE_CANARY |
218 | * @item: identifying number of the smem item |
219 | * @size: size of the data, including padding bytes |
220 | * @padding_data: number of bytes of padding of data |
221 | * @padding_hdr: number of bytes of padding between the header and the data |
222 | * @reserved: for now reserved entry |
223 | */ |
224 | struct smem_private_entry { |
225 | u16 canary; /* bytes are the same so no swapping needed */ |
226 | __le16 item; |
227 | __le32 size; /* includes padding bytes */ |
228 | __le16 padding_data; |
229 | __le16 padding_hdr; |
230 | __le32 reserved; |
231 | }; |
232 | #define SMEM_PRIVATE_CANARY 0xa5a5 |
233 | |
234 | /** |
235 | * struct smem_info - smem region info located after the table of contents |
236 | * @magic: magic number, must be SMEM_INFO_MAGIC |
237 | * @size: size of the smem region |
238 | * @base_addr: base address of the smem region |
239 | * @reserved: for now reserved entry |
240 | * @num_items: highest accepted item number |
241 | */ |
242 | struct smem_info { |
243 | u8 magic[4]; |
244 | __le32 size; |
245 | __le32 base_addr; |
246 | __le32 reserved; |
247 | __le16 num_items; |
248 | }; |
249 | |
250 | static const u8 SMEM_INFO_MAGIC[] = { 0x53, 0x49, 0x49, 0x49 }; /* SIII */ |
251 | |
252 | /** |
253 | * struct smem_region - representation of a chunk of memory used for smem |
254 | * @aux_base: identifier of aux_mem base |
255 | * @virt_base: virtual base address of memory with this aux_mem identifier |
256 | * @size: size of the memory region |
257 | */ |
258 | struct smem_region { |
259 | phys_addr_t aux_base; |
260 | void __iomem *virt_base; |
261 | size_t size; |
262 | }; |
263 | |
264 | /** |
265 | * struct qcom_smem - device data for the smem device |
266 | * @dev: device pointer |
267 | * @hwlock: reference to a hwspinlock |
268 | * @ptable: virtual base of partition table |
269 | * @global_partition: describes for global partition when in use |
270 | * @partitions: list of partitions of current processor/host |
271 | * @item_count: max accepted item number |
272 | * @socinfo: platform device pointer |
273 | * @num_regions: number of @regions |
274 | * @regions: list of the memory regions defining the shared memory |
275 | */ |
276 | struct qcom_smem { |
277 | struct device *dev; |
278 | |
279 | struct hwspinlock *hwlock; |
280 | |
281 | u32 item_count; |
282 | struct platform_device *socinfo; |
283 | struct smem_ptable *ptable; |
284 | struct smem_partition global_partition; |
285 | struct smem_partition partitions[SMEM_HOST_COUNT]; |
286 | |
287 | unsigned num_regions; |
288 | struct smem_region regions[] __counted_by(num_regions); |
289 | }; |
290 | |
291 | static void * |
292 | phdr_to_last_uncached_entry(struct smem_partition_header *phdr) |
293 | { |
294 | void *p = phdr; |
295 | |
296 | return p + le32_to_cpu(phdr->offset_free_uncached); |
297 | } |
298 | |
299 | static struct smem_private_entry * |
300 | phdr_to_first_cached_entry(struct smem_partition_header *phdr, |
301 | size_t cacheline) |
302 | { |
303 | void *p = phdr; |
304 | struct smem_private_entry *e; |
305 | |
306 | return p + le32_to_cpu(phdr->size) - ALIGN(sizeof(*e), cacheline); |
307 | } |
308 | |
309 | static void * |
310 | phdr_to_last_cached_entry(struct smem_partition_header *phdr) |
311 | { |
312 | void *p = phdr; |
313 | |
314 | return p + le32_to_cpu(phdr->offset_free_cached); |
315 | } |
316 | |
317 | static struct smem_private_entry * |
318 | phdr_to_first_uncached_entry(struct smem_partition_header *phdr) |
319 | { |
320 | void *p = phdr; |
321 | |
322 | return p + sizeof(*phdr); |
323 | } |
324 | |
325 | static struct smem_private_entry * |
326 | uncached_entry_next(struct smem_private_entry *e) |
327 | { |
328 | void *p = e; |
329 | |
330 | return p + sizeof(*e) + le16_to_cpu(e->padding_hdr) + |
331 | le32_to_cpu(e->size); |
332 | } |
333 | |
334 | static struct smem_private_entry * |
335 | cached_entry_next(struct smem_private_entry *e, size_t cacheline) |
336 | { |
337 | void *p = e; |
338 | |
339 | return p - le32_to_cpu(e->size) - ALIGN(sizeof(*e), cacheline); |
340 | } |
341 | |
342 | static void *uncached_entry_to_item(struct smem_private_entry *e) |
343 | { |
344 | void *p = e; |
345 | |
346 | return p + sizeof(*e) + le16_to_cpu(e->padding_hdr); |
347 | } |
348 | |
349 | static void *cached_entry_to_item(struct smem_private_entry *e) |
350 | { |
351 | void *p = e; |
352 | |
353 | return p - le32_to_cpu(e->size); |
354 | } |
355 | |
356 | /* Pointer to the one and only smem handle */ |
357 | static struct qcom_smem *__smem; |
358 | |
359 | /* Timeout (ms) for the trylock of remote spinlocks */ |
360 | #define HWSPINLOCK_TIMEOUT 1000 |
361 | |
362 | /** |
363 | * qcom_smem_is_available() - Check if SMEM is available |
364 | * |
365 | * Return: true if SMEM is available, false otherwise. |
366 | */ |
367 | bool qcom_smem_is_available(void) |
368 | { |
369 | return !!__smem; |
370 | } |
371 | EXPORT_SYMBOL_GPL(qcom_smem_is_available); |
372 | |
373 | static int qcom_smem_alloc_private(struct qcom_smem *smem, |
374 | struct smem_partition *part, |
375 | unsigned item, |
376 | size_t size) |
377 | { |
378 | struct smem_private_entry *hdr, *end; |
379 | struct smem_partition_header *phdr; |
380 | size_t alloc_size; |
381 | void *cached; |
382 | void *p_end; |
383 | |
384 | phdr = (struct smem_partition_header __force *)part->virt_base; |
385 | p_end = (void *)phdr + part->size; |
386 | |
387 | hdr = phdr_to_first_uncached_entry(phdr); |
388 | end = phdr_to_last_uncached_entry(phdr); |
389 | cached = phdr_to_last_cached_entry(phdr); |
390 | |
391 | if (WARN_ON((void *)end > p_end || cached > p_end)) |
392 | return -EINVAL; |
393 | |
394 | while (hdr < end) { |
395 | if (hdr->canary != SMEM_PRIVATE_CANARY) |
396 | goto bad_canary; |
397 | if (le16_to_cpu(hdr->item) == item) |
398 | return -EEXIST; |
399 | |
400 | hdr = uncached_entry_next(e: hdr); |
401 | } |
402 | |
403 | if (WARN_ON((void *)hdr > p_end)) |
404 | return -EINVAL; |
405 | |
406 | /* Check that we don't grow into the cached region */ |
407 | alloc_size = sizeof(*hdr) + ALIGN(size, 8); |
408 | if ((void *)hdr + alloc_size > cached) { |
409 | dev_err(smem->dev, "Out of memory\n" ); |
410 | return -ENOSPC; |
411 | } |
412 | |
413 | hdr->canary = SMEM_PRIVATE_CANARY; |
414 | hdr->item = cpu_to_le16(item); |
415 | hdr->size = cpu_to_le32(ALIGN(size, 8)); |
416 | hdr->padding_data = cpu_to_le16(le32_to_cpu(hdr->size) - size); |
417 | hdr->padding_hdr = 0; |
418 | |
419 | /* |
420 | * Ensure the header is written before we advance the free offset, so |
421 | * that remote processors that does not take the remote spinlock still |
422 | * gets a consistent view of the linked list. |
423 | */ |
424 | wmb(); |
425 | le32_add_cpu(var: &phdr->offset_free_uncached, val: alloc_size); |
426 | |
427 | return 0; |
428 | bad_canary: |
429 | dev_err(smem->dev, "Found invalid canary in hosts %hu:%hu partition\n" , |
430 | le16_to_cpu(phdr->host0), le16_to_cpu(phdr->host1)); |
431 | |
432 | return -EINVAL; |
433 | } |
434 | |
435 | static int qcom_smem_alloc_global(struct qcom_smem *smem, |
436 | unsigned item, |
437 | size_t size) |
438 | { |
439 | struct smem_global_entry *entry; |
440 | struct smem_header *; |
441 | |
442 | header = smem->regions[0].virt_base; |
443 | entry = &header->toc[item]; |
444 | if (entry->allocated) |
445 | return -EEXIST; |
446 | |
447 | size = ALIGN(size, 8); |
448 | if (WARN_ON(size > le32_to_cpu(header->available))) |
449 | return -ENOMEM; |
450 | |
451 | entry->offset = header->free_offset; |
452 | entry->size = cpu_to_le32(size); |
453 | |
454 | /* |
455 | * Ensure the header is consistent before we mark the item allocated, |
456 | * so that remote processors will get a consistent view of the item |
457 | * even though they do not take the spinlock on read. |
458 | */ |
459 | wmb(); |
460 | entry->allocated = cpu_to_le32(1); |
461 | |
462 | le32_add_cpu(var: &header->free_offset, val: size); |
463 | le32_add_cpu(var: &header->available, val: -size); |
464 | |
465 | return 0; |
466 | } |
467 | |
468 | /** |
469 | * qcom_smem_alloc() - allocate space for a smem item |
470 | * @host: remote processor id, or -1 |
471 | * @item: smem item handle |
472 | * @size: number of bytes to be allocated |
473 | * |
474 | * Allocate space for a given smem item of size @size, given that the item is |
475 | * not yet allocated. |
476 | */ |
477 | int qcom_smem_alloc(unsigned host, unsigned item, size_t size) |
478 | { |
479 | struct smem_partition *part; |
480 | unsigned long flags; |
481 | int ret; |
482 | |
483 | if (!__smem) |
484 | return -EPROBE_DEFER; |
485 | |
486 | if (item < SMEM_ITEM_LAST_FIXED) { |
487 | dev_err(__smem->dev, |
488 | "Rejecting allocation of static entry %d\n" , item); |
489 | return -EINVAL; |
490 | } |
491 | |
492 | if (WARN_ON(item >= __smem->item_count)) |
493 | return -EINVAL; |
494 | |
495 | ret = hwspin_lock_timeout_irqsave(hwlock: __smem->hwlock, |
496 | HWSPINLOCK_TIMEOUT, |
497 | flags: &flags); |
498 | if (ret) |
499 | return ret; |
500 | |
501 | if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) { |
502 | part = &__smem->partitions[host]; |
503 | ret = qcom_smem_alloc_private(smem: __smem, part, item, size); |
504 | } else if (__smem->global_partition.virt_base) { |
505 | part = &__smem->global_partition; |
506 | ret = qcom_smem_alloc_private(smem: __smem, part, item, size); |
507 | } else { |
508 | ret = qcom_smem_alloc_global(smem: __smem, item, size); |
509 | } |
510 | |
511 | hwspin_unlock_irqrestore(hwlock: __smem->hwlock, flags: &flags); |
512 | |
513 | return ret; |
514 | } |
515 | EXPORT_SYMBOL_GPL(qcom_smem_alloc); |
516 | |
517 | static void *qcom_smem_get_global(struct qcom_smem *smem, |
518 | unsigned item, |
519 | size_t *size) |
520 | { |
521 | struct smem_header *; |
522 | struct smem_region *region; |
523 | struct smem_global_entry *entry; |
524 | u64 entry_offset; |
525 | u32 e_size; |
526 | u32 aux_base; |
527 | unsigned i; |
528 | |
529 | header = smem->regions[0].virt_base; |
530 | entry = &header->toc[item]; |
531 | if (!entry->allocated) |
532 | return ERR_PTR(error: -ENXIO); |
533 | |
534 | aux_base = le32_to_cpu(entry->aux_base) & AUX_BASE_MASK; |
535 | |
536 | for (i = 0; i < smem->num_regions; i++) { |
537 | region = &smem->regions[i]; |
538 | |
539 | if ((u32)region->aux_base == aux_base || !aux_base) { |
540 | e_size = le32_to_cpu(entry->size); |
541 | entry_offset = le32_to_cpu(entry->offset); |
542 | |
543 | if (WARN_ON(e_size + entry_offset > region->size)) |
544 | return ERR_PTR(error: -EINVAL); |
545 | |
546 | if (size != NULL) |
547 | *size = e_size; |
548 | |
549 | return region->virt_base + entry_offset; |
550 | } |
551 | } |
552 | |
553 | return ERR_PTR(error: -ENOENT); |
554 | } |
555 | |
556 | static void *qcom_smem_get_private(struct qcom_smem *smem, |
557 | struct smem_partition *part, |
558 | unsigned item, |
559 | size_t *size) |
560 | { |
561 | struct smem_private_entry *e, *end; |
562 | struct smem_partition_header *phdr; |
563 | void *item_ptr, *p_end; |
564 | u32 padding_data; |
565 | u32 e_size; |
566 | |
567 | phdr = (struct smem_partition_header __force *)part->virt_base; |
568 | p_end = (void *)phdr + part->size; |
569 | |
570 | e = phdr_to_first_uncached_entry(phdr); |
571 | end = phdr_to_last_uncached_entry(phdr); |
572 | |
573 | while (e < end) { |
574 | if (e->canary != SMEM_PRIVATE_CANARY) |
575 | goto invalid_canary; |
576 | |
577 | if (le16_to_cpu(e->item) == item) { |
578 | if (size != NULL) { |
579 | e_size = le32_to_cpu(e->size); |
580 | padding_data = le16_to_cpu(e->padding_data); |
581 | |
582 | if (WARN_ON(e_size > part->size || padding_data > e_size)) |
583 | return ERR_PTR(error: -EINVAL); |
584 | |
585 | *size = e_size - padding_data; |
586 | } |
587 | |
588 | item_ptr = uncached_entry_to_item(e); |
589 | if (WARN_ON(item_ptr > p_end)) |
590 | return ERR_PTR(error: -EINVAL); |
591 | |
592 | return item_ptr; |
593 | } |
594 | |
595 | e = uncached_entry_next(e); |
596 | } |
597 | |
598 | if (WARN_ON((void *)e > p_end)) |
599 | return ERR_PTR(error: -EINVAL); |
600 | |
601 | /* Item was not found in the uncached list, search the cached list */ |
602 | |
603 | e = phdr_to_first_cached_entry(phdr, cacheline: part->cacheline); |
604 | end = phdr_to_last_cached_entry(phdr); |
605 | |
606 | if (WARN_ON((void *)e < (void *)phdr || (void *)end > p_end)) |
607 | return ERR_PTR(error: -EINVAL); |
608 | |
609 | while (e > end) { |
610 | if (e->canary != SMEM_PRIVATE_CANARY) |
611 | goto invalid_canary; |
612 | |
613 | if (le16_to_cpu(e->item) == item) { |
614 | if (size != NULL) { |
615 | e_size = le32_to_cpu(e->size); |
616 | padding_data = le16_to_cpu(e->padding_data); |
617 | |
618 | if (WARN_ON(e_size > part->size || padding_data > e_size)) |
619 | return ERR_PTR(error: -EINVAL); |
620 | |
621 | *size = e_size - padding_data; |
622 | } |
623 | |
624 | item_ptr = cached_entry_to_item(e); |
625 | if (WARN_ON(item_ptr < (void *)phdr)) |
626 | return ERR_PTR(error: -EINVAL); |
627 | |
628 | return item_ptr; |
629 | } |
630 | |
631 | e = cached_entry_next(e, cacheline: part->cacheline); |
632 | } |
633 | |
634 | if (WARN_ON((void *)e < (void *)phdr)) |
635 | return ERR_PTR(error: -EINVAL); |
636 | |
637 | return ERR_PTR(error: -ENOENT); |
638 | |
639 | invalid_canary: |
640 | dev_err(smem->dev, "Found invalid canary in hosts %hu:%hu partition\n" , |
641 | le16_to_cpu(phdr->host0), le16_to_cpu(phdr->host1)); |
642 | |
643 | return ERR_PTR(error: -EINVAL); |
644 | } |
645 | |
646 | /** |
647 | * qcom_smem_get() - resolve ptr of size of a smem item |
648 | * @host: the remote processor, or -1 |
649 | * @item: smem item handle |
650 | * @size: pointer to be filled out with size of the item |
651 | * |
652 | * Looks up smem item and returns pointer to it. Size of smem |
653 | * item is returned in @size. |
654 | */ |
655 | void *qcom_smem_get(unsigned host, unsigned item, size_t *size) |
656 | { |
657 | struct smem_partition *part; |
658 | unsigned long flags; |
659 | int ret; |
660 | void *ptr = ERR_PTR(error: -EPROBE_DEFER); |
661 | |
662 | if (!__smem) |
663 | return ptr; |
664 | |
665 | if (WARN_ON(item >= __smem->item_count)) |
666 | return ERR_PTR(error: -EINVAL); |
667 | |
668 | ret = hwspin_lock_timeout_irqsave(hwlock: __smem->hwlock, |
669 | HWSPINLOCK_TIMEOUT, |
670 | flags: &flags); |
671 | if (ret) |
672 | return ERR_PTR(error: ret); |
673 | |
674 | if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) { |
675 | part = &__smem->partitions[host]; |
676 | ptr = qcom_smem_get_private(smem: __smem, part, item, size); |
677 | } else if (__smem->global_partition.virt_base) { |
678 | part = &__smem->global_partition; |
679 | ptr = qcom_smem_get_private(smem: __smem, part, item, size); |
680 | } else { |
681 | ptr = qcom_smem_get_global(smem: __smem, item, size); |
682 | } |
683 | |
684 | hwspin_unlock_irqrestore(hwlock: __smem->hwlock, flags: &flags); |
685 | |
686 | return ptr; |
687 | |
688 | } |
689 | EXPORT_SYMBOL_GPL(qcom_smem_get); |
690 | |
691 | /** |
692 | * qcom_smem_get_free_space() - retrieve amount of free space in a partition |
693 | * @host: the remote processor identifying a partition, or -1 |
694 | * |
695 | * To be used by smem clients as a quick way to determine if any new |
696 | * allocations has been made. |
697 | */ |
698 | int qcom_smem_get_free_space(unsigned host) |
699 | { |
700 | struct smem_partition *part; |
701 | struct smem_partition_header *phdr; |
702 | struct smem_header *; |
703 | unsigned ret; |
704 | |
705 | if (!__smem) |
706 | return -EPROBE_DEFER; |
707 | |
708 | if (host < SMEM_HOST_COUNT && __smem->partitions[host].virt_base) { |
709 | part = &__smem->partitions[host]; |
710 | phdr = part->virt_base; |
711 | ret = le32_to_cpu(phdr->offset_free_cached) - |
712 | le32_to_cpu(phdr->offset_free_uncached); |
713 | |
714 | if (ret > le32_to_cpu(part->size)) |
715 | return -EINVAL; |
716 | } else if (__smem->global_partition.virt_base) { |
717 | part = &__smem->global_partition; |
718 | phdr = part->virt_base; |
719 | ret = le32_to_cpu(phdr->offset_free_cached) - |
720 | le32_to_cpu(phdr->offset_free_uncached); |
721 | |
722 | if (ret > le32_to_cpu(part->size)) |
723 | return -EINVAL; |
724 | } else { |
725 | header = __smem->regions[0].virt_base; |
726 | ret = le32_to_cpu(header->available); |
727 | |
728 | if (ret > __smem->regions[0].size) |
729 | return -EINVAL; |
730 | } |
731 | |
732 | return ret; |
733 | } |
734 | EXPORT_SYMBOL_GPL(qcom_smem_get_free_space); |
735 | |
736 | static bool addr_in_range(void __iomem *base, size_t size, void *addr) |
737 | { |
738 | return base && ((void __iomem *)addr >= base && (void __iomem *)addr < base + size); |
739 | } |
740 | |
741 | /** |
742 | * qcom_smem_virt_to_phys() - return the physical address associated |
743 | * with an smem item pointer (previously returned by qcom_smem_get() |
744 | * @p: the virtual address to convert |
745 | * |
746 | * Returns 0 if the pointer provided is not within any smem region. |
747 | */ |
748 | phys_addr_t qcom_smem_virt_to_phys(void *p) |
749 | { |
750 | struct smem_partition *part; |
751 | struct smem_region *area; |
752 | u64 offset; |
753 | u32 i; |
754 | |
755 | for (i = 0; i < SMEM_HOST_COUNT; i++) { |
756 | part = &__smem->partitions[i]; |
757 | |
758 | if (addr_in_range(base: part->virt_base, size: part->size, addr: p)) { |
759 | offset = p - part->virt_base; |
760 | |
761 | return (phys_addr_t)part->phys_base + offset; |
762 | } |
763 | } |
764 | |
765 | part = &__smem->global_partition; |
766 | |
767 | if (addr_in_range(base: part->virt_base, size: part->size, addr: p)) { |
768 | offset = p - part->virt_base; |
769 | |
770 | return (phys_addr_t)part->phys_base + offset; |
771 | } |
772 | |
773 | for (i = 0; i < __smem->num_regions; i++) { |
774 | area = &__smem->regions[i]; |
775 | |
776 | if (addr_in_range(base: area->virt_base, size: area->size, addr: p)) { |
777 | offset = p - area->virt_base; |
778 | |
779 | return (phys_addr_t)area->aux_base + offset; |
780 | } |
781 | } |
782 | |
783 | return 0; |
784 | } |
785 | EXPORT_SYMBOL_GPL(qcom_smem_virt_to_phys); |
786 | |
787 | /** |
788 | * qcom_smem_get_soc_id() - return the SoC ID |
789 | * @id: On success, we return the SoC ID here. |
790 | * |
791 | * Look up SoC ID from HW/SW build ID and return it. |
792 | * |
793 | * Return: 0 on success, negative errno on failure. |
794 | */ |
795 | int qcom_smem_get_soc_id(u32 *id) |
796 | { |
797 | struct socinfo *info; |
798 | |
799 | info = qcom_smem_get(QCOM_SMEM_HOST_ANY, SMEM_HW_SW_BUILD_ID, NULL); |
800 | if (IS_ERR(ptr: info)) |
801 | return PTR_ERR(ptr: info); |
802 | |
803 | *id = __le32_to_cpu(info->id); |
804 | |
805 | return 0; |
806 | } |
807 | EXPORT_SYMBOL_GPL(qcom_smem_get_soc_id); |
808 | |
809 | static int qcom_smem_get_sbl_version(struct qcom_smem *smem) |
810 | { |
811 | struct smem_header *; |
812 | __le32 *versions; |
813 | |
814 | header = smem->regions[0].virt_base; |
815 | versions = header->version; |
816 | |
817 | return le32_to_cpu(versions[SMEM_MASTER_SBL_VERSION_INDEX]); |
818 | } |
819 | |
820 | static struct smem_ptable *qcom_smem_get_ptable(struct qcom_smem *smem) |
821 | { |
822 | struct smem_ptable *ptable; |
823 | u32 version; |
824 | |
825 | ptable = smem->ptable; |
826 | if (memcmp(p: ptable->magic, q: SMEM_PTABLE_MAGIC, size: sizeof(ptable->magic))) |
827 | return ERR_PTR(error: -ENOENT); |
828 | |
829 | version = le32_to_cpu(ptable->version); |
830 | if (version != 1) { |
831 | dev_err(smem->dev, |
832 | "Unsupported partition header version %d\n" , version); |
833 | return ERR_PTR(error: -EINVAL); |
834 | } |
835 | return ptable; |
836 | } |
837 | |
838 | static u32 qcom_smem_get_item_count(struct qcom_smem *smem) |
839 | { |
840 | struct smem_ptable *ptable; |
841 | struct smem_info *info; |
842 | |
843 | ptable = qcom_smem_get_ptable(smem); |
844 | if (IS_ERR_OR_NULL(ptr: ptable)) |
845 | return SMEM_ITEM_COUNT; |
846 | |
847 | info = (struct smem_info *)&ptable->entry[ptable->num_entries]; |
848 | if (memcmp(p: info->magic, q: SMEM_INFO_MAGIC, size: sizeof(info->magic))) |
849 | return SMEM_ITEM_COUNT; |
850 | |
851 | return le16_to_cpu(info->num_items); |
852 | } |
853 | |
854 | /* |
855 | * Validate the partition header for a partition whose partition |
856 | * table entry is supplied. Returns a pointer to its header if |
857 | * valid, or a null pointer otherwise. |
858 | */ |
859 | static struct smem_partition_header * |
860 | (struct qcom_smem *smem, |
861 | struct smem_ptable_entry *entry, u16 host0, u16 host1) |
862 | { |
863 | struct smem_partition_header *; |
864 | u32 phys_addr; |
865 | u32 size; |
866 | |
867 | phys_addr = smem->regions[0].aux_base + le32_to_cpu(entry->offset); |
868 | header = devm_ioremap_wc(dev: smem->dev, offset: phys_addr, le32_to_cpu(entry->size)); |
869 | |
870 | if (!header) |
871 | return NULL; |
872 | |
873 | if (memcmp(p: header->magic, q: SMEM_PART_MAGIC, size: sizeof(header->magic))) { |
874 | dev_err(smem->dev, "bad partition magic %4ph\n" , header->magic); |
875 | return NULL; |
876 | } |
877 | |
878 | if (host0 != le16_to_cpu(header->host0)) { |
879 | dev_err(smem->dev, "bad host0 (%hu != %hu)\n" , |
880 | host0, le16_to_cpu(header->host0)); |
881 | return NULL; |
882 | } |
883 | if (host1 != le16_to_cpu(header->host1)) { |
884 | dev_err(smem->dev, "bad host1 (%hu != %hu)\n" , |
885 | host1, le16_to_cpu(header->host1)); |
886 | return NULL; |
887 | } |
888 | |
889 | size = le32_to_cpu(header->size); |
890 | if (size != le32_to_cpu(entry->size)) { |
891 | dev_err(smem->dev, "bad partition size (%u != %u)\n" , |
892 | size, le32_to_cpu(entry->size)); |
893 | return NULL; |
894 | } |
895 | |
896 | if (le32_to_cpu(header->offset_free_uncached) > size) { |
897 | dev_err(smem->dev, "bad partition free uncached (%u > %u)\n" , |
898 | le32_to_cpu(header->offset_free_uncached), size); |
899 | return NULL; |
900 | } |
901 | |
902 | return header; |
903 | } |
904 | |
905 | static int qcom_smem_set_global_partition(struct qcom_smem *smem) |
906 | { |
907 | struct smem_partition_header *; |
908 | struct smem_ptable_entry *entry; |
909 | struct smem_ptable *ptable; |
910 | bool found = false; |
911 | int i; |
912 | |
913 | if (smem->global_partition.virt_base) { |
914 | dev_err(smem->dev, "Already found the global partition\n" ); |
915 | return -EINVAL; |
916 | } |
917 | |
918 | ptable = qcom_smem_get_ptable(smem); |
919 | if (IS_ERR(ptr: ptable)) |
920 | return PTR_ERR(ptr: ptable); |
921 | |
922 | for (i = 0; i < le32_to_cpu(ptable->num_entries); i++) { |
923 | entry = &ptable->entry[i]; |
924 | if (!le32_to_cpu(entry->offset)) |
925 | continue; |
926 | if (!le32_to_cpu(entry->size)) |
927 | continue; |
928 | |
929 | if (le16_to_cpu(entry->host0) != SMEM_GLOBAL_HOST) |
930 | continue; |
931 | |
932 | if (le16_to_cpu(entry->host1) == SMEM_GLOBAL_HOST) { |
933 | found = true; |
934 | break; |
935 | } |
936 | } |
937 | |
938 | if (!found) { |
939 | dev_err(smem->dev, "Missing entry for global partition\n" ); |
940 | return -EINVAL; |
941 | } |
942 | |
943 | header = qcom_smem_partition_header(smem, entry, |
944 | SMEM_GLOBAL_HOST, SMEM_GLOBAL_HOST); |
945 | if (!header) |
946 | return -EINVAL; |
947 | |
948 | smem->global_partition.virt_base = (void __iomem *)header; |
949 | smem->global_partition.phys_base = smem->regions[0].aux_base + |
950 | le32_to_cpu(entry->offset); |
951 | smem->global_partition.size = le32_to_cpu(entry->size); |
952 | smem->global_partition.cacheline = le32_to_cpu(entry->cacheline); |
953 | |
954 | return 0; |
955 | } |
956 | |
957 | static int |
958 | qcom_smem_enumerate_partitions(struct qcom_smem *smem, u16 local_host) |
959 | { |
960 | struct smem_partition_header *; |
961 | struct smem_ptable_entry *entry; |
962 | struct smem_ptable *ptable; |
963 | u16 remote_host; |
964 | u16 host0, host1; |
965 | int i; |
966 | |
967 | ptable = qcom_smem_get_ptable(smem); |
968 | if (IS_ERR(ptr: ptable)) |
969 | return PTR_ERR(ptr: ptable); |
970 | |
971 | for (i = 0; i < le32_to_cpu(ptable->num_entries); i++) { |
972 | entry = &ptable->entry[i]; |
973 | if (!le32_to_cpu(entry->offset)) |
974 | continue; |
975 | if (!le32_to_cpu(entry->size)) |
976 | continue; |
977 | |
978 | host0 = le16_to_cpu(entry->host0); |
979 | host1 = le16_to_cpu(entry->host1); |
980 | if (host0 == local_host) |
981 | remote_host = host1; |
982 | else if (host1 == local_host) |
983 | remote_host = host0; |
984 | else |
985 | continue; |
986 | |
987 | if (remote_host >= SMEM_HOST_COUNT) { |
988 | dev_err(smem->dev, "bad host %u\n" , remote_host); |
989 | return -EINVAL; |
990 | } |
991 | |
992 | if (smem->partitions[remote_host].virt_base) { |
993 | dev_err(smem->dev, "duplicate host %u\n" , remote_host); |
994 | return -EINVAL; |
995 | } |
996 | |
997 | header = qcom_smem_partition_header(smem, entry, host0, host1); |
998 | if (!header) |
999 | return -EINVAL; |
1000 | |
1001 | smem->partitions[remote_host].virt_base = (void __iomem *)header; |
1002 | smem->partitions[remote_host].phys_base = smem->regions[0].aux_base + |
1003 | le32_to_cpu(entry->offset); |
1004 | smem->partitions[remote_host].size = le32_to_cpu(entry->size); |
1005 | smem->partitions[remote_host].cacheline = le32_to_cpu(entry->cacheline); |
1006 | } |
1007 | |
1008 | return 0; |
1009 | } |
1010 | |
1011 | static int qcom_smem_map_toc(struct qcom_smem *smem, struct smem_region *region) |
1012 | { |
1013 | u32 ptable_start; |
1014 | |
1015 | /* map starting 4K for smem header */ |
1016 | region->virt_base = devm_ioremap_wc(dev: smem->dev, offset: region->aux_base, SZ_4K); |
1017 | ptable_start = region->aux_base + region->size - SZ_4K; |
1018 | /* map last 4k for toc */ |
1019 | smem->ptable = devm_ioremap_wc(dev: smem->dev, offset: ptable_start, SZ_4K); |
1020 | |
1021 | if (!region->virt_base || !smem->ptable) |
1022 | return -ENOMEM; |
1023 | |
1024 | return 0; |
1025 | } |
1026 | |
1027 | static int qcom_smem_map_global(struct qcom_smem *smem, u32 size) |
1028 | { |
1029 | u32 phys_addr; |
1030 | |
1031 | phys_addr = smem->regions[0].aux_base; |
1032 | |
1033 | smem->regions[0].size = size; |
1034 | smem->regions[0].virt_base = devm_ioremap_wc(dev: smem->dev, offset: phys_addr, size); |
1035 | |
1036 | if (!smem->regions[0].virt_base) |
1037 | return -ENOMEM; |
1038 | |
1039 | return 0; |
1040 | } |
1041 | |
1042 | static int qcom_smem_resolve_mem(struct qcom_smem *smem, const char *name, |
1043 | struct smem_region *region) |
1044 | { |
1045 | struct device *dev = smem->dev; |
1046 | struct device_node *np; |
1047 | struct resource r; |
1048 | int ret; |
1049 | |
1050 | np = of_parse_phandle(np: dev->of_node, phandle_name: name, index: 0); |
1051 | if (!np) { |
1052 | dev_err(dev, "No %s specified\n" , name); |
1053 | return -EINVAL; |
1054 | } |
1055 | |
1056 | ret = of_address_to_resource(dev: np, index: 0, r: &r); |
1057 | of_node_put(node: np); |
1058 | if (ret) |
1059 | return ret; |
1060 | |
1061 | region->aux_base = r.start; |
1062 | region->size = resource_size(res: &r); |
1063 | |
1064 | return 0; |
1065 | } |
1066 | |
1067 | static int qcom_smem_probe(struct platform_device *pdev) |
1068 | { |
1069 | struct smem_header *; |
1070 | struct reserved_mem *rmem; |
1071 | struct qcom_smem *smem; |
1072 | unsigned long flags; |
1073 | int num_regions; |
1074 | int hwlock_id; |
1075 | u32 version; |
1076 | u32 size; |
1077 | int ret; |
1078 | int i; |
1079 | |
1080 | num_regions = 1; |
1081 | if (of_property_present(np: pdev->dev.of_node, propname: "qcom,rpm-msg-ram" )) |
1082 | num_regions++; |
1083 | |
1084 | smem = devm_kzalloc(dev: &pdev->dev, struct_size(smem, regions, num_regions), |
1085 | GFP_KERNEL); |
1086 | if (!smem) |
1087 | return -ENOMEM; |
1088 | |
1089 | smem->dev = &pdev->dev; |
1090 | smem->num_regions = num_regions; |
1091 | |
1092 | rmem = of_reserved_mem_lookup(np: pdev->dev.of_node); |
1093 | if (rmem) { |
1094 | smem->regions[0].aux_base = rmem->base; |
1095 | smem->regions[0].size = rmem->size; |
1096 | } else { |
1097 | /* |
1098 | * Fall back to the memory-region reference, if we're not a |
1099 | * reserved-memory node. |
1100 | */ |
1101 | ret = qcom_smem_resolve_mem(smem, name: "memory-region" , region: &smem->regions[0]); |
1102 | if (ret) |
1103 | return ret; |
1104 | } |
1105 | |
1106 | if (num_regions > 1) { |
1107 | ret = qcom_smem_resolve_mem(smem, name: "qcom,rpm-msg-ram" , region: &smem->regions[1]); |
1108 | if (ret) |
1109 | return ret; |
1110 | } |
1111 | |
1112 | |
1113 | ret = qcom_smem_map_toc(smem, region: &smem->regions[0]); |
1114 | if (ret) |
1115 | return ret; |
1116 | |
1117 | for (i = 1; i < num_regions; i++) { |
1118 | smem->regions[i].virt_base = devm_ioremap_wc(dev: &pdev->dev, |
1119 | offset: smem->regions[i].aux_base, |
1120 | size: smem->regions[i].size); |
1121 | if (!smem->regions[i].virt_base) { |
1122 | dev_err(&pdev->dev, "failed to remap %pa\n" , &smem->regions[i].aux_base); |
1123 | return -ENOMEM; |
1124 | } |
1125 | } |
1126 | |
1127 | header = smem->regions[0].virt_base; |
1128 | if (le32_to_cpu(header->initialized) != 1 || |
1129 | le32_to_cpu(header->reserved)) { |
1130 | dev_err(&pdev->dev, "SMEM is not initialized by SBL\n" ); |
1131 | return -EINVAL; |
1132 | } |
1133 | |
1134 | hwlock_id = of_hwspin_lock_get_id(np: pdev->dev.of_node, index: 0); |
1135 | if (hwlock_id < 0) { |
1136 | if (hwlock_id != -EPROBE_DEFER) |
1137 | dev_err(&pdev->dev, "failed to retrieve hwlock\n" ); |
1138 | return hwlock_id; |
1139 | } |
1140 | |
1141 | smem->hwlock = hwspin_lock_request_specific(id: hwlock_id); |
1142 | if (!smem->hwlock) |
1143 | return -ENXIO; |
1144 | |
1145 | ret = hwspin_lock_timeout_irqsave(hwlock: smem->hwlock, HWSPINLOCK_TIMEOUT, flags: &flags); |
1146 | if (ret) |
1147 | return ret; |
1148 | size = readl_relaxed(&header->available) + readl_relaxed(&header->free_offset); |
1149 | hwspin_unlock_irqrestore(hwlock: smem->hwlock, flags: &flags); |
1150 | |
1151 | version = qcom_smem_get_sbl_version(smem); |
1152 | /* |
1153 | * smem header mapping is required only in heap version scheme, so unmap |
1154 | * it here. It will be remapped in qcom_smem_map_global() when whole |
1155 | * partition is mapped again. |
1156 | */ |
1157 | devm_iounmap(dev: smem->dev, addr: smem->regions[0].virt_base); |
1158 | switch (version >> 16) { |
1159 | case SMEM_GLOBAL_PART_VERSION: |
1160 | ret = qcom_smem_set_global_partition(smem); |
1161 | if (ret < 0) |
1162 | return ret; |
1163 | smem->item_count = qcom_smem_get_item_count(smem); |
1164 | break; |
1165 | case SMEM_GLOBAL_HEAP_VERSION: |
1166 | qcom_smem_map_global(smem, size); |
1167 | smem->item_count = SMEM_ITEM_COUNT; |
1168 | break; |
1169 | default: |
1170 | dev_err(&pdev->dev, "Unsupported SMEM version 0x%x\n" , version); |
1171 | return -EINVAL; |
1172 | } |
1173 | |
1174 | BUILD_BUG_ON(SMEM_HOST_APPS >= SMEM_HOST_COUNT); |
1175 | ret = qcom_smem_enumerate_partitions(smem, SMEM_HOST_APPS); |
1176 | if (ret < 0 && ret != -ENOENT) |
1177 | return ret; |
1178 | |
1179 | __smem = smem; |
1180 | |
1181 | smem->socinfo = platform_device_register_data(parent: &pdev->dev, name: "qcom-socinfo" , |
1182 | PLATFORM_DEVID_NONE, NULL, |
1183 | size: 0); |
1184 | if (IS_ERR(ptr: smem->socinfo)) |
1185 | dev_dbg(&pdev->dev, "failed to register socinfo device\n" ); |
1186 | |
1187 | return 0; |
1188 | } |
1189 | |
1190 | static void qcom_smem_remove(struct platform_device *pdev) |
1191 | { |
1192 | platform_device_unregister(__smem->socinfo); |
1193 | |
1194 | hwspin_lock_free(hwlock: __smem->hwlock); |
1195 | __smem = NULL; |
1196 | } |
1197 | |
1198 | static const struct of_device_id qcom_smem_of_match[] = { |
1199 | { .compatible = "qcom,smem" }, |
1200 | {} |
1201 | }; |
1202 | MODULE_DEVICE_TABLE(of, qcom_smem_of_match); |
1203 | |
1204 | static struct platform_driver qcom_smem_driver = { |
1205 | .probe = qcom_smem_probe, |
1206 | .remove_new = qcom_smem_remove, |
1207 | .driver = { |
1208 | .name = "qcom-smem" , |
1209 | .of_match_table = qcom_smem_of_match, |
1210 | .suppress_bind_attrs = true, |
1211 | }, |
1212 | }; |
1213 | |
1214 | static int __init qcom_smem_init(void) |
1215 | { |
1216 | return platform_driver_register(&qcom_smem_driver); |
1217 | } |
1218 | arch_initcall(qcom_smem_init); |
1219 | |
1220 | static void __exit qcom_smem_exit(void) |
1221 | { |
1222 | platform_driver_unregister(&qcom_smem_driver); |
1223 | } |
1224 | module_exit(qcom_smem_exit) |
1225 | |
1226 | MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@sonymobile.com>" ); |
1227 | MODULE_DESCRIPTION("Qualcomm Shared Memory Manager" ); |
1228 | MODULE_LICENSE("GPL v2" ); |
1229 | |