1 | /* |
2 | * Copyright (c) 2006 Oracle. All rights reserved. |
3 | * |
4 | * This software is available to you under a choice of one of two |
5 | * licenses. You may choose to be licensed under the terms of the GNU |
6 | * General Public License (GPL) Version 2, available from the file |
7 | * COPYING in the main directory of this source tree, or the |
8 | * OpenIB.org BSD license below: |
9 | * |
10 | * Redistribution and use in source and binary forms, with or |
11 | * without modification, are permitted provided that the following |
12 | * conditions are met: |
13 | * |
14 | * - Redistributions of source code must retain the above |
15 | * copyright notice, this list of conditions and the following |
16 | * disclaimer. |
17 | * |
18 | * - Redistributions in binary form must reproduce the above |
19 | * copyright notice, this list of conditions and the following |
20 | * disclaimer in the documentation and/or other materials |
21 | * provided with the distribution. |
22 | * |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
30 | * SOFTWARE. |
31 | * |
32 | */ |
33 | #include <linux/highmem.h> |
34 | #include <linux/gfp.h> |
35 | #include <linux/cpu.h> |
36 | #include <linux/export.h> |
37 | |
38 | #include "rds.h" |
39 | |
40 | struct rds_page_remainder { |
41 | struct page *r_page; |
42 | unsigned long r_offset; |
43 | }; |
44 | |
45 | static |
46 | DEFINE_PER_CPU_SHARED_ALIGNED(struct rds_page_remainder, rds_page_remainders); |
47 | |
48 | /** |
49 | * rds_page_remainder_alloc - build up regions of a message. |
50 | * |
51 | * @scat: Scatter list for message |
52 | * @bytes: the number of bytes needed. |
53 | * @gfp: the waiting behaviour of the allocation |
54 | * |
55 | * @gfp is always ored with __GFP_HIGHMEM. Callers must be prepared to |
56 | * kmap the pages, etc. |
57 | * |
58 | * If @bytes is at least a full page then this just returns a page from |
59 | * alloc_page(). |
60 | * |
61 | * If @bytes is a partial page then this stores the unused region of the |
62 | * page in a per-cpu structure. Future partial-page allocations may be |
63 | * satisfied from that cached region. This lets us waste less memory on |
64 | * small allocations with minimal complexity. It works because the transmit |
65 | * path passes read-only page regions down to devices. They hold a page |
66 | * reference until they are done with the region. |
67 | */ |
68 | int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes, |
69 | gfp_t gfp) |
70 | { |
71 | struct rds_page_remainder *rem; |
72 | unsigned long flags; |
73 | struct page *page; |
74 | int ret; |
75 | |
76 | gfp |= __GFP_HIGHMEM; |
77 | |
78 | /* jump straight to allocation if we're trying for a huge page */ |
79 | if (bytes >= PAGE_SIZE) { |
80 | page = alloc_page(gfp); |
81 | if (!page) { |
82 | ret = -ENOMEM; |
83 | } else { |
84 | sg_set_page(sg: scat, page, PAGE_SIZE, offset: 0); |
85 | ret = 0; |
86 | } |
87 | goto out; |
88 | } |
89 | |
90 | rem = &per_cpu(rds_page_remainders, get_cpu()); |
91 | local_irq_save(flags); |
92 | |
93 | while (1) { |
94 | /* avoid a tiny region getting stuck by tossing it */ |
95 | if (rem->r_page && bytes > (PAGE_SIZE - rem->r_offset)) { |
96 | rds_stats_inc(s_page_remainder_miss); |
97 | __free_page(rem->r_page); |
98 | rem->r_page = NULL; |
99 | } |
100 | |
101 | /* hand out a fragment from the cached page */ |
102 | if (rem->r_page && bytes <= (PAGE_SIZE - rem->r_offset)) { |
103 | sg_set_page(sg: scat, page: rem->r_page, len: bytes, offset: rem->r_offset); |
104 | get_page(page: sg_page(sg: scat)); |
105 | |
106 | if (rem->r_offset != 0) |
107 | rds_stats_inc(s_page_remainder_hit); |
108 | |
109 | rem->r_offset += ALIGN(bytes, 8); |
110 | if (rem->r_offset >= PAGE_SIZE) { |
111 | __free_page(rem->r_page); |
112 | rem->r_page = NULL; |
113 | } |
114 | ret = 0; |
115 | break; |
116 | } |
117 | |
118 | /* alloc if there is nothing for us to use */ |
119 | local_irq_restore(flags); |
120 | put_cpu(); |
121 | |
122 | page = alloc_page(gfp); |
123 | |
124 | rem = &per_cpu(rds_page_remainders, get_cpu()); |
125 | local_irq_save(flags); |
126 | |
127 | if (!page) { |
128 | ret = -ENOMEM; |
129 | break; |
130 | } |
131 | |
132 | /* did someone race to fill the remainder before us? */ |
133 | if (rem->r_page) { |
134 | __free_page(page); |
135 | continue; |
136 | } |
137 | |
138 | /* otherwise install our page and loop around to alloc */ |
139 | rem->r_page = page; |
140 | rem->r_offset = 0; |
141 | } |
142 | |
143 | local_irq_restore(flags); |
144 | put_cpu(); |
145 | out: |
146 | rdsdebug("bytes %lu ret %d %p %u %u\n" , bytes, ret, |
147 | ret ? NULL : sg_page(scat), ret ? 0 : scat->offset, |
148 | ret ? 0 : scat->length); |
149 | return ret; |
150 | } |
151 | EXPORT_SYMBOL_GPL(rds_page_remainder_alloc); |
152 | |
153 | void rds_page_exit(void) |
154 | { |
155 | unsigned int cpu; |
156 | |
157 | for_each_possible_cpu(cpu) { |
158 | struct rds_page_remainder *rem; |
159 | |
160 | rem = &per_cpu(rds_page_remainders, cpu); |
161 | rdsdebug("cpu %u\n" , cpu); |
162 | |
163 | if (rem->r_page) |
164 | __free_page(rem->r_page); |
165 | rem->r_page = NULL; |
166 | } |
167 | } |
168 | |