1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2019 Facebook |
4 | * Copyright 2020 Google LLC. |
5 | */ |
6 | |
7 | #include <linux/rculist.h> |
8 | #include <linux/list.h> |
9 | #include <linux/hash.h> |
10 | #include <linux/types.h> |
11 | #include <linux/spinlock.h> |
12 | #include <linux/bpf.h> |
13 | #include <linux/bpf_local_storage.h> |
14 | #include <net/sock.h> |
15 | #include <uapi/linux/sock_diag.h> |
16 | #include <uapi/linux/btf.h> |
17 | #include <linux/bpf_lsm.h> |
18 | #include <linux/btf_ids.h> |
19 | #include <linux/fdtable.h> |
20 | #include <linux/rcupdate_trace.h> |
21 | |
22 | DEFINE_BPF_STORAGE_CACHE(inode_cache); |
23 | |
24 | static struct bpf_local_storage __rcu ** |
25 | inode_storage_ptr(void *owner) |
26 | { |
27 | struct inode *inode = owner; |
28 | struct bpf_storage_blob *bsb; |
29 | |
30 | bsb = bpf_inode(inode); |
31 | if (!bsb) |
32 | return NULL; |
33 | return &bsb->storage; |
34 | } |
35 | |
36 | static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode, |
37 | struct bpf_map *map, |
38 | bool cacheit_lockit) |
39 | { |
40 | struct bpf_local_storage *inode_storage; |
41 | struct bpf_local_storage_map *smap; |
42 | struct bpf_storage_blob *bsb; |
43 | |
44 | bsb = bpf_inode(inode); |
45 | if (!bsb) |
46 | return NULL; |
47 | |
48 | inode_storage = |
49 | rcu_dereference_check(bsb->storage, bpf_rcu_lock_held()); |
50 | if (!inode_storage) |
51 | return NULL; |
52 | |
53 | smap = (struct bpf_local_storage_map *)map; |
54 | return bpf_local_storage_lookup(local_storage: inode_storage, smap, cacheit_lockit); |
55 | } |
56 | |
57 | void bpf_inode_storage_free(struct inode *inode) |
58 | { |
59 | struct bpf_local_storage *local_storage; |
60 | struct bpf_storage_blob *bsb; |
61 | |
62 | bsb = bpf_inode(inode); |
63 | if (!bsb) |
64 | return; |
65 | |
66 | rcu_read_lock(); |
67 | |
68 | local_storage = rcu_dereference(bsb->storage); |
69 | if (!local_storage) { |
70 | rcu_read_unlock(); |
71 | return; |
72 | } |
73 | |
74 | bpf_local_storage_destroy(local_storage); |
75 | rcu_read_unlock(); |
76 | } |
77 | |
78 | static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key) |
79 | { |
80 | struct bpf_local_storage_data *sdata; |
81 | struct fd f = fdget_raw(fd: *(int *)key); |
82 | |
83 | if (!f.file) |
84 | return ERR_PTR(error: -EBADF); |
85 | |
86 | sdata = inode_storage_lookup(inode: file_inode(f: f.file), map, cacheit_lockit: true); |
87 | fdput(fd: f); |
88 | return sdata ? sdata->data : NULL; |
89 | } |
90 | |
91 | static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key, |
92 | void *value, u64 map_flags) |
93 | { |
94 | struct bpf_local_storage_data *sdata; |
95 | struct fd f = fdget_raw(fd: *(int *)key); |
96 | |
97 | if (!f.file) |
98 | return -EBADF; |
99 | if (!inode_storage_ptr(owner: file_inode(f: f.file))) { |
100 | fdput(fd: f); |
101 | return -EBADF; |
102 | } |
103 | |
104 | sdata = bpf_local_storage_update(owner: file_inode(f: f.file), |
105 | smap: (struct bpf_local_storage_map *)map, |
106 | value, map_flags, GFP_ATOMIC); |
107 | fdput(fd: f); |
108 | return PTR_ERR_OR_ZERO(ptr: sdata); |
109 | } |
110 | |
111 | static int inode_storage_delete(struct inode *inode, struct bpf_map *map) |
112 | { |
113 | struct bpf_local_storage_data *sdata; |
114 | |
115 | sdata = inode_storage_lookup(inode, map, cacheit_lockit: false); |
116 | if (!sdata) |
117 | return -ENOENT; |
118 | |
119 | bpf_selem_unlink(SELEM(sdata), reuse_now: false); |
120 | |
121 | return 0; |
122 | } |
123 | |
124 | static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key) |
125 | { |
126 | struct fd f = fdget_raw(fd: *(int *)key); |
127 | int err; |
128 | |
129 | if (!f.file) |
130 | return -EBADF; |
131 | |
132 | err = inode_storage_delete(inode: file_inode(f: f.file), map); |
133 | fdput(fd: f); |
134 | return err; |
135 | } |
136 | |
137 | /* *gfp_flags* is a hidden argument provided by the verifier */ |
138 | BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode, |
139 | void *, value, u64, flags, gfp_t, gfp_flags) |
140 | { |
141 | struct bpf_local_storage_data *sdata; |
142 | |
143 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
144 | if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) |
145 | return (unsigned long)NULL; |
146 | |
147 | /* explicitly check that the inode_storage_ptr is not |
148 | * NULL as inode_storage_lookup returns NULL in this case and |
149 | * bpf_local_storage_update expects the owner to have a |
150 | * valid storage pointer. |
151 | */ |
152 | if (!inode || !inode_storage_ptr(owner: inode)) |
153 | return (unsigned long)NULL; |
154 | |
155 | sdata = inode_storage_lookup(inode, map, cacheit_lockit: true); |
156 | if (sdata) |
157 | return (unsigned long)sdata->data; |
158 | |
159 | /* This helper must only called from where the inode is guaranteed |
160 | * to have a refcount and cannot be freed. |
161 | */ |
162 | if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) { |
163 | sdata = bpf_local_storage_update( |
164 | owner: inode, smap: (struct bpf_local_storage_map *)map, value, |
165 | map_flags: BPF_NOEXIST, gfp_flags); |
166 | return IS_ERR(ptr: sdata) ? (unsigned long)NULL : |
167 | (unsigned long)sdata->data; |
168 | } |
169 | |
170 | return (unsigned long)NULL; |
171 | } |
172 | |
173 | BPF_CALL_2(bpf_inode_storage_delete, |
174 | struct bpf_map *, map, struct inode *, inode) |
175 | { |
176 | WARN_ON_ONCE(!bpf_rcu_lock_held()); |
177 | if (!inode) |
178 | return -EINVAL; |
179 | |
180 | /* This helper must only called from where the inode is guaranteed |
181 | * to have a refcount and cannot be freed. |
182 | */ |
183 | return inode_storage_delete(inode, map); |
184 | } |
185 | |
186 | static int notsupp_get_next_key(struct bpf_map *map, void *key, |
187 | void *next_key) |
188 | { |
189 | return -ENOTSUPP; |
190 | } |
191 | |
192 | static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr) |
193 | { |
194 | return bpf_local_storage_map_alloc(attr, cache: &inode_cache, bpf_ma: false); |
195 | } |
196 | |
197 | static void inode_storage_map_free(struct bpf_map *map) |
198 | { |
199 | bpf_local_storage_map_free(map, cache: &inode_cache, NULL); |
200 | } |
201 | |
202 | const struct bpf_map_ops inode_storage_map_ops = { |
203 | .map_meta_equal = bpf_map_meta_equal, |
204 | .map_alloc_check = bpf_local_storage_map_alloc_check, |
205 | .map_alloc = inode_storage_map_alloc, |
206 | .map_free = inode_storage_map_free, |
207 | .map_get_next_key = notsupp_get_next_key, |
208 | .map_lookup_elem = bpf_fd_inode_storage_lookup_elem, |
209 | .map_update_elem = bpf_fd_inode_storage_update_elem, |
210 | .map_delete_elem = bpf_fd_inode_storage_delete_elem, |
211 | .map_check_btf = bpf_local_storage_map_check_btf, |
212 | .map_mem_usage = bpf_local_storage_map_mem_usage, |
213 | .map_btf_id = &bpf_local_storage_map_btf_id[0], |
214 | .map_owner_storage_ptr = inode_storage_ptr, |
215 | }; |
216 | |
217 | BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode) |
218 | |
219 | const struct bpf_func_proto bpf_inode_storage_get_proto = { |
220 | .func = bpf_inode_storage_get, |
221 | .gpl_only = false, |
222 | .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, |
223 | .arg1_type = ARG_CONST_MAP_PTR, |
224 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
225 | .arg2_btf_id = &bpf_inode_storage_btf_ids[0], |
226 | .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, |
227 | .arg4_type = ARG_ANYTHING, |
228 | }; |
229 | |
230 | const struct bpf_func_proto bpf_inode_storage_delete_proto = { |
231 | .func = bpf_inode_storage_delete, |
232 | .gpl_only = false, |
233 | .ret_type = RET_INTEGER, |
234 | .arg1_type = ARG_CONST_MAP_PTR, |
235 | .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, |
236 | .arg2_btf_id = &bpf_inode_storage_btf_ids[0], |
237 | }; |
238 | |