1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
4 */
5#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
6#include <linux/libnvdimm.h>
7#include <linux/sched/mm.h>
8#include <linux/vmalloc.h>
9#include <linux/uaccess.h>
10#include <linux/module.h>
11#include <linux/blkdev.h>
12#include <linux/fcntl.h>
13#include <linux/async.h>
14#include <linux/ndctl.h>
15#include <linux/sched.h>
16#include <linux/slab.h>
17#include <linux/cpu.h>
18#include <linux/fs.h>
19#include <linux/io.h>
20#include <linux/mm.h>
21#include <linux/nd.h>
22#include "nd-core.h"
23#include "nd.h"
24#include "pfn.h"
25
26int nvdimm_major;
27static int nvdimm_bus_major;
28static struct class *nd_class;
29static DEFINE_IDA(nd_ida);
30
31static int to_nd_device_type(const struct device *dev)
32{
33 if (is_nvdimm(dev))
34 return ND_DEVICE_DIMM;
35 else if (is_memory(dev))
36 return ND_DEVICE_REGION_PMEM;
37 else if (is_nd_dax(dev))
38 return ND_DEVICE_DAX_PMEM;
39 else if (is_nd_region(dev: dev->parent))
40 return nd_region_to_nstype(nd_region: to_nd_region(dev: dev->parent));
41
42 return 0;
43}
44
45static int nvdimm_bus_uevent(const struct device *dev, struct kobj_uevent_env *env)
46{
47 return add_uevent_var(env, format: "MODALIAS=" ND_DEVICE_MODALIAS_FMT,
48 to_nd_device_type(dev));
49}
50
51static struct module *to_bus_provider(struct device *dev)
52{
53 /* pin bus providers while regions are enabled */
54 if (is_nd_region(dev)) {
55 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
56
57 return nvdimm_bus->nd_desc->module;
58 }
59 return NULL;
60}
61
62static void nvdimm_bus_probe_start(struct nvdimm_bus *nvdimm_bus)
63{
64 nvdimm_bus_lock(dev: &nvdimm_bus->dev);
65 nvdimm_bus->probe_active++;
66 nvdimm_bus_unlock(dev: &nvdimm_bus->dev);
67}
68
69static void nvdimm_bus_probe_end(struct nvdimm_bus *nvdimm_bus)
70{
71 nvdimm_bus_lock(dev: &nvdimm_bus->dev);
72 if (--nvdimm_bus->probe_active == 0)
73 wake_up(&nvdimm_bus->wait);
74 nvdimm_bus_unlock(dev: &nvdimm_bus->dev);
75}
76
77static int nvdimm_bus_probe(struct device *dev)
78{
79 struct nd_device_driver *nd_drv = to_nd_device_driver(drv: dev->driver);
80 struct module *provider = to_bus_provider(dev);
81 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
82 int rc;
83
84 if (!try_module_get(module: provider))
85 return -ENXIO;
86
87 dev_dbg(&nvdimm_bus->dev, "START: %s.probe(%s)\n",
88 dev->driver->name, dev_name(dev));
89
90 nvdimm_bus_probe_start(nvdimm_bus);
91 rc = nd_drv->probe(dev);
92 if ((rc == 0 || rc == -EOPNOTSUPP) &&
93 dev->parent && is_nd_region(dev: dev->parent))
94 nd_region_advance_seeds(nd_region: to_nd_region(dev: dev->parent), dev);
95 nvdimm_bus_probe_end(nvdimm_bus);
96
97 dev_dbg(&nvdimm_bus->dev, "END: %s.probe(%s) = %d\n", dev->driver->name,
98 dev_name(dev), rc);
99
100 if (rc != 0)
101 module_put(module: provider);
102 return rc;
103}
104
105static void nvdimm_bus_remove(struct device *dev)
106{
107 struct nd_device_driver *nd_drv = to_nd_device_driver(drv: dev->driver);
108 struct module *provider = to_bus_provider(dev);
109 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
110
111 if (nd_drv->remove)
112 nd_drv->remove(dev);
113
114 dev_dbg(&nvdimm_bus->dev, "%s.remove(%s)\n", dev->driver->name,
115 dev_name(dev));
116 module_put(module: provider);
117}
118
119static void nvdimm_bus_shutdown(struct device *dev)
120{
121 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
122 struct nd_device_driver *nd_drv = NULL;
123
124 if (dev->driver)
125 nd_drv = to_nd_device_driver(drv: dev->driver);
126
127 if (nd_drv && nd_drv->shutdown) {
128 nd_drv->shutdown(dev);
129 dev_dbg(&nvdimm_bus->dev, "%s.shutdown(%s)\n",
130 dev->driver->name, dev_name(dev));
131 }
132}
133
134void nd_device_notify(struct device *dev, enum nvdimm_event event)
135{
136 device_lock(dev);
137 if (dev->driver) {
138 struct nd_device_driver *nd_drv;
139
140 nd_drv = to_nd_device_driver(drv: dev->driver);
141 if (nd_drv->notify)
142 nd_drv->notify(dev, event);
143 }
144 device_unlock(dev);
145}
146EXPORT_SYMBOL(nd_device_notify);
147
148void nvdimm_region_notify(struct nd_region *nd_region, enum nvdimm_event event)
149{
150 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: &nd_region->dev);
151
152 if (!nvdimm_bus)
153 return;
154
155 /* caller is responsible for holding a reference on the device */
156 nd_device_notify(&nd_region->dev, event);
157}
158EXPORT_SYMBOL_GPL(nvdimm_region_notify);
159
160struct clear_badblocks_context {
161 resource_size_t phys, cleared;
162};
163
164static int nvdimm_clear_badblocks_region(struct device *dev, void *data)
165{
166 struct clear_badblocks_context *ctx = data;
167 struct nd_region *nd_region;
168 resource_size_t ndr_end;
169 sector_t sector;
170
171 /* make sure device is a region */
172 if (!is_memory(dev))
173 return 0;
174
175 nd_region = to_nd_region(dev);
176 ndr_end = nd_region->ndr_start + nd_region->ndr_size - 1;
177
178 /* make sure we are in the region */
179 if (ctx->phys < nd_region->ndr_start ||
180 (ctx->phys + ctx->cleared - 1) > ndr_end)
181 return 0;
182
183 sector = (ctx->phys - nd_region->ndr_start) / 512;
184 badblocks_clear(bb: &nd_region->bb, s: sector, sectors: ctx->cleared / 512);
185
186 if (nd_region->bb_state)
187 sysfs_notify_dirent(kn: nd_region->bb_state);
188
189 return 0;
190}
191
192static void nvdimm_clear_badblocks_regions(struct nvdimm_bus *nvdimm_bus,
193 phys_addr_t phys, u64 cleared)
194{
195 struct clear_badblocks_context ctx = {
196 .phys = phys,
197 .cleared = cleared,
198 };
199
200 device_for_each_child(dev: &nvdimm_bus->dev, data: &ctx,
201 fn: nvdimm_clear_badblocks_region);
202}
203
204static void nvdimm_account_cleared_poison(struct nvdimm_bus *nvdimm_bus,
205 phys_addr_t phys, u64 cleared)
206{
207 if (cleared > 0)
208 badrange_forget(badrange: &nvdimm_bus->badrange, start: phys, len: cleared);
209
210 if (cleared > 0 && cleared / 512)
211 nvdimm_clear_badblocks_regions(nvdimm_bus, phys, cleared);
212}
213
214long nvdimm_clear_poison(struct device *dev, phys_addr_t phys,
215 unsigned int len)
216{
217 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
218 struct nvdimm_bus_descriptor *nd_desc;
219 struct nd_cmd_clear_error clear_err;
220 struct nd_cmd_ars_cap ars_cap;
221 u32 clear_err_unit, mask;
222 unsigned int noio_flag;
223 int cmd_rc, rc;
224
225 if (!nvdimm_bus)
226 return -ENXIO;
227
228 nd_desc = nvdimm_bus->nd_desc;
229 /*
230 * if ndctl does not exist, it's PMEM_LEGACY and
231 * we want to just pretend everything is handled.
232 */
233 if (!nd_desc->ndctl)
234 return len;
235
236 memset(&ars_cap, 0, sizeof(ars_cap));
237 ars_cap.address = phys;
238 ars_cap.length = len;
239 noio_flag = memalloc_noio_save();
240 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_ARS_CAP, &ars_cap,
241 sizeof(ars_cap), &cmd_rc);
242 memalloc_noio_restore(flags: noio_flag);
243 if (rc < 0)
244 return rc;
245 if (cmd_rc < 0)
246 return cmd_rc;
247 clear_err_unit = ars_cap.clear_err_unit;
248 if (!clear_err_unit || !is_power_of_2(n: clear_err_unit))
249 return -ENXIO;
250
251 mask = clear_err_unit - 1;
252 if ((phys | len) & mask)
253 return -ENXIO;
254 memset(&clear_err, 0, sizeof(clear_err));
255 clear_err.address = phys;
256 clear_err.length = len;
257 noio_flag = memalloc_noio_save();
258 rc = nd_desc->ndctl(nd_desc, NULL, ND_CMD_CLEAR_ERROR, &clear_err,
259 sizeof(clear_err), &cmd_rc);
260 memalloc_noio_restore(flags: noio_flag);
261 if (rc < 0)
262 return rc;
263 if (cmd_rc < 0)
264 return cmd_rc;
265
266 nvdimm_account_cleared_poison(nvdimm_bus, phys, cleared: clear_err.cleared);
267
268 return clear_err.cleared;
269}
270EXPORT_SYMBOL_GPL(nvdimm_clear_poison);
271
272static int nvdimm_bus_match(struct device *dev, struct device_driver *drv);
273
274static struct bus_type nvdimm_bus_type = {
275 .name = "nd",
276 .uevent = nvdimm_bus_uevent,
277 .match = nvdimm_bus_match,
278 .probe = nvdimm_bus_probe,
279 .remove = nvdimm_bus_remove,
280 .shutdown = nvdimm_bus_shutdown,
281};
282
283static void nvdimm_bus_release(struct device *dev)
284{
285 struct nvdimm_bus *nvdimm_bus;
286
287 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
288 ida_simple_remove(&nd_ida, nvdimm_bus->id);
289 kfree(objp: nvdimm_bus);
290}
291
292static const struct device_type nvdimm_bus_dev_type = {
293 .release = nvdimm_bus_release,
294 .groups = nvdimm_bus_attribute_groups,
295};
296
297bool is_nvdimm_bus(struct device *dev)
298{
299 return dev->type == &nvdimm_bus_dev_type;
300}
301
302struct nvdimm_bus *walk_to_nvdimm_bus(struct device *nd_dev)
303{
304 struct device *dev;
305
306 for (dev = nd_dev; dev; dev = dev->parent)
307 if (is_nvdimm_bus(dev))
308 break;
309 dev_WARN_ONCE(nd_dev, !dev, "invalid dev, not on nd bus\n");
310 if (dev)
311 return to_nvdimm_bus(dev);
312 return NULL;
313}
314
315struct nvdimm_bus *to_nvdimm_bus(struct device *dev)
316{
317 struct nvdimm_bus *nvdimm_bus;
318
319 nvdimm_bus = container_of(dev, struct nvdimm_bus, dev);
320 WARN_ON(!is_nvdimm_bus(dev));
321 return nvdimm_bus;
322}
323EXPORT_SYMBOL_GPL(to_nvdimm_bus);
324
325struct nvdimm_bus *nvdimm_to_bus(struct nvdimm *nvdimm)
326{
327 return to_nvdimm_bus(nvdimm->dev.parent);
328}
329EXPORT_SYMBOL_GPL(nvdimm_to_bus);
330
331static struct lock_class_key nvdimm_bus_key;
332
333struct nvdimm_bus *nvdimm_bus_register(struct device *parent,
334 struct nvdimm_bus_descriptor *nd_desc)
335{
336 struct nvdimm_bus *nvdimm_bus;
337 int rc;
338
339 nvdimm_bus = kzalloc(size: sizeof(*nvdimm_bus), GFP_KERNEL);
340 if (!nvdimm_bus)
341 return NULL;
342 INIT_LIST_HEAD(list: &nvdimm_bus->list);
343 INIT_LIST_HEAD(list: &nvdimm_bus->mapping_list);
344 init_waitqueue_head(&nvdimm_bus->wait);
345 nvdimm_bus->id = ida_simple_get(&nd_ida, 0, 0, GFP_KERNEL);
346 if (nvdimm_bus->id < 0) {
347 kfree(objp: nvdimm_bus);
348 return NULL;
349 }
350 mutex_init(&nvdimm_bus->reconfig_mutex);
351 badrange_init(badrange: &nvdimm_bus->badrange);
352 nvdimm_bus->nd_desc = nd_desc;
353 nvdimm_bus->dev.parent = parent;
354 nvdimm_bus->dev.type = &nvdimm_bus_dev_type;
355 nvdimm_bus->dev.groups = nd_desc->attr_groups;
356 nvdimm_bus->dev.bus = &nvdimm_bus_type;
357 nvdimm_bus->dev.of_node = nd_desc->of_node;
358 device_initialize(dev: &nvdimm_bus->dev);
359 lockdep_set_class(&nvdimm_bus->dev.mutex, &nvdimm_bus_key);
360 device_set_pm_not_required(dev: &nvdimm_bus->dev);
361 rc = dev_set_name(dev: &nvdimm_bus->dev, name: "ndbus%d", nvdimm_bus->id);
362 if (rc)
363 goto err;
364
365 rc = device_add(dev: &nvdimm_bus->dev);
366 if (rc) {
367 dev_dbg(&nvdimm_bus->dev, "registration failed: %d\n", rc);
368 goto err;
369 }
370
371 return nvdimm_bus;
372 err:
373 put_device(dev: &nvdimm_bus->dev);
374 return NULL;
375}
376EXPORT_SYMBOL_GPL(nvdimm_bus_register);
377
378void nvdimm_bus_unregister(struct nvdimm_bus *nvdimm_bus)
379{
380 if (!nvdimm_bus)
381 return;
382 device_unregister(dev: &nvdimm_bus->dev);
383}
384EXPORT_SYMBOL_GPL(nvdimm_bus_unregister);
385
386static int child_unregister(struct device *dev, void *data)
387{
388 /*
389 * the singular ndctl class device per bus needs to be
390 * "device_destroy"ed, so skip it here
391 *
392 * i.e. remove classless children
393 */
394 if (dev->class)
395 return 0;
396
397 if (is_nvdimm(dev))
398 nvdimm_delete(nvdimm: to_nvdimm(dev));
399 else
400 nd_device_unregister(dev, mode: ND_SYNC);
401
402 return 0;
403}
404
405static void free_badrange_list(struct list_head *badrange_list)
406{
407 struct badrange_entry *bre, *next;
408
409 list_for_each_entry_safe(bre, next, badrange_list, list) {
410 list_del(entry: &bre->list);
411 kfree(objp: bre);
412 }
413 list_del_init(entry: badrange_list);
414}
415
416static void nd_bus_remove(struct device *dev)
417{
418 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
419
420 mutex_lock(&nvdimm_bus_list_mutex);
421 list_del_init(entry: &nvdimm_bus->list);
422 mutex_unlock(lock: &nvdimm_bus_list_mutex);
423
424 wait_event(nvdimm_bus->wait,
425 atomic_read(&nvdimm_bus->ioctl_active) == 0);
426
427 nd_synchronize();
428 device_for_each_child(dev: &nvdimm_bus->dev, NULL, fn: child_unregister);
429
430 spin_lock(lock: &nvdimm_bus->badrange.lock);
431 free_badrange_list(badrange_list: &nvdimm_bus->badrange.list);
432 spin_unlock(lock: &nvdimm_bus->badrange.lock);
433
434 nvdimm_bus_destroy_ndctl(nvdimm_bus);
435}
436
437static int nd_bus_probe(struct device *dev)
438{
439 struct nvdimm_bus *nvdimm_bus = to_nvdimm_bus(dev);
440 int rc;
441
442 rc = nvdimm_bus_create_ndctl(nvdimm_bus);
443 if (rc)
444 return rc;
445
446 mutex_lock(&nvdimm_bus_list_mutex);
447 list_add_tail(new: &nvdimm_bus->list, head: &nvdimm_bus_list);
448 mutex_unlock(lock: &nvdimm_bus_list_mutex);
449
450 /* enable bus provider attributes to look up their local context */
451 dev_set_drvdata(dev, data: nvdimm_bus->nd_desc);
452
453 return 0;
454}
455
456static struct nd_device_driver nd_bus_driver = {
457 .probe = nd_bus_probe,
458 .remove = nd_bus_remove,
459 .drv = {
460 .name = "nd_bus",
461 .suppress_bind_attrs = true,
462 .bus = &nvdimm_bus_type,
463 .owner = THIS_MODULE,
464 .mod_name = KBUILD_MODNAME,
465 },
466};
467
468static int nvdimm_bus_match(struct device *dev, struct device_driver *drv)
469{
470 struct nd_device_driver *nd_drv = to_nd_device_driver(drv);
471
472 if (is_nvdimm_bus(dev) && nd_drv == &nd_bus_driver)
473 return true;
474
475 return !!test_bit(to_nd_device_type(dev), &nd_drv->type);
476}
477
478static ASYNC_DOMAIN_EXCLUSIVE(nd_async_domain);
479
480void nd_synchronize(void)
481{
482 async_synchronize_full_domain(domain: &nd_async_domain);
483}
484EXPORT_SYMBOL_GPL(nd_synchronize);
485
486static void nd_async_device_register(void *d, async_cookie_t cookie)
487{
488 struct device *dev = d;
489
490 if (device_add(dev) != 0) {
491 dev_err(dev, "%s: failed\n", __func__);
492 put_device(dev);
493 }
494 put_device(dev);
495 if (dev->parent)
496 put_device(dev: dev->parent);
497}
498
499static void nd_async_device_unregister(void *d, async_cookie_t cookie)
500{
501 struct device *dev = d;
502
503 /* flush bus operations before delete */
504 nvdimm_bus_lock(dev);
505 nvdimm_bus_unlock(dev);
506
507 device_unregister(dev);
508 put_device(dev);
509}
510
511static void __nd_device_register(struct device *dev, bool sync)
512{
513 if (!dev)
514 return;
515
516 /*
517 * Ensure that region devices always have their NUMA node set as
518 * early as possible. This way we are able to make certain that
519 * any memory associated with the creation and the creation
520 * itself of the region is associated with the correct node.
521 */
522 if (is_nd_region(dev))
523 set_dev_node(dev, node: to_nd_region(dev)->numa_node);
524
525 dev->bus = &nvdimm_bus_type;
526 device_set_pm_not_required(dev);
527 if (dev->parent) {
528 get_device(dev: dev->parent);
529 if (dev_to_node(dev) == NUMA_NO_NODE)
530 set_dev_node(dev, node: dev_to_node(dev: dev->parent));
531 }
532 get_device(dev);
533
534 if (sync)
535 nd_async_device_register(d: dev, cookie: 0);
536 else
537 async_schedule_dev_domain(func: nd_async_device_register, dev,
538 domain: &nd_async_domain);
539}
540
541void nd_device_register(struct device *dev)
542{
543 __nd_device_register(dev, sync: false);
544}
545EXPORT_SYMBOL(nd_device_register);
546
547void nd_device_register_sync(struct device *dev)
548{
549 __nd_device_register(dev, sync: true);
550}
551
552void nd_device_unregister(struct device *dev, enum nd_async_mode mode)
553{
554 bool killed;
555
556 switch (mode) {
557 case ND_ASYNC:
558 /*
559 * In the async case this is being triggered with the
560 * device lock held and the unregistration work needs to
561 * be moved out of line iff this is thread has won the
562 * race to schedule the deletion.
563 */
564 if (!kill_device(dev))
565 return;
566
567 get_device(dev);
568 async_schedule_domain(func: nd_async_device_unregister, data: dev,
569 domain: &nd_async_domain);
570 break;
571 case ND_SYNC:
572 /*
573 * In the sync case the device is being unregistered due
574 * to a state change of the parent. Claim the kill state
575 * to synchronize against other unregistration requests,
576 * or otherwise let the async path handle it if the
577 * unregistration was already queued.
578 */
579 device_lock(dev);
580 killed = kill_device(dev);
581 device_unlock(dev);
582
583 if (!killed)
584 return;
585
586 nd_synchronize();
587 device_unregister(dev);
588 break;
589 }
590}
591EXPORT_SYMBOL(nd_device_unregister);
592
593/**
594 * __nd_driver_register() - register a region or a namespace driver
595 * @nd_drv: driver to register
596 * @owner: automatically set by nd_driver_register() macro
597 * @mod_name: automatically set by nd_driver_register() macro
598 */
599int __nd_driver_register(struct nd_device_driver *nd_drv, struct module *owner,
600 const char *mod_name)
601{
602 struct device_driver *drv = &nd_drv->drv;
603
604 if (!nd_drv->type) {
605 pr_debug("driver type bitmask not set (%ps)\n",
606 __builtin_return_address(0));
607 return -EINVAL;
608 }
609
610 if (!nd_drv->probe) {
611 pr_debug("%s ->probe() must be specified\n", mod_name);
612 return -EINVAL;
613 }
614
615 drv->bus = &nvdimm_bus_type;
616 drv->owner = owner;
617 drv->mod_name = mod_name;
618
619 return driver_register(drv);
620}
621EXPORT_SYMBOL(__nd_driver_register);
622
623void nvdimm_check_and_set_ro(struct gendisk *disk)
624{
625 struct device *dev = disk_to_dev(disk)->parent;
626 struct nd_region *nd_region = to_nd_region(dev: dev->parent);
627 int disk_ro = get_disk_ro(disk);
628
629 /* catch the disk up with the region ro state */
630 if (disk_ro == nd_region->ro)
631 return;
632
633 dev_info(dev, "%s read-%s, marking %s read-%s\n",
634 dev_name(&nd_region->dev), nd_region->ro ? "only" : "write",
635 disk->disk_name, nd_region->ro ? "only" : "write");
636 set_disk_ro(disk, read_only: nd_region->ro);
637}
638EXPORT_SYMBOL(nvdimm_check_and_set_ro);
639
640static ssize_t modalias_show(struct device *dev, struct device_attribute *attr,
641 char *buf)
642{
643 return sprintf(buf, ND_DEVICE_MODALIAS_FMT "\n",
644 to_nd_device_type(dev));
645}
646static DEVICE_ATTR_RO(modalias);
647
648static ssize_t devtype_show(struct device *dev, struct device_attribute *attr,
649 char *buf)
650{
651 return sprintf(buf, fmt: "%s\n", dev->type->name);
652}
653static DEVICE_ATTR_RO(devtype);
654
655static struct attribute *nd_device_attributes[] = {
656 &dev_attr_modalias.attr,
657 &dev_attr_devtype.attr,
658 NULL,
659};
660
661/*
662 * nd_device_attribute_group - generic attributes for all devices on an nd bus
663 */
664const struct attribute_group nd_device_attribute_group = {
665 .attrs = nd_device_attributes,
666};
667
668static ssize_t numa_node_show(struct device *dev,
669 struct device_attribute *attr, char *buf)
670{
671 return sprintf(buf, fmt: "%d\n", dev_to_node(dev));
672}
673static DEVICE_ATTR_RO(numa_node);
674
675static int nvdimm_dev_to_target_node(struct device *dev)
676{
677 struct device *parent = dev->parent;
678 struct nd_region *nd_region = NULL;
679
680 if (is_nd_region(dev))
681 nd_region = to_nd_region(dev);
682 else if (parent && is_nd_region(dev: parent))
683 nd_region = to_nd_region(dev: parent);
684
685 if (!nd_region)
686 return NUMA_NO_NODE;
687 return nd_region->target_node;
688}
689
690static ssize_t target_node_show(struct device *dev,
691 struct device_attribute *attr, char *buf)
692{
693 return sprintf(buf, fmt: "%d\n", nvdimm_dev_to_target_node(dev));
694}
695static DEVICE_ATTR_RO(target_node);
696
697static struct attribute *nd_numa_attributes[] = {
698 &dev_attr_numa_node.attr,
699 &dev_attr_target_node.attr,
700 NULL,
701};
702
703static umode_t nd_numa_attr_visible(struct kobject *kobj, struct attribute *a,
704 int n)
705{
706 struct device *dev = container_of(kobj, typeof(*dev), kobj);
707
708 if (!IS_ENABLED(CONFIG_NUMA))
709 return 0;
710
711 if (a == &dev_attr_target_node.attr &&
712 nvdimm_dev_to_target_node(dev) == NUMA_NO_NODE)
713 return 0;
714
715 return a->mode;
716}
717
718/*
719 * nd_numa_attribute_group - NUMA attributes for all devices on an nd bus
720 */
721const struct attribute_group nd_numa_attribute_group = {
722 .attrs = nd_numa_attributes,
723 .is_visible = nd_numa_attr_visible,
724};
725
726static void ndctl_release(struct device *dev)
727{
728 kfree(objp: dev);
729}
730
731static struct lock_class_key nvdimm_ndctl_key;
732
733int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus)
734{
735 dev_t devt = MKDEV(nvdimm_bus_major, nvdimm_bus->id);
736 struct device *dev;
737 int rc;
738
739 dev = kzalloc(size: sizeof(*dev), GFP_KERNEL);
740 if (!dev)
741 return -ENOMEM;
742 device_initialize(dev);
743 lockdep_set_class(&dev->mutex, &nvdimm_ndctl_key);
744 device_set_pm_not_required(dev);
745 dev->class = nd_class;
746 dev->parent = &nvdimm_bus->dev;
747 dev->devt = devt;
748 dev->release = ndctl_release;
749 rc = dev_set_name(dev, name: "ndctl%d", nvdimm_bus->id);
750 if (rc)
751 goto err;
752
753 rc = device_add(dev);
754 if (rc) {
755 dev_dbg(&nvdimm_bus->dev, "failed to register ndctl%d: %d\n",
756 nvdimm_bus->id, rc);
757 goto err;
758 }
759 return 0;
760
761err:
762 put_device(dev);
763 return rc;
764}
765
766void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus)
767{
768 device_destroy(cls: nd_class, MKDEV(nvdimm_bus_major, nvdimm_bus->id));
769}
770
771static const struct nd_cmd_desc __nd_cmd_dimm_descs[] = {
772 [ND_CMD_IMPLEMENTED] = { },
773 [ND_CMD_SMART] = {
774 .out_num = 2,
775 .out_sizes = { 4, 128, },
776 },
777 [ND_CMD_SMART_THRESHOLD] = {
778 .out_num = 2,
779 .out_sizes = { 4, 8, },
780 },
781 [ND_CMD_DIMM_FLAGS] = {
782 .out_num = 2,
783 .out_sizes = { 4, 4 },
784 },
785 [ND_CMD_GET_CONFIG_SIZE] = {
786 .out_num = 3,
787 .out_sizes = { 4, 4, 4, },
788 },
789 [ND_CMD_GET_CONFIG_DATA] = {
790 .in_num = 2,
791 .in_sizes = { 4, 4, },
792 .out_num = 2,
793 .out_sizes = { 4, UINT_MAX, },
794 },
795 [ND_CMD_SET_CONFIG_DATA] = {
796 .in_num = 3,
797 .in_sizes = { 4, 4, UINT_MAX, },
798 .out_num = 1,
799 .out_sizes = { 4, },
800 },
801 [ND_CMD_VENDOR] = {
802 .in_num = 3,
803 .in_sizes = { 4, 4, UINT_MAX, },
804 .out_num = 3,
805 .out_sizes = { 4, 4, UINT_MAX, },
806 },
807 [ND_CMD_CALL] = {
808 .in_num = 2,
809 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
810 .out_num = 1,
811 .out_sizes = { UINT_MAX, },
812 },
813};
814
815const struct nd_cmd_desc *nd_cmd_dimm_desc(int cmd)
816{
817 if (cmd < ARRAY_SIZE(__nd_cmd_dimm_descs))
818 return &__nd_cmd_dimm_descs[cmd];
819 return NULL;
820}
821EXPORT_SYMBOL_GPL(nd_cmd_dimm_desc);
822
823static const struct nd_cmd_desc __nd_cmd_bus_descs[] = {
824 [ND_CMD_IMPLEMENTED] = { },
825 [ND_CMD_ARS_CAP] = {
826 .in_num = 2,
827 .in_sizes = { 8, 8, },
828 .out_num = 4,
829 .out_sizes = { 4, 4, 4, 4, },
830 },
831 [ND_CMD_ARS_START] = {
832 .in_num = 5,
833 .in_sizes = { 8, 8, 2, 1, 5, },
834 .out_num = 2,
835 .out_sizes = { 4, 4, },
836 },
837 [ND_CMD_ARS_STATUS] = {
838 .out_num = 3,
839 .out_sizes = { 4, 4, UINT_MAX, },
840 },
841 [ND_CMD_CLEAR_ERROR] = {
842 .in_num = 2,
843 .in_sizes = { 8, 8, },
844 .out_num = 3,
845 .out_sizes = { 4, 4, 8, },
846 },
847 [ND_CMD_CALL] = {
848 .in_num = 2,
849 .in_sizes = { sizeof(struct nd_cmd_pkg), UINT_MAX, },
850 .out_num = 1,
851 .out_sizes = { UINT_MAX, },
852 },
853};
854
855const struct nd_cmd_desc *nd_cmd_bus_desc(int cmd)
856{
857 if (cmd < ARRAY_SIZE(__nd_cmd_bus_descs))
858 return &__nd_cmd_bus_descs[cmd];
859 return NULL;
860}
861EXPORT_SYMBOL_GPL(nd_cmd_bus_desc);
862
863u32 nd_cmd_in_size(struct nvdimm *nvdimm, int cmd,
864 const struct nd_cmd_desc *desc, int idx, void *buf)
865{
866 if (idx >= desc->in_num)
867 return UINT_MAX;
868
869 if (desc->in_sizes[idx] < UINT_MAX)
870 return desc->in_sizes[idx];
871
872 if (nvdimm && cmd == ND_CMD_SET_CONFIG_DATA && idx == 2) {
873 struct nd_cmd_set_config_hdr *hdr = buf;
874
875 return hdr->in_length;
876 } else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2) {
877 struct nd_cmd_vendor_hdr *hdr = buf;
878
879 return hdr->in_length;
880 } else if (cmd == ND_CMD_CALL) {
881 struct nd_cmd_pkg *pkg = buf;
882
883 return pkg->nd_size_in;
884 }
885
886 return UINT_MAX;
887}
888EXPORT_SYMBOL_GPL(nd_cmd_in_size);
889
890u32 nd_cmd_out_size(struct nvdimm *nvdimm, int cmd,
891 const struct nd_cmd_desc *desc, int idx, const u32 *in_field,
892 const u32 *out_field, unsigned long remainder)
893{
894 if (idx >= desc->out_num)
895 return UINT_MAX;
896
897 if (desc->out_sizes[idx] < UINT_MAX)
898 return desc->out_sizes[idx];
899
900 if (nvdimm && cmd == ND_CMD_GET_CONFIG_DATA && idx == 1)
901 return in_field[1];
902 else if (nvdimm && cmd == ND_CMD_VENDOR && idx == 2)
903 return out_field[1];
904 else if (!nvdimm && cmd == ND_CMD_ARS_STATUS && idx == 2) {
905 /*
906 * Per table 9-276 ARS Data in ACPI 6.1, out_field[1] is
907 * "Size of Output Buffer in bytes, including this
908 * field."
909 */
910 if (out_field[1] < 4)
911 return 0;
912 /*
913 * ACPI 6.1 is ambiguous if 'status' is included in the
914 * output size. If we encounter an output size that
915 * overshoots the remainder by 4 bytes, assume it was
916 * including 'status'.
917 */
918 if (out_field[1] - 4 == remainder)
919 return remainder;
920 return out_field[1] - 8;
921 } else if (cmd == ND_CMD_CALL) {
922 struct nd_cmd_pkg *pkg = (struct nd_cmd_pkg *) in_field;
923
924 return pkg->nd_size_out;
925 }
926
927
928 return UINT_MAX;
929}
930EXPORT_SYMBOL_GPL(nd_cmd_out_size);
931
932void wait_nvdimm_bus_probe_idle(struct device *dev)
933{
934 struct nvdimm_bus *nvdimm_bus = walk_to_nvdimm_bus(nd_dev: dev);
935
936 do {
937 if (nvdimm_bus->probe_active == 0)
938 break;
939 nvdimm_bus_unlock(dev);
940 device_unlock(dev);
941 wait_event(nvdimm_bus->wait,
942 nvdimm_bus->probe_active == 0);
943 device_lock(dev);
944 nvdimm_bus_lock(dev);
945 } while (true);
946}
947
948static int nd_pmem_forget_poison_check(struct device *dev, void *data)
949{
950 struct nd_cmd_clear_error *clear_err =
951 (struct nd_cmd_clear_error *)data;
952 struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL;
953 struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL;
954 struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL;
955 struct nd_namespace_common *ndns = NULL;
956 struct nd_namespace_io *nsio;
957 resource_size_t offset = 0, end_trunc = 0, start, end, pstart, pend;
958
959 if (nd_dax || !dev->driver)
960 return 0;
961
962 start = clear_err->address;
963 end = clear_err->address + clear_err->cleared - 1;
964
965 if (nd_btt || nd_pfn || nd_dax) {
966 if (nd_btt)
967 ndns = nd_btt->ndns;
968 else if (nd_pfn)
969 ndns = nd_pfn->ndns;
970 else if (nd_dax)
971 ndns = nd_dax->nd_pfn.ndns;
972
973 if (!ndns)
974 return 0;
975 } else
976 ndns = to_ndns(dev);
977
978 nsio = to_nd_namespace_io(dev: &ndns->dev);
979 pstart = nsio->res.start + offset;
980 pend = nsio->res.end - end_trunc;
981
982 if ((pstart >= start) && (pend <= end))
983 return -EBUSY;
984
985 return 0;
986
987}
988
989static int nd_ns_forget_poison_check(struct device *dev, void *data)
990{
991 return device_for_each_child(dev, data, fn: nd_pmem_forget_poison_check);
992}
993
994/* set_config requires an idle interleave set */
995static int nd_cmd_clear_to_send(struct nvdimm_bus *nvdimm_bus,
996 struct nvdimm *nvdimm, unsigned int cmd, void *data)
997{
998 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
999
1000 /* ask the bus provider if it would like to block this request */
1001 if (nd_desc->clear_to_send) {
1002 int rc = nd_desc->clear_to_send(nd_desc, nvdimm, cmd, data);
1003
1004 if (rc)
1005 return rc;
1006 }
1007
1008 /* require clear error to go through the pmem driver */
1009 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR)
1010 return device_for_each_child(dev: &nvdimm_bus->dev, data,
1011 fn: nd_ns_forget_poison_check);
1012
1013 if (!nvdimm || cmd != ND_CMD_SET_CONFIG_DATA)
1014 return 0;
1015
1016 /* prevent label manipulation while the kernel owns label updates */
1017 wait_nvdimm_bus_probe_idle(dev: &nvdimm_bus->dev);
1018 if (atomic_read(v: &nvdimm->busy))
1019 return -EBUSY;
1020 return 0;
1021}
1022
1023static int __nd_ioctl(struct nvdimm_bus *nvdimm_bus, struct nvdimm *nvdimm,
1024 int read_only, unsigned int ioctl_cmd, unsigned long arg)
1025{
1026 struct nvdimm_bus_descriptor *nd_desc = nvdimm_bus->nd_desc;
1027 const struct nd_cmd_desc *desc = NULL;
1028 unsigned int cmd = _IOC_NR(ioctl_cmd);
1029 struct device *dev = &nvdimm_bus->dev;
1030 void __user *p = (void __user *) arg;
1031 char *out_env = NULL, *in_env = NULL;
1032 const char *cmd_name, *dimm_name;
1033 u32 in_len = 0, out_len = 0;
1034 unsigned int func = cmd;
1035 unsigned long cmd_mask;
1036 struct nd_cmd_pkg pkg;
1037 int rc, i, cmd_rc;
1038 void *buf = NULL;
1039 u64 buf_len = 0;
1040
1041 if (nvdimm) {
1042 desc = nd_cmd_dimm_desc(cmd);
1043 cmd_name = nvdimm_cmd_name(cmd);
1044 cmd_mask = nvdimm->cmd_mask;
1045 dimm_name = dev_name(dev: &nvdimm->dev);
1046 } else {
1047 desc = nd_cmd_bus_desc(cmd);
1048 cmd_name = nvdimm_bus_cmd_name(cmd);
1049 cmd_mask = nd_desc->cmd_mask;
1050 dimm_name = "bus";
1051 }
1052
1053 /* Validate command family support against bus declared support */
1054 if (cmd == ND_CMD_CALL) {
1055 unsigned long *mask;
1056
1057 if (copy_from_user(to: &pkg, from: p, n: sizeof(pkg)))
1058 return -EFAULT;
1059
1060 if (nvdimm) {
1061 if (pkg.nd_family > NVDIMM_FAMILY_MAX)
1062 return -EINVAL;
1063 mask = &nd_desc->dimm_family_mask;
1064 } else {
1065 if (pkg.nd_family > NVDIMM_BUS_FAMILY_MAX)
1066 return -EINVAL;
1067 mask = &nd_desc->bus_family_mask;
1068 }
1069
1070 if (!test_bit(pkg.nd_family, mask))
1071 return -EINVAL;
1072 }
1073
1074 if (!desc ||
1075 (desc->out_num + desc->in_num == 0) ||
1076 cmd > ND_CMD_CALL ||
1077 !test_bit(cmd, &cmd_mask))
1078 return -ENOTTY;
1079
1080 /* fail write commands (when read-only) */
1081 if (read_only)
1082 switch (cmd) {
1083 case ND_CMD_VENDOR:
1084 case ND_CMD_SET_CONFIG_DATA:
1085 case ND_CMD_ARS_START:
1086 case ND_CMD_CLEAR_ERROR:
1087 case ND_CMD_CALL:
1088 dev_dbg(dev, "'%s' command while read-only.\n",
1089 nvdimm ? nvdimm_cmd_name(cmd)
1090 : nvdimm_bus_cmd_name(cmd));
1091 return -EPERM;
1092 default:
1093 break;
1094 }
1095
1096 /* process an input envelope */
1097 in_env = kzalloc(size: ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1098 if (!in_env)
1099 return -ENOMEM;
1100 for (i = 0; i < desc->in_num; i++) {
1101 u32 in_size, copy;
1102
1103 in_size = nd_cmd_in_size(nvdimm, cmd, desc, i, in_env);
1104 if (in_size == UINT_MAX) {
1105 dev_err(dev, "%s:%s unknown input size cmd: %s field: %d\n",
1106 __func__, dimm_name, cmd_name, i);
1107 rc = -ENXIO;
1108 goto out;
1109 }
1110 if (in_len < ND_CMD_MAX_ENVELOPE)
1111 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - in_len, in_size);
1112 else
1113 copy = 0;
1114 if (copy && copy_from_user(to: &in_env[in_len], from: p + in_len, n: copy)) {
1115 rc = -EFAULT;
1116 goto out;
1117 }
1118 in_len += in_size;
1119 }
1120
1121 if (cmd == ND_CMD_CALL) {
1122 func = pkg.nd_command;
1123 dev_dbg(dev, "%s, idx: %llu, in: %u, out: %u, len %llu\n",
1124 dimm_name, pkg.nd_command,
1125 in_len, out_len, buf_len);
1126 }
1127
1128 /* process an output envelope */
1129 out_env = kzalloc(size: ND_CMD_MAX_ENVELOPE, GFP_KERNEL);
1130 if (!out_env) {
1131 rc = -ENOMEM;
1132 goto out;
1133 }
1134
1135 for (i = 0; i < desc->out_num; i++) {
1136 u32 out_size = nd_cmd_out_size(nvdimm, cmd, desc, i,
1137 (u32 *) in_env, (u32 *) out_env, 0);
1138 u32 copy;
1139
1140 if (out_size == UINT_MAX) {
1141 dev_dbg(dev, "%s unknown output size cmd: %s field: %d\n",
1142 dimm_name, cmd_name, i);
1143 rc = -EFAULT;
1144 goto out;
1145 }
1146 if (out_len < ND_CMD_MAX_ENVELOPE)
1147 copy = min_t(u32, ND_CMD_MAX_ENVELOPE - out_len, out_size);
1148 else
1149 copy = 0;
1150 if (copy && copy_from_user(to: &out_env[out_len],
1151 from: p + in_len + out_len, n: copy)) {
1152 rc = -EFAULT;
1153 goto out;
1154 }
1155 out_len += out_size;
1156 }
1157
1158 buf_len = (u64) out_len + (u64) in_len;
1159 if (buf_len > ND_IOCTL_MAX_BUFLEN) {
1160 dev_dbg(dev, "%s cmd: %s buf_len: %llu > %d\n", dimm_name,
1161 cmd_name, buf_len, ND_IOCTL_MAX_BUFLEN);
1162 rc = -EINVAL;
1163 goto out;
1164 }
1165
1166 buf = vmalloc(size: buf_len);
1167 if (!buf) {
1168 rc = -ENOMEM;
1169 goto out;
1170 }
1171
1172 if (copy_from_user(to: buf, from: p, n: buf_len)) {
1173 rc = -EFAULT;
1174 goto out;
1175 }
1176
1177 device_lock(dev);
1178 nvdimm_bus_lock(dev);
1179 rc = nd_cmd_clear_to_send(nvdimm_bus, nvdimm, cmd: func, data: buf);
1180 if (rc)
1181 goto out_unlock;
1182
1183 rc = nd_desc->ndctl(nd_desc, nvdimm, cmd, buf, buf_len, &cmd_rc);
1184 if (rc < 0)
1185 goto out_unlock;
1186
1187 if (!nvdimm && cmd == ND_CMD_CLEAR_ERROR && cmd_rc >= 0) {
1188 struct nd_cmd_clear_error *clear_err = buf;
1189
1190 nvdimm_account_cleared_poison(nvdimm_bus, phys: clear_err->address,
1191 cleared: clear_err->cleared);
1192 }
1193
1194 if (copy_to_user(to: p, from: buf, n: buf_len))
1195 rc = -EFAULT;
1196
1197out_unlock:
1198 nvdimm_bus_unlock(dev);
1199 device_unlock(dev);
1200out:
1201 kfree(objp: in_env);
1202 kfree(objp: out_env);
1203 vfree(addr: buf);
1204 return rc;
1205}
1206
1207enum nd_ioctl_mode {
1208 BUS_IOCTL,
1209 DIMM_IOCTL,
1210};
1211
1212static int match_dimm(struct device *dev, void *data)
1213{
1214 long id = (long) data;
1215
1216 if (is_nvdimm(dev)) {
1217 struct nvdimm *nvdimm = to_nvdimm(dev);
1218
1219 return nvdimm->id == id;
1220 }
1221
1222 return 0;
1223}
1224
1225static long nd_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
1226 enum nd_ioctl_mode mode)
1227
1228{
1229 struct nvdimm_bus *nvdimm_bus, *found = NULL;
1230 long id = (long) file->private_data;
1231 struct nvdimm *nvdimm = NULL;
1232 int rc, ro;
1233
1234 ro = ((file->f_flags & O_ACCMODE) == O_RDONLY);
1235 mutex_lock(&nvdimm_bus_list_mutex);
1236 list_for_each_entry(nvdimm_bus, &nvdimm_bus_list, list) {
1237 if (mode == DIMM_IOCTL) {
1238 struct device *dev;
1239
1240 dev = device_find_child(dev: &nvdimm_bus->dev,
1241 data: file->private_data, match: match_dimm);
1242 if (!dev)
1243 continue;
1244 nvdimm = to_nvdimm(dev);
1245 found = nvdimm_bus;
1246 } else if (nvdimm_bus->id == id) {
1247 found = nvdimm_bus;
1248 }
1249
1250 if (found) {
1251 atomic_inc(v: &nvdimm_bus->ioctl_active);
1252 break;
1253 }
1254 }
1255 mutex_unlock(lock: &nvdimm_bus_list_mutex);
1256
1257 if (!found)
1258 return -ENXIO;
1259
1260 nvdimm_bus = found;
1261 rc = __nd_ioctl(nvdimm_bus, nvdimm, read_only: ro, ioctl_cmd: cmd, arg);
1262
1263 if (nvdimm)
1264 put_device(dev: &nvdimm->dev);
1265 if (atomic_dec_and_test(v: &nvdimm_bus->ioctl_active))
1266 wake_up(&nvdimm_bus->wait);
1267
1268 return rc;
1269}
1270
1271static long bus_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1272{
1273 return nd_ioctl(file, cmd, arg, mode: BUS_IOCTL);
1274}
1275
1276static long dimm_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1277{
1278 return nd_ioctl(file, cmd, arg, mode: DIMM_IOCTL);
1279}
1280
1281static int nd_open(struct inode *inode, struct file *file)
1282{
1283 long minor = iminor(inode);
1284
1285 file->private_data = (void *) minor;
1286 return 0;
1287}
1288
1289static const struct file_operations nvdimm_bus_fops = {
1290 .owner = THIS_MODULE,
1291 .open = nd_open,
1292 .unlocked_ioctl = bus_ioctl,
1293 .compat_ioctl = compat_ptr_ioctl,
1294 .llseek = noop_llseek,
1295};
1296
1297static const struct file_operations nvdimm_fops = {
1298 .owner = THIS_MODULE,
1299 .open = nd_open,
1300 .unlocked_ioctl = dimm_ioctl,
1301 .compat_ioctl = compat_ptr_ioctl,
1302 .llseek = noop_llseek,
1303};
1304
1305int __init nvdimm_bus_init(void)
1306{
1307 int rc;
1308
1309 rc = bus_register(bus: &nvdimm_bus_type);
1310 if (rc)
1311 return rc;
1312
1313 rc = register_chrdev(major: 0, name: "ndctl", fops: &nvdimm_bus_fops);
1314 if (rc < 0)
1315 goto err_bus_chrdev;
1316 nvdimm_bus_major = rc;
1317
1318 rc = register_chrdev(major: 0, name: "dimmctl", fops: &nvdimm_fops);
1319 if (rc < 0)
1320 goto err_dimm_chrdev;
1321 nvdimm_major = rc;
1322
1323 nd_class = class_create(name: "nd");
1324 if (IS_ERR(ptr: nd_class)) {
1325 rc = PTR_ERR(ptr: nd_class);
1326 goto err_class;
1327 }
1328
1329 rc = driver_register(drv: &nd_bus_driver.drv);
1330 if (rc)
1331 goto err_nd_bus;
1332
1333 return 0;
1334
1335 err_nd_bus:
1336 class_destroy(cls: nd_class);
1337 err_class:
1338 unregister_chrdev(major: nvdimm_major, name: "dimmctl");
1339 err_dimm_chrdev:
1340 unregister_chrdev(major: nvdimm_bus_major, name: "ndctl");
1341 err_bus_chrdev:
1342 bus_unregister(bus: &nvdimm_bus_type);
1343
1344 return rc;
1345}
1346
1347void nvdimm_bus_exit(void)
1348{
1349 driver_unregister(drv: &nd_bus_driver.drv);
1350 class_destroy(cls: nd_class);
1351 unregister_chrdev(major: nvdimm_bus_major, name: "ndctl");
1352 unregister_chrdev(major: nvdimm_major, name: "dimmctl");
1353 bus_unregister(bus: &nvdimm_bus_type);
1354 ida_destroy(ida: &nd_ida);
1355}
1356

source code of linux/drivers/nvdimm/bus.c