1/*
2 * Copyright(c) 2015 - 2018 Intel Corporation.
3 *
4 * This file is provided under a dual BSD/GPLv2 license. When using or
5 * redistributing this file, you may do so under either license.
6 *
7 * GPL LICENSE SUMMARY
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of version 2 of the GNU General Public License as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 * General Public License for more details.
17 *
18 * BSD LICENSE
19 *
20 * Redistribution and use in source and binary forms, with or without
21 * modification, are permitted provided that the following conditions
22 * are met:
23 *
24 * - Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in
28 * the documentation and/or other materials provided with the
29 * distribution.
30 * - Neither the name of Intel Corporation nor the names of its
31 * contributors may be used to endorse or promote products derived
32 * from this software without specific prior written permission.
33 *
34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 *
46 */
47#include <linux/topology.h>
48#include <linux/cpumask.h>
49#include <linux/module.h>
50#include <linux/interrupt.h>
51#include <linux/numa.h>
52
53#include "hfi.h"
54#include "affinity.h"
55#include "sdma.h"
56#include "trace.h"
57
58struct hfi1_affinity_node_list node_affinity = {
59 .list = LIST_HEAD_INIT(node_affinity.list),
60 .lock = __MUTEX_INITIALIZER(node_affinity.lock)
61};
62
63/* Name of IRQ types, indexed by enum irq_type */
64static const char * const irq_type_names[] = {
65 "SDMA",
66 "RCVCTXT",
67 "GENERAL",
68 "OTHER",
69};
70
71/* Per NUMA node count of HFI devices */
72static unsigned int *hfi1_per_node_cntr;
73
74static inline void init_cpu_mask_set(struct cpu_mask_set *set)
75{
76 cpumask_clear(&set->mask);
77 cpumask_clear(&set->used);
78 set->gen = 0;
79}
80
81/* Increment generation of CPU set if needed */
82static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
83{
84 if (cpumask_equal(&set->mask, &set->used)) {
85 /*
86 * We've used up all the CPUs, bump up the generation
87 * and reset the 'used' map
88 */
89 set->gen++;
90 cpumask_clear(&set->used);
91 }
92}
93
94static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
95{
96 if (cpumask_empty(&set->used) && set->gen) {
97 set->gen--;
98 cpumask_copy(&set->used, &set->mask);
99 }
100}
101
102/* Get the first CPU from the list of unused CPUs in a CPU set data structure */
103static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
104{
105 int cpu;
106
107 if (!diff || !set)
108 return -EINVAL;
109
110 _cpu_mask_set_gen_inc(set);
111
112 /* Find out CPUs left in CPU mask */
113 cpumask_andnot(diff, &set->mask, &set->used);
114
115 cpu = cpumask_first(diff);
116 if (cpu >= nr_cpu_ids) /* empty */
117 cpu = -EINVAL;
118 else
119 cpumask_set_cpu(cpu, &set->used);
120
121 return cpu;
122}
123
124static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
125{
126 if (!set)
127 return;
128
129 cpumask_clear_cpu(cpu, &set->used);
130 _cpu_mask_set_gen_dec(set);
131}
132
133/* Initialize non-HT cpu cores mask */
134void init_real_cpu_mask(void)
135{
136 int possible, curr_cpu, i, ht;
137
138 cpumask_clear(&node_affinity.real_cpu_mask);
139
140 /* Start with cpu online mask as the real cpu mask */
141 cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
142
143 /*
144 * Remove HT cores from the real cpu mask. Do this in two steps below.
145 */
146 possible = cpumask_weight(&node_affinity.real_cpu_mask);
147 ht = cpumask_weight(topology_sibling_cpumask(
148 cpumask_first(&node_affinity.real_cpu_mask)));
149 /*
150 * Step 1. Skip over the first N HT siblings and use them as the
151 * "real" cores. Assumes that HT cores are not enumerated in
152 * succession (except in the single core case).
153 */
154 curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
155 for (i = 0; i < possible / ht; i++)
156 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
157 /*
158 * Step 2. Remove the remaining HT siblings. Use cpumask_next() to
159 * skip any gaps.
160 */
161 for (; i < possible; i++) {
162 cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
163 curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
164 }
165}
166
167int node_affinity_init(void)
168{
169 int node;
170 struct pci_dev *dev = NULL;
171 const struct pci_device_id *ids = hfi1_pci_tbl;
172
173 cpumask_clear(&node_affinity.proc.used);
174 cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
175
176 node_affinity.proc.gen = 0;
177 node_affinity.num_core_siblings =
178 cpumask_weight(topology_sibling_cpumask(
179 cpumask_first(&node_affinity.proc.mask)
180 ));
181 node_affinity.num_possible_nodes = num_possible_nodes();
182 node_affinity.num_online_nodes = num_online_nodes();
183 node_affinity.num_online_cpus = num_online_cpus();
184
185 /*
186 * The real cpu mask is part of the affinity struct but it has to be
187 * initialized early. It is needed to calculate the number of user
188 * contexts in set_up_context_variables().
189 */
190 init_real_cpu_mask();
191
192 hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
193 sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
194 if (!hfi1_per_node_cntr)
195 return -ENOMEM;
196
197 while (ids->vendor) {
198 dev = NULL;
199 while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
200 node = pcibus_to_node(dev->bus);
201 if (node < 0)
202 goto out;
203
204 hfi1_per_node_cntr[node]++;
205 }
206 ids++;
207 }
208
209 return 0;
210
211out:
212 /*
213 * Invalid PCI NUMA node information found, note it, and populate
214 * our database 1:1.
215 */
216 pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
217 pr_err("HFI: System BIOS may need to be upgraded\n");
218 for (node = 0; node < node_affinity.num_possible_nodes; node++)
219 hfi1_per_node_cntr[node] = 1;
220
221 return 0;
222}
223
224static void node_affinity_destroy(struct hfi1_affinity_node *entry)
225{
226 free_percpu(entry->comp_vect_affinity);
227 kfree(entry);
228}
229
230void node_affinity_destroy_all(void)
231{
232 struct list_head *pos, *q;
233 struct hfi1_affinity_node *entry;
234
235 mutex_lock(&node_affinity.lock);
236 list_for_each_safe(pos, q, &node_affinity.list) {
237 entry = list_entry(pos, struct hfi1_affinity_node,
238 list);
239 list_del(pos);
240 node_affinity_destroy(entry);
241 }
242 mutex_unlock(&node_affinity.lock);
243 kfree(hfi1_per_node_cntr);
244}
245
246static struct hfi1_affinity_node *node_affinity_allocate(int node)
247{
248 struct hfi1_affinity_node *entry;
249
250 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
251 if (!entry)
252 return NULL;
253 entry->node = node;
254 entry->comp_vect_affinity = alloc_percpu(u16);
255 INIT_LIST_HEAD(&entry->list);
256
257 return entry;
258}
259
260/*
261 * It appends an entry to the list.
262 * It *must* be called with node_affinity.lock held.
263 */
264static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
265{
266 list_add_tail(&entry->list, &node_affinity.list);
267}
268
269/* It must be called with node_affinity.lock held */
270static struct hfi1_affinity_node *node_affinity_lookup(int node)
271{
272 struct list_head *pos;
273 struct hfi1_affinity_node *entry;
274
275 list_for_each(pos, &node_affinity.list) {
276 entry = list_entry(pos, struct hfi1_affinity_node, list);
277 if (entry->node == node)
278 return entry;
279 }
280
281 return NULL;
282}
283
284static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
285 u16 __percpu *comp_vect_affinity)
286{
287 int curr_cpu;
288 u16 cntr;
289 u16 prev_cntr;
290 int ret_cpu;
291
292 if (!possible_cpumask) {
293 ret_cpu = -EINVAL;
294 goto fail;
295 }
296
297 if (!comp_vect_affinity) {
298 ret_cpu = -EINVAL;
299 goto fail;
300 }
301
302 ret_cpu = cpumask_first(possible_cpumask);
303 if (ret_cpu >= nr_cpu_ids) {
304 ret_cpu = -EINVAL;
305 goto fail;
306 }
307
308 prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
309 for_each_cpu(curr_cpu, possible_cpumask) {
310 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
311
312 if (cntr < prev_cntr) {
313 ret_cpu = curr_cpu;
314 prev_cntr = cntr;
315 }
316 }
317
318 *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
319
320fail:
321 return ret_cpu;
322}
323
324static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
325 u16 __percpu *comp_vect_affinity)
326{
327 int curr_cpu;
328 int max_cpu;
329 u16 cntr;
330 u16 prev_cntr;
331
332 if (!possible_cpumask)
333 return -EINVAL;
334
335 if (!comp_vect_affinity)
336 return -EINVAL;
337
338 max_cpu = cpumask_first(possible_cpumask);
339 if (max_cpu >= nr_cpu_ids)
340 return -EINVAL;
341
342 prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
343 for_each_cpu(curr_cpu, possible_cpumask) {
344 cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
345
346 if (cntr > prev_cntr) {
347 max_cpu = curr_cpu;
348 prev_cntr = cntr;
349 }
350 }
351
352 *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
353
354 return max_cpu;
355}
356
357/*
358 * Non-interrupt CPUs are used first, then interrupt CPUs.
359 * Two already allocated cpu masks must be passed.
360 */
361static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
362 struct hfi1_affinity_node *entry,
363 cpumask_var_t non_intr_cpus,
364 cpumask_var_t available_cpus)
365 __must_hold(&node_affinity.lock)
366{
367 int cpu;
368 struct cpu_mask_set *set = dd->comp_vect;
369
370 lockdep_assert_held(&node_affinity.lock);
371 if (!non_intr_cpus) {
372 cpu = -1;
373 goto fail;
374 }
375
376 if (!available_cpus) {
377 cpu = -1;
378 goto fail;
379 }
380
381 /* Available CPUs for pinning completion vectors */
382 _cpu_mask_set_gen_inc(set);
383 cpumask_andnot(available_cpus, &set->mask, &set->used);
384
385 /* Available CPUs without SDMA engine interrupts */
386 cpumask_andnot(non_intr_cpus, available_cpus,
387 &entry->def_intr.used);
388
389 /* If there are non-interrupt CPUs available, use them first */
390 if (!cpumask_empty(non_intr_cpus))
391 cpu = cpumask_first(non_intr_cpus);
392 else /* Otherwise, use interrupt CPUs */
393 cpu = cpumask_first(available_cpus);
394
395 if (cpu >= nr_cpu_ids) { /* empty */
396 cpu = -1;
397 goto fail;
398 }
399 cpumask_set_cpu(cpu, &set->used);
400
401fail:
402 return cpu;
403}
404
405static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
406{
407 struct cpu_mask_set *set = dd->comp_vect;
408
409 if (cpu < 0)
410 return;
411
412 cpu_mask_set_put(set, cpu);
413}
414
415/* _dev_comp_vect_mappings_destroy() is reentrant */
416static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
417{
418 int i, cpu;
419
420 if (!dd->comp_vect_mappings)
421 return;
422
423 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
424 cpu = dd->comp_vect_mappings[i];
425 _dev_comp_vect_cpu_put(dd, cpu);
426 dd->comp_vect_mappings[i] = -1;
427 hfi1_cdbg(AFFINITY,
428 "[%s] Release CPU %d from completion vector %d",
429 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
430 }
431
432 kfree(dd->comp_vect_mappings);
433 dd->comp_vect_mappings = NULL;
434}
435
436/*
437 * This function creates the table for looking up CPUs for completion vectors.
438 * num_comp_vectors needs to have been initilized before calling this function.
439 */
440static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
441 struct hfi1_affinity_node *entry)
442 __must_hold(&node_affinity.lock)
443{
444 int i, cpu, ret;
445 cpumask_var_t non_intr_cpus;
446 cpumask_var_t available_cpus;
447
448 lockdep_assert_held(&node_affinity.lock);
449
450 if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
451 return -ENOMEM;
452
453 if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
454 free_cpumask_var(non_intr_cpus);
455 return -ENOMEM;
456 }
457
458 dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
459 sizeof(*dd->comp_vect_mappings),
460 GFP_KERNEL);
461 if (!dd->comp_vect_mappings) {
462 ret = -ENOMEM;
463 goto fail;
464 }
465 for (i = 0; i < dd->comp_vect_possible_cpus; i++)
466 dd->comp_vect_mappings[i] = -1;
467
468 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
469 cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
470 available_cpus);
471 if (cpu < 0) {
472 ret = -EINVAL;
473 goto fail;
474 }
475
476 dd->comp_vect_mappings[i] = cpu;
477 hfi1_cdbg(AFFINITY,
478 "[%s] Completion Vector %d -> CPU %d",
479 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
480 }
481
482 return 0;
483
484fail:
485 free_cpumask_var(available_cpus);
486 free_cpumask_var(non_intr_cpus);
487 _dev_comp_vect_mappings_destroy(dd);
488
489 return ret;
490}
491
492int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
493{
494 int ret;
495 struct hfi1_affinity_node *entry;
496
497 mutex_lock(&node_affinity.lock);
498 entry = node_affinity_lookup(dd->node);
499 if (!entry) {
500 ret = -EINVAL;
501 goto unlock;
502 }
503 ret = _dev_comp_vect_mappings_create(dd, entry);
504unlock:
505 mutex_unlock(&node_affinity.lock);
506
507 return ret;
508}
509
510void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
511{
512 _dev_comp_vect_mappings_destroy(dd);
513}
514
515int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
516{
517 struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
518 struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
519
520 if (!dd->comp_vect_mappings)
521 return -EINVAL;
522 if (comp_vect >= dd->comp_vect_possible_cpus)
523 return -EINVAL;
524
525 return dd->comp_vect_mappings[comp_vect];
526}
527
528/*
529 * It assumes dd->comp_vect_possible_cpus is available.
530 */
531static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
532 struct hfi1_affinity_node *entry,
533 bool first_dev_init)
534 __must_hold(&node_affinity.lock)
535{
536 int i, j, curr_cpu;
537 int possible_cpus_comp_vect = 0;
538 struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
539
540 lockdep_assert_held(&node_affinity.lock);
541 /*
542 * If there's only one CPU available for completion vectors, then
543 * there will only be one completion vector available. Othewise,
544 * the number of completion vector available will be the number of
545 * available CPUs divide it by the number of devices in the
546 * local NUMA node.
547 */
548 if (cpumask_weight(&entry->comp_vect_mask) == 1) {
549 possible_cpus_comp_vect = 1;
550 dd_dev_warn(dd,
551 "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
552 } else {
553 possible_cpus_comp_vect +=
554 cpumask_weight(&entry->comp_vect_mask) /
555 hfi1_per_node_cntr[dd->node];
556
557 /*
558 * If the completion vector CPUs available doesn't divide
559 * evenly among devices, then the first device device to be
560 * initialized gets an extra CPU.
561 */
562 if (first_dev_init &&
563 cpumask_weight(&entry->comp_vect_mask) %
564 hfi1_per_node_cntr[dd->node] != 0)
565 possible_cpus_comp_vect++;
566 }
567
568 dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
569
570 /* Reserving CPUs for device completion vector */
571 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
572 curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
573 entry->comp_vect_affinity);
574 if (curr_cpu < 0)
575 goto fail;
576
577 cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
578 }
579
580 hfi1_cdbg(AFFINITY,
581 "[%s] Completion vector affinity CPU set(s) %*pbl",
582 rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
583 cpumask_pr_args(dev_comp_vect_mask));
584
585 return 0;
586
587fail:
588 for (j = 0; j < i; j++)
589 per_cpu_affinity_put_max(&entry->comp_vect_mask,
590 entry->comp_vect_affinity);
591
592 return curr_cpu;
593}
594
595/*
596 * It assumes dd->comp_vect_possible_cpus is available.
597 */
598static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
599 struct hfi1_affinity_node *entry)
600 __must_hold(&node_affinity.lock)
601{
602 int i, cpu;
603
604 lockdep_assert_held(&node_affinity.lock);
605 if (!dd->comp_vect_possible_cpus)
606 return;
607
608 for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
609 cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
610 entry->comp_vect_affinity);
611 /* Clearing CPU in device completion vector cpu mask */
612 if (cpu >= 0)
613 cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
614 }
615
616 dd->comp_vect_possible_cpus = 0;
617}
618
619/*
620 * Interrupt affinity.
621 *
622 * non-rcv avail gets a default mask that
623 * starts as possible cpus with threads reset
624 * and each rcv avail reset.
625 *
626 * rcv avail gets node relative 1 wrapping back
627 * to the node relative 1 as necessary.
628 *
629 */
630int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
631{
632 int node = pcibus_to_node(dd->pcidev->bus);
633 struct hfi1_affinity_node *entry;
634 const struct cpumask *local_mask;
635 int curr_cpu, possible, i, ret;
636 bool new_entry = false;
637
638 /*
639 * If the BIOS does not have the NUMA node information set, select
640 * NUMA 0 so we get consistent performance.
641 */
642 if (node < 0) {
643 dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
644 node = 0;
645 }
646 dd->node = node;
647
648 local_mask = cpumask_of_node(dd->node);
649 if (cpumask_first(local_mask) >= nr_cpu_ids)
650 local_mask = topology_core_cpumask(0);
651
652 mutex_lock(&node_affinity.lock);
653 entry = node_affinity_lookup(dd->node);
654
655 /*
656 * If this is the first time this NUMA node's affinity is used,
657 * create an entry in the global affinity structure and initialize it.
658 */
659 if (!entry) {
660 entry = node_affinity_allocate(node);
661 if (!entry) {
662 dd_dev_err(dd,
663 "Unable to allocate global affinity node\n");
664 ret = -ENOMEM;
665 goto fail;
666 }
667 new_entry = true;
668
669 init_cpu_mask_set(&entry->def_intr);
670 init_cpu_mask_set(&entry->rcv_intr);
671 cpumask_clear(&entry->comp_vect_mask);
672 cpumask_clear(&entry->general_intr_mask);
673 /* Use the "real" cpu mask of this node as the default */
674 cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
675 local_mask);
676
677 /* fill in the receive list */
678 possible = cpumask_weight(&entry->def_intr.mask);
679 curr_cpu = cpumask_first(&entry->def_intr.mask);
680
681 if (possible == 1) {
682 /* only one CPU, everyone will use it */
683 cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
684 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
685 } else {
686 /*
687 * The general/control context will be the first CPU in
688 * the default list, so it is removed from the default
689 * list and added to the general interrupt list.
690 */
691 cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
692 cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
693 curr_cpu = cpumask_next(curr_cpu,
694 &entry->def_intr.mask);
695
696 /*
697 * Remove the remaining kernel receive queues from
698 * the default list and add them to the receive list.
699 */
700 for (i = 0;
701 i < (dd->n_krcv_queues - 1) *
702 hfi1_per_node_cntr[dd->node];
703 i++) {
704 cpumask_clear_cpu(curr_cpu,
705 &entry->def_intr.mask);
706 cpumask_set_cpu(curr_cpu,
707 &entry->rcv_intr.mask);
708 curr_cpu = cpumask_next(curr_cpu,
709 &entry->def_intr.mask);
710 if (curr_cpu >= nr_cpu_ids)
711 break;
712 }
713
714 /*
715 * If there ends up being 0 CPU cores leftover for SDMA
716 * engines, use the same CPU cores as general/control
717 * context.
718 */
719 if (cpumask_weight(&entry->def_intr.mask) == 0)
720 cpumask_copy(&entry->def_intr.mask,
721 &entry->general_intr_mask);
722 }
723
724 /* Determine completion vector CPUs for the entire node */
725 cpumask_and(&entry->comp_vect_mask,
726 &node_affinity.real_cpu_mask, local_mask);
727 cpumask_andnot(&entry->comp_vect_mask,
728 &entry->comp_vect_mask,
729 &entry->rcv_intr.mask);
730 cpumask_andnot(&entry->comp_vect_mask,
731 &entry->comp_vect_mask,
732 &entry->general_intr_mask);
733
734 /*
735 * If there ends up being 0 CPU cores leftover for completion
736 * vectors, use the same CPU core as the general/control
737 * context.
738 */
739 if (cpumask_weight(&entry->comp_vect_mask) == 0)
740 cpumask_copy(&entry->comp_vect_mask,
741 &entry->general_intr_mask);
742 }
743
744 ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
745 if (ret < 0)
746 goto fail;
747
748 if (new_entry)
749 node_affinity_add_tail(entry);
750
751 mutex_unlock(&node_affinity.lock);
752
753 return 0;
754
755fail:
756 if (new_entry)
757 node_affinity_destroy(entry);
758 mutex_unlock(&node_affinity.lock);
759 return ret;
760}
761
762void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
763{
764 struct hfi1_affinity_node *entry;
765
766 if (dd->node < 0)
767 return;
768
769 mutex_lock(&node_affinity.lock);
770 entry = node_affinity_lookup(dd->node);
771 if (!entry)
772 goto unlock;
773
774 /*
775 * Free device completion vector CPUs to be used by future
776 * completion vectors
777 */
778 _dev_comp_vect_cpu_mask_clean_up(dd, entry);
779unlock:
780 mutex_unlock(&node_affinity.lock);
781 dd->node = NUMA_NO_NODE;
782}
783
784/*
785 * Function updates the irq affinity hint for msix after it has been changed
786 * by the user using the /proc/irq interface. This function only accepts
787 * one cpu in the mask.
788 */
789static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
790{
791 struct sdma_engine *sde = msix->arg;
792 struct hfi1_devdata *dd = sde->dd;
793 struct hfi1_affinity_node *entry;
794 struct cpu_mask_set *set;
795 int i, old_cpu;
796
797 if (cpu > num_online_cpus() || cpu == sde->cpu)
798 return;
799
800 mutex_lock(&node_affinity.lock);
801 entry = node_affinity_lookup(dd->node);
802 if (!entry)
803 goto unlock;
804
805 old_cpu = sde->cpu;
806 sde->cpu = cpu;
807 cpumask_clear(&msix->mask);
808 cpumask_set_cpu(cpu, &msix->mask);
809 dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
810 msix->irq, irq_type_names[msix->type],
811 sde->this_idx, cpu);
812 irq_set_affinity_hint(msix->irq, &msix->mask);
813
814 /*
815 * Set the new cpu in the hfi1_affinity_node and clean
816 * the old cpu if it is not used by any other IRQ
817 */
818 set = &entry->def_intr;
819 cpumask_set_cpu(cpu, &set->mask);
820 cpumask_set_cpu(cpu, &set->used);
821 for (i = 0; i < dd->msix_info.max_requested; i++) {
822 struct hfi1_msix_entry *other_msix;
823
824 other_msix = &dd->msix_info.msix_entries[i];
825 if (other_msix->type != IRQ_SDMA || other_msix == msix)
826 continue;
827
828 if (cpumask_test_cpu(old_cpu, &other_msix->mask))
829 goto unlock;
830 }
831 cpumask_clear_cpu(old_cpu, &set->mask);
832 cpumask_clear_cpu(old_cpu, &set->used);
833unlock:
834 mutex_unlock(&node_affinity.lock);
835}
836
837static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
838 const cpumask_t *mask)
839{
840 int cpu = cpumask_first(mask);
841 struct hfi1_msix_entry *msix = container_of(notify,
842 struct hfi1_msix_entry,
843 notify);
844
845 /* Only one CPU configuration supported currently */
846 hfi1_update_sdma_affinity(msix, cpu);
847}
848
849static void hfi1_irq_notifier_release(struct kref *ref)
850{
851 /*
852 * This is required by affinity notifier. We don't have anything to
853 * free here.
854 */
855}
856
857static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
858{
859 struct irq_affinity_notify *notify = &msix->notify;
860
861 notify->irq = msix->irq;
862 notify->notify = hfi1_irq_notifier_notify;
863 notify->release = hfi1_irq_notifier_release;
864
865 if (irq_set_affinity_notifier(notify->irq, notify))
866 pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
867 notify->irq);
868}
869
870static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
871{
872 struct irq_affinity_notify *notify = &msix->notify;
873
874 if (irq_set_affinity_notifier(notify->irq, NULL))
875 pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
876 notify->irq);
877}
878
879/*
880 * Function sets the irq affinity for msix.
881 * It *must* be called with node_affinity.lock held.
882 */
883static int get_irq_affinity(struct hfi1_devdata *dd,
884 struct hfi1_msix_entry *msix)
885{
886 cpumask_var_t diff;
887 struct hfi1_affinity_node *entry;
888 struct cpu_mask_set *set = NULL;
889 struct sdma_engine *sde = NULL;
890 struct hfi1_ctxtdata *rcd = NULL;
891 char extra[64];
892 int cpu = -1;
893
894 extra[0] = '\0';
895 cpumask_clear(&msix->mask);
896
897 entry = node_affinity_lookup(dd->node);
898
899 switch (msix->type) {
900 case IRQ_SDMA:
901 sde = (struct sdma_engine *)msix->arg;
902 scnprintf(extra, 64, "engine %u", sde->this_idx);
903 set = &entry->def_intr;
904 break;
905 case IRQ_GENERAL:
906 cpu = cpumask_first(&entry->general_intr_mask);
907 break;
908 case IRQ_RCVCTXT:
909 rcd = (struct hfi1_ctxtdata *)msix->arg;
910 if (rcd->ctxt == HFI1_CTRL_CTXT)
911 cpu = cpumask_first(&entry->general_intr_mask);
912 else
913 set = &entry->rcv_intr;
914 scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
915 break;
916 default:
917 dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
918 return -EINVAL;
919 }
920
921 /*
922 * The general and control contexts are placed on a particular
923 * CPU, which is set above. Skip accounting for it. Everything else
924 * finds its CPU here.
925 */
926 if (cpu == -1 && set) {
927 if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
928 return -ENOMEM;
929
930 cpu = cpu_mask_set_get_first(set, diff);
931 if (cpu < 0) {
932 free_cpumask_var(diff);
933 dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
934 return cpu;
935 }
936
937 free_cpumask_var(diff);
938 }
939
940 cpumask_set_cpu(cpu, &msix->mask);
941 dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
942 msix->irq, irq_type_names[msix->type],
943 extra, cpu);
944 irq_set_affinity_hint(msix->irq, &msix->mask);
945
946 if (msix->type == IRQ_SDMA) {
947 sde->cpu = cpu;
948 hfi1_setup_sdma_notifier(msix);
949 }
950
951 return 0;
952}
953
954int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
955{
956 int ret;
957
958 mutex_lock(&node_affinity.lock);
959 ret = get_irq_affinity(dd, msix);
960 mutex_unlock(&node_affinity.lock);
961 return ret;
962}
963
964void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
965 struct hfi1_msix_entry *msix)
966{
967 struct cpu_mask_set *set = NULL;
968 struct hfi1_ctxtdata *rcd;
969 struct hfi1_affinity_node *entry;
970
971 mutex_lock(&node_affinity.lock);
972 entry = node_affinity_lookup(dd->node);
973
974 switch (msix->type) {
975 case IRQ_SDMA:
976 set = &entry->def_intr;
977 hfi1_cleanup_sdma_notifier(msix);
978 break;
979 case IRQ_GENERAL:
980 /* Don't do accounting for general contexts */
981 break;
982 case IRQ_RCVCTXT:
983 rcd = (struct hfi1_ctxtdata *)msix->arg;
984 /* Don't do accounting for control contexts */
985 if (rcd->ctxt != HFI1_CTRL_CTXT)
986 set = &entry->rcv_intr;
987 break;
988 default:
989 mutex_unlock(&node_affinity.lock);
990 return;
991 }
992
993 if (set) {
994 cpumask_andnot(&set->used, &set->used, &msix->mask);
995 _cpu_mask_set_gen_dec(set);
996 }
997
998 irq_set_affinity_hint(msix->irq, NULL);
999 cpumask_clear(&msix->mask);
1000 mutex_unlock(&node_affinity.lock);
1001}
1002
1003/* This should be called with node_affinity.lock held */
1004static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
1005 struct hfi1_affinity_node_list *affinity)
1006{
1007 int possible, curr_cpu, i;
1008 uint num_cores_per_socket = node_affinity.num_online_cpus /
1009 affinity->num_core_siblings /
1010 node_affinity.num_online_nodes;
1011
1012 cpumask_copy(hw_thread_mask, &affinity->proc.mask);
1013 if (affinity->num_core_siblings > 0) {
1014 /* Removing other siblings not needed for now */
1015 possible = cpumask_weight(hw_thread_mask);
1016 curr_cpu = cpumask_first(hw_thread_mask);
1017 for (i = 0;
1018 i < num_cores_per_socket * node_affinity.num_online_nodes;
1019 i++)
1020 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1021
1022 for (; i < possible; i++) {
1023 cpumask_clear_cpu(curr_cpu, hw_thread_mask);
1024 curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1025 }
1026
1027 /* Identifying correct HW threads within physical cores */
1028 cpumask_shift_left(hw_thread_mask, hw_thread_mask,
1029 num_cores_per_socket *
1030 node_affinity.num_online_nodes *
1031 hw_thread_no);
1032 }
1033}
1034
1035int hfi1_get_proc_affinity(int node)
1036{
1037 int cpu = -1, ret, i;
1038 struct hfi1_affinity_node *entry;
1039 cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
1040 const struct cpumask *node_mask,
1041 *proc_mask = &current->cpus_allowed;
1042 struct hfi1_affinity_node_list *affinity = &node_affinity;
1043 struct cpu_mask_set *set = &affinity->proc;
1044
1045 /*
1046 * check whether process/context affinity has already
1047 * been set
1048 */
1049 if (cpumask_weight(proc_mask) == 1) {
1050 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
1051 current->pid, current->comm,
1052 cpumask_pr_args(proc_mask));
1053 /*
1054 * Mark the pre-set CPU as used. This is atomic so we don't
1055 * need the lock
1056 */
1057 cpu = cpumask_first(proc_mask);
1058 cpumask_set_cpu(cpu, &set->used);
1059 goto done;
1060 } else if (cpumask_weight(proc_mask) < cpumask_weight(&set->mask)) {
1061 hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
1062 current->pid, current->comm,
1063 cpumask_pr_args(proc_mask));
1064 goto done;
1065 }
1066
1067 /*
1068 * The process does not have a preset CPU affinity so find one to
1069 * recommend using the following algorithm:
1070 *
1071 * For each user process that is opening a context on HFI Y:
1072 * a) If all cores are filled, reinitialize the bitmask
1073 * b) Fill real cores first, then HT cores (First set of HT
1074 * cores on all physical cores, then second set of HT core,
1075 * and, so on) in the following order:
1076 *
1077 * 1. Same NUMA node as HFI Y and not running an IRQ
1078 * handler
1079 * 2. Same NUMA node as HFI Y and running an IRQ handler
1080 * 3. Different NUMA node to HFI Y and not running an IRQ
1081 * handler
1082 * 4. Different NUMA node to HFI Y and running an IRQ
1083 * handler
1084 * c) Mark core as filled in the bitmask. As user processes are
1085 * done, clear cores from the bitmask.
1086 */
1087
1088 ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
1089 if (!ret)
1090 goto done;
1091 ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
1092 if (!ret)
1093 goto free_diff;
1094 ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
1095 if (!ret)
1096 goto free_hw_thread_mask;
1097 ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
1098 if (!ret)
1099 goto free_available_mask;
1100
1101 mutex_lock(&affinity->lock);
1102 /*
1103 * If we've used all available HW threads, clear the mask and start
1104 * overloading.
1105 */
1106 _cpu_mask_set_gen_inc(set);
1107
1108 /*
1109 * If NUMA node has CPUs used by interrupt handlers, include them in the
1110 * interrupt handler mask.
1111 */
1112 entry = node_affinity_lookup(node);
1113 if (entry) {
1114 cpumask_copy(intrs_mask, (entry->def_intr.gen ?
1115 &entry->def_intr.mask :
1116 &entry->def_intr.used));
1117 cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
1118 &entry->rcv_intr.mask :
1119 &entry->rcv_intr.used));
1120 cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
1121 }
1122 hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
1123 cpumask_pr_args(intrs_mask));
1124
1125 cpumask_copy(hw_thread_mask, &set->mask);
1126
1127 /*
1128 * If HT cores are enabled, identify which HW threads within the
1129 * physical cores should be used.
1130 */
1131 if (affinity->num_core_siblings > 0) {
1132 for (i = 0; i < affinity->num_core_siblings; i++) {
1133 find_hw_thread_mask(i, hw_thread_mask, affinity);
1134
1135 /*
1136 * If there's at least one available core for this HW
1137 * thread number, stop looking for a core.
1138 *
1139 * diff will always be not empty at least once in this
1140 * loop as the used mask gets reset when
1141 * (set->mask == set->used) before this loop.
1142 */
1143 cpumask_andnot(diff, hw_thread_mask, &set->used);
1144 if (!cpumask_empty(diff))
1145 break;
1146 }
1147 }
1148 hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
1149 cpumask_pr_args(hw_thread_mask));
1150
1151 node_mask = cpumask_of_node(node);
1152 hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
1153 cpumask_pr_args(node_mask));
1154
1155 /* Get cpumask of available CPUs on preferred NUMA */
1156 cpumask_and(available_mask, hw_thread_mask, node_mask);
1157 cpumask_andnot(available_mask, available_mask, &set->used);
1158 hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
1159 cpumask_pr_args(available_mask));
1160
1161 /*
1162 * At first, we don't want to place processes on the same
1163 * CPUs as interrupt handlers. Then, CPUs running interrupt
1164 * handlers are used.
1165 *
1166 * 1) If diff is not empty, then there are CPUs not running
1167 * non-interrupt handlers available, so diff gets copied
1168 * over to available_mask.
1169 * 2) If diff is empty, then all CPUs not running interrupt
1170 * handlers are taken, so available_mask contains all
1171 * available CPUs running interrupt handlers.
1172 * 3) If available_mask is empty, then all CPUs on the
1173 * preferred NUMA node are taken, so other NUMA nodes are
1174 * used for process assignments using the same method as
1175 * the preferred NUMA node.
1176 */
1177 cpumask_andnot(diff, available_mask, intrs_mask);
1178 if (!cpumask_empty(diff))
1179 cpumask_copy(available_mask, diff);
1180
1181 /* If we don't have CPUs on the preferred node, use other NUMA nodes */
1182 if (cpumask_empty(available_mask)) {
1183 cpumask_andnot(available_mask, hw_thread_mask, &set->used);
1184 /* Excluding preferred NUMA cores */
1185 cpumask_andnot(available_mask, available_mask, node_mask);
1186 hfi1_cdbg(PROC,
1187 "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
1188 cpumask_pr_args(available_mask));
1189
1190 /*
1191 * At first, we don't want to place processes on the same
1192 * CPUs as interrupt handlers.
1193 */
1194 cpumask_andnot(diff, available_mask, intrs_mask);
1195 if (!cpumask_empty(diff))
1196 cpumask_copy(available_mask, diff);
1197 }
1198 hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
1199 cpumask_pr_args(available_mask));
1200
1201 cpu = cpumask_first(available_mask);
1202 if (cpu >= nr_cpu_ids) /* empty */
1203 cpu = -1;
1204 else
1205 cpumask_set_cpu(cpu, &set->used);
1206
1207 mutex_unlock(&affinity->lock);
1208 hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
1209
1210 free_cpumask_var(intrs_mask);
1211free_available_mask:
1212 free_cpumask_var(available_mask);
1213free_hw_thread_mask:
1214 free_cpumask_var(hw_thread_mask);
1215free_diff:
1216 free_cpumask_var(diff);
1217done:
1218 return cpu;
1219}
1220
1221void hfi1_put_proc_affinity(int cpu)
1222{
1223 struct hfi1_affinity_node_list *affinity = &node_affinity;
1224 struct cpu_mask_set *set = &affinity->proc;
1225
1226 if (cpu < 0)
1227 return;
1228
1229 mutex_lock(&affinity->lock);
1230 cpu_mask_set_put(set, cpu);
1231 hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
1232 mutex_unlock(&affinity->lock);
1233}
1234