1 | // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
2 | /* |
3 | * Copyright(c) 2018 Intel Corporation. |
4 | * |
5 | */ |
6 | #include "iowait.h" |
7 | #include "trace_iowait.h" |
8 | |
9 | /* 1 priority == 16 starve_cnt */ |
10 | #define IOWAIT_PRIORITY_STARVE_SHIFT 4 |
11 | |
12 | void iowait_set_flag(struct iowait *wait, u32 flag) |
13 | { |
14 | trace_hfi1_iowait_set(wait, flag); |
15 | set_bit(nr: flag, addr: &wait->flags); |
16 | } |
17 | |
18 | bool iowait_flag_set(struct iowait *wait, u32 flag) |
19 | { |
20 | return test_bit(flag, &wait->flags); |
21 | } |
22 | |
23 | inline void iowait_clear_flag(struct iowait *wait, u32 flag) |
24 | { |
25 | trace_hfi1_iowait_clear(wait, flag); |
26 | clear_bit(nr: flag, addr: &wait->flags); |
27 | } |
28 | |
29 | /* |
30 | * iowait_init() - initialize wait structure |
31 | * @wait: wait struct to initialize |
32 | * @tx_limit: limit for overflow queuing |
33 | * @func: restart function for workqueue |
34 | * @sleep: sleep function for no space |
35 | * @resume: wakeup function for no space |
36 | * |
37 | * This function initializes the iowait |
38 | * structure embedded in the QP or PQ. |
39 | * |
40 | */ |
41 | void iowait_init(struct iowait *wait, u32 tx_limit, |
42 | void (*func)(struct work_struct *work), |
43 | void (*tidfunc)(struct work_struct *work), |
44 | int (*sleep)(struct sdma_engine *sde, |
45 | struct iowait_work *wait, |
46 | struct sdma_txreq *tx, |
47 | uint seq, |
48 | bool pkts_sent), |
49 | void (*wakeup)(struct iowait *wait, int reason), |
50 | void (*sdma_drained)(struct iowait *wait), |
51 | void (*init_priority)(struct iowait *wait)) |
52 | { |
53 | int i; |
54 | |
55 | wait->count = 0; |
56 | INIT_LIST_HEAD(list: &wait->list); |
57 | init_waitqueue_head(&wait->wait_dma); |
58 | init_waitqueue_head(&wait->wait_pio); |
59 | atomic_set(v: &wait->sdma_busy, i: 0); |
60 | atomic_set(v: &wait->pio_busy, i: 0); |
61 | wait->tx_limit = tx_limit; |
62 | wait->sleep = sleep; |
63 | wait->wakeup = wakeup; |
64 | wait->sdma_drained = sdma_drained; |
65 | wait->init_priority = init_priority; |
66 | wait->flags = 0; |
67 | for (i = 0; i < IOWAIT_SES; i++) { |
68 | wait->wait[i].iow = wait; |
69 | INIT_LIST_HEAD(list: &wait->wait[i].tx_head); |
70 | if (i == IOWAIT_IB_SE) |
71 | INIT_WORK(&wait->wait[i].iowork, func); |
72 | else |
73 | INIT_WORK(&wait->wait[i].iowork, tidfunc); |
74 | } |
75 | } |
76 | |
77 | /** |
78 | * iowait_cancel_work - cancel all work in iowait |
79 | * @w: the iowait struct |
80 | */ |
81 | void iowait_cancel_work(struct iowait *w) |
82 | { |
83 | cancel_work_sync(work: &iowait_get_ib_work(w)->iowork); |
84 | /* Make sure that the iowork for TID RDMA is used */ |
85 | if (iowait_get_tid_work(w)->iowork.func) |
86 | cancel_work_sync(work: &iowait_get_tid_work(w)->iowork); |
87 | } |
88 | |
89 | /** |
90 | * iowait_set_work_flag - set work flag based on leg |
91 | * @w: the iowait work struct |
92 | */ |
93 | int iowait_set_work_flag(struct iowait_work *w) |
94 | { |
95 | if (w == &w->iow->wait[IOWAIT_IB_SE]) { |
96 | iowait_set_flag(wait: w->iow, IOWAIT_PENDING_IB); |
97 | return IOWAIT_IB_SE; |
98 | } |
99 | iowait_set_flag(wait: w->iow, IOWAIT_PENDING_TID); |
100 | return IOWAIT_TID_SE; |
101 | } |
102 | |
103 | /** |
104 | * iowait_priority_update_top - update the top priority entry |
105 | * @w: the iowait struct |
106 | * @top: a pointer to the top priority entry |
107 | * @idx: the index of the current iowait in an array |
108 | * @top_idx: the array index for the iowait entry that has the top priority |
109 | * |
110 | * This function is called to compare the priority of a given |
111 | * iowait with the given top priority entry. The top index will |
112 | * be returned. |
113 | */ |
114 | uint iowait_priority_update_top(struct iowait *w, |
115 | struct iowait *top, |
116 | uint idx, uint top_idx) |
117 | { |
118 | u8 cnt, tcnt; |
119 | |
120 | /* Convert priority into starve_cnt and compare the total.*/ |
121 | cnt = (w->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + w->starved_cnt; |
122 | tcnt = (top->priority << IOWAIT_PRIORITY_STARVE_SHIFT) + |
123 | top->starved_cnt; |
124 | if (cnt > tcnt) |
125 | return idx; |
126 | else |
127 | return top_idx; |
128 | } |
129 | |