1// SPDX-License-Identifier: GPL-2.0-only
2//
3// Copyright(c) 2021-2022 Intel Corporation. All rights reserved.
4//
5// Authors: Cezary Rojewski <cezary.rojewski@intel.com>
6// Amadeusz Slawinski <amadeuszx.slawinski@linux.intel.com>
7//
8
9#include <linux/io-64-nonatomic-lo-hi.h>
10#include <linux/slab.h>
11#include <sound/hdaudio_ext.h>
12#include "avs.h"
13#include "messages.h"
14#include "registers.h"
15#include "trace.h"
16
17#define AVS_IPC_TIMEOUT_MS 300
18#define AVS_D0IX_DELAY_MS 300
19
20static int
21avs_dsp_set_d0ix(struct avs_dev *adev, bool enable)
22{
23 struct avs_ipc *ipc = adev->ipc;
24 int ret;
25
26 /* Is transition required? */
27 if (ipc->in_d0ix == enable)
28 return 0;
29
30 ret = avs_dsp_op(adev, set_d0ix, enable);
31 if (ret) {
32 /* Prevent further d0ix attempts on conscious IPC failure. */
33 if (ret == -AVS_EIPC)
34 atomic_inc(v: &ipc->d0ix_disable_depth);
35
36 ipc->in_d0ix = false;
37 return ret;
38 }
39
40 ipc->in_d0ix = enable;
41 return 0;
42}
43
44static void avs_dsp_schedule_d0ix(struct avs_dev *adev, struct avs_ipc_msg *tx)
45{
46 if (atomic_read(v: &adev->ipc->d0ix_disable_depth))
47 return;
48
49 mod_delayed_work(wq: system_power_efficient_wq, dwork: &adev->ipc->d0ix_work,
50 delay: msecs_to_jiffies(AVS_D0IX_DELAY_MS));
51}
52
53static void avs_dsp_d0ix_work(struct work_struct *work)
54{
55 struct avs_ipc *ipc = container_of(work, struct avs_ipc, d0ix_work.work);
56
57 avs_dsp_set_d0ix(to_avs_dev(ipc->dev), enable: true);
58}
59
60static int avs_dsp_wake_d0i0(struct avs_dev *adev, struct avs_ipc_msg *tx)
61{
62 struct avs_ipc *ipc = adev->ipc;
63
64 if (!atomic_read(v: &ipc->d0ix_disable_depth)) {
65 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
66 return avs_dsp_set_d0ix(adev, enable: false);
67 }
68
69 return 0;
70}
71
72int avs_dsp_disable_d0ix(struct avs_dev *adev)
73{
74 struct avs_ipc *ipc = adev->ipc;
75
76 /* Prevent PG only on the first disable. */
77 if (atomic_inc_return(v: &ipc->d0ix_disable_depth) == 1) {
78 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
79 return avs_dsp_set_d0ix(adev, enable: false);
80 }
81
82 return 0;
83}
84
85int avs_dsp_enable_d0ix(struct avs_dev *adev)
86{
87 struct avs_ipc *ipc = adev->ipc;
88
89 if (atomic_dec_and_test(v: &ipc->d0ix_disable_depth))
90 queue_delayed_work(wq: system_power_efficient_wq, dwork: &ipc->d0ix_work,
91 delay: msecs_to_jiffies(AVS_D0IX_DELAY_MS));
92 return 0;
93}
94
95static void avs_dsp_recovery(struct avs_dev *adev)
96{
97 struct avs_soc_component *acomp;
98 unsigned int core_mask;
99 int ret;
100
101 mutex_lock(&adev->comp_list_mutex);
102 /* disconnect all running streams */
103 list_for_each_entry(acomp, &adev->comp_list, node) {
104 struct snd_soc_pcm_runtime *rtd;
105 struct snd_soc_card *card;
106
107 card = acomp->base.card;
108 if (!card)
109 continue;
110
111 for_each_card_rtds(card, rtd) {
112 struct snd_pcm *pcm;
113 int dir;
114
115 pcm = rtd->pcm;
116 if (!pcm || rtd->dai_link->no_pcm)
117 continue;
118
119 for_each_pcm_streams(dir) {
120 struct snd_pcm_substream *substream;
121
122 substream = pcm->streams[dir].substream;
123 if (!substream || !substream->runtime)
124 continue;
125
126 /* No need for _irq() as we are in nonatomic context. */
127 snd_pcm_stream_lock(substream);
128 snd_pcm_stop(substream, SNDRV_PCM_STATE_DISCONNECTED);
129 snd_pcm_stream_unlock(substream);
130 }
131 }
132 }
133 mutex_unlock(lock: &adev->comp_list_mutex);
134
135 /* forcibly shutdown all cores */
136 core_mask = GENMASK(adev->hw_cfg.dsp_cores - 1, 0);
137 avs_dsp_core_disable(adev, core_mask);
138
139 /* attempt dsp reboot */
140 ret = avs_dsp_boot_firmware(adev, purge: true);
141 if (ret < 0)
142 dev_err(adev->dev, "dsp reboot failed: %d\n", ret);
143
144 pm_runtime_mark_last_busy(dev: adev->dev);
145 pm_runtime_enable(dev: adev->dev);
146 pm_request_autosuspend(dev: adev->dev);
147
148 atomic_set(v: &adev->ipc->recovering, i: 0);
149}
150
151static void avs_dsp_recovery_work(struct work_struct *work)
152{
153 struct avs_ipc *ipc = container_of(work, struct avs_ipc, recovery_work);
154
155 avs_dsp_recovery(to_avs_dev(ipc->dev));
156}
157
158static void avs_dsp_exception_caught(struct avs_dev *adev, union avs_notify_msg *msg)
159{
160 struct avs_ipc *ipc = adev->ipc;
161
162 /* Account for the double-exception case. */
163 ipc->ready = false;
164
165 if (!atomic_add_unless(v: &ipc->recovering, a: 1, u: 1)) {
166 dev_err(adev->dev, "dsp recovery is already in progress\n");
167 return;
168 }
169
170 dev_crit(adev->dev, "communication severed, rebooting dsp..\n");
171
172 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
173 ipc->in_d0ix = false;
174 /* Re-enabled on recovery completion. */
175 pm_runtime_disable(dev: adev->dev);
176
177 /* Process received notification. */
178 avs_dsp_op(adev, coredump, msg);
179
180 schedule_work(work: &ipc->recovery_work);
181}
182
183static void avs_dsp_receive_rx(struct avs_dev *adev, u64 header)
184{
185 struct avs_ipc *ipc = adev->ipc;
186 union avs_reply_msg msg = AVS_MSG(header);
187 u64 reg;
188
189 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
190 trace_avs_ipc_reply_msg(header, fwregs: reg);
191
192 ipc->rx.header = header;
193 /* Abort copying payload if request processing was unsuccessful. */
194 if (!msg.status) {
195 /* update size in case of LARGE_CONFIG_GET */
196 if (msg.msg_target == AVS_MOD_MSG &&
197 msg.global_msg_type == AVS_MOD_LARGE_CONFIG_GET)
198 ipc->rx.size = min_t(u32, AVS_MAILBOX_SIZE,
199 msg.ext.large_config.data_off_size);
200
201 memcpy_fromio(ipc->rx.data, avs_uplink_addr(adev), ipc->rx.size);
202 trace_avs_msg_payload(data: ipc->rx.data, size: ipc->rx.size);
203 }
204}
205
206static void avs_dsp_process_notification(struct avs_dev *adev, u64 header)
207{
208 struct avs_notify_mod_data mod_data;
209 union avs_notify_msg msg = AVS_MSG(header);
210 size_t data_size = 0;
211 void *data = NULL;
212 u64 reg;
213
214 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
215 trace_avs_ipc_notify_msg(header, fwregs: reg);
216
217 /* Ignore spurious notifications until handshake is established. */
218 if (!adev->ipc->ready && msg.notify_msg_type != AVS_NOTIFY_FW_READY) {
219 dev_dbg(adev->dev, "FW not ready, skip notification: 0x%08x\n", msg.primary);
220 return;
221 }
222
223 /* Calculate notification payload size. */
224 switch (msg.notify_msg_type) {
225 case AVS_NOTIFY_FW_READY:
226 break;
227
228 case AVS_NOTIFY_PHRASE_DETECTED:
229 data_size = sizeof(struct avs_notify_voice_data);
230 break;
231
232 case AVS_NOTIFY_RESOURCE_EVENT:
233 data_size = sizeof(struct avs_notify_res_data);
234 break;
235
236 case AVS_NOTIFY_LOG_BUFFER_STATUS:
237 case AVS_NOTIFY_EXCEPTION_CAUGHT:
238 break;
239
240 case AVS_NOTIFY_MODULE_EVENT:
241 /* To know the total payload size, header needs to be read first. */
242 memcpy_fromio(&mod_data, avs_uplink_addr(adev), sizeof(mod_data));
243 data_size = sizeof(mod_data) + mod_data.data_size;
244 break;
245
246 default:
247 dev_info(adev->dev, "unknown notification: 0x%08x\n", msg.primary);
248 break;
249 }
250
251 if (data_size) {
252 data = kmalloc(size: data_size, GFP_KERNEL);
253 if (!data)
254 return;
255
256 memcpy_fromio(data, avs_uplink_addr(adev), data_size);
257 trace_avs_msg_payload(data, size: data_size);
258 }
259
260 /* Perform notification-specific operations. */
261 switch (msg.notify_msg_type) {
262 case AVS_NOTIFY_FW_READY:
263 dev_dbg(adev->dev, "FW READY 0x%08x\n", msg.primary);
264 adev->ipc->ready = true;
265 complete(&adev->fw_ready);
266 break;
267
268 case AVS_NOTIFY_LOG_BUFFER_STATUS:
269 avs_log_buffer_status_locked(adev, msg: &msg);
270 break;
271
272 case AVS_NOTIFY_EXCEPTION_CAUGHT:
273 avs_dsp_exception_caught(adev, msg: &msg);
274 break;
275
276 default:
277 break;
278 }
279
280 kfree(objp: data);
281}
282
283void avs_dsp_process_response(struct avs_dev *adev, u64 header)
284{
285 struct avs_ipc *ipc = adev->ipc;
286
287 /*
288 * Response may either be solicited - a reply for a request that has
289 * been sent beforehand - or unsolicited (notification).
290 */
291 if (avs_msg_is_reply(header)) {
292 /* Response processing is invoked from IRQ thread. */
293 spin_lock_irq(lock: &ipc->rx_lock);
294 avs_dsp_receive_rx(adev, header);
295 ipc->rx_completed = true;
296 spin_unlock_irq(lock: &ipc->rx_lock);
297 } else {
298 avs_dsp_process_notification(adev, header);
299 }
300
301 complete(&ipc->busy_completion);
302}
303
304irqreturn_t avs_irq_handler(struct avs_dev *adev)
305{
306 struct avs_ipc *ipc = adev->ipc;
307 const struct avs_spec *const spec = adev->spec;
308 u32 adspis, hipc_rsp, hipc_ack;
309 irqreturn_t ret = IRQ_NONE;
310
311 adspis = snd_hdac_adsp_readl(adev, AVS_ADSP_REG_ADSPIS);
312 if (adspis == UINT_MAX || !(adspis & AVS_ADSP_ADSPIS_IPC))
313 return ret;
314
315 hipc_ack = snd_hdac_adsp_readl(adev, spec->hipc->ack_offset);
316 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
317
318 /* DSP acked host's request */
319 if (hipc_ack & spec->hipc->ack_done_mask) {
320 /*
321 * As an extra precaution, mask done interrupt. Code executed
322 * due to complete() found below does not assume any masking.
323 */
324 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
325 AVS_ADSP_HIPCCTL_DONE, 0);
326
327 complete(&ipc->done_completion);
328
329 /* tell DSP it has our attention */
330 snd_hdac_adsp_updatel(adev, spec->hipc->ack_offset,
331 spec->hipc->ack_done_mask,
332 spec->hipc->ack_done_mask);
333 /* unmask done interrupt */
334 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
335 AVS_ADSP_HIPCCTL_DONE,
336 AVS_ADSP_HIPCCTL_DONE);
337 ret = IRQ_HANDLED;
338 }
339
340 /* DSP sent new response to process */
341 if (hipc_rsp & spec->hipc->rsp_busy_mask) {
342 /* mask busy interrupt */
343 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset,
344 AVS_ADSP_HIPCCTL_BUSY, 0);
345
346 ret = IRQ_WAKE_THREAD;
347 }
348
349 return ret;
350}
351
352static bool avs_ipc_is_busy(struct avs_ipc *ipc)
353{
354 struct avs_dev *adev = to_avs_dev(ipc->dev);
355 const struct avs_spec *const spec = adev->spec;
356 u32 hipc_rsp;
357
358 hipc_rsp = snd_hdac_adsp_readl(adev, spec->hipc->rsp_offset);
359 return hipc_rsp & spec->hipc->rsp_busy_mask;
360}
361
362static int avs_ipc_wait_busy_completion(struct avs_ipc *ipc, int timeout)
363{
364 u32 repeats_left = 128; /* to avoid infinite looping */
365 int ret;
366
367again:
368 ret = wait_for_completion_timeout(x: &ipc->busy_completion, timeout: msecs_to_jiffies(m: timeout));
369
370 /* DSP could be unresponsive at this point. */
371 if (!ipc->ready)
372 return -EPERM;
373
374 if (!ret) {
375 if (!avs_ipc_is_busy(ipc))
376 return -ETIMEDOUT;
377 /*
378 * Firmware did its job, either notification or reply
379 * has been received - now wait until it's processed.
380 */
381 wait_for_completion_killable(x: &ipc->busy_completion);
382 }
383
384 /* Ongoing notification's bottom-half may cause early wakeup */
385 spin_lock(lock: &ipc->rx_lock);
386 if (!ipc->rx_completed) {
387 if (repeats_left) {
388 /* Reply delayed due to notification. */
389 repeats_left--;
390 reinit_completion(x: &ipc->busy_completion);
391 spin_unlock(lock: &ipc->rx_lock);
392 goto again;
393 }
394
395 spin_unlock(lock: &ipc->rx_lock);
396 return -ETIMEDOUT;
397 }
398
399 spin_unlock(lock: &ipc->rx_lock);
400 return 0;
401}
402
403static void avs_ipc_msg_init(struct avs_ipc *ipc, struct avs_ipc_msg *reply)
404{
405 lockdep_assert_held(&ipc->rx_lock);
406
407 ipc->rx.header = 0;
408 ipc->rx.size = reply ? reply->size : 0;
409 ipc->rx_completed = false;
410
411 reinit_completion(x: &ipc->done_completion);
412 reinit_completion(x: &ipc->busy_completion);
413}
414
415static void avs_dsp_send_tx(struct avs_dev *adev, struct avs_ipc_msg *tx, bool read_fwregs)
416{
417 const struct avs_spec *const spec = adev->spec;
418 u64 reg = ULONG_MAX;
419
420 tx->header |= spec->hipc->req_busy_mask;
421 if (read_fwregs)
422 reg = readq(avs_sram_addr(adev, AVS_FW_REGS_WINDOW));
423
424 trace_avs_request(tx, reg);
425
426 if (tx->size)
427 memcpy_toio(avs_downlink_addr(adev), tx->data, tx->size);
428 snd_hdac_adsp_writel(adev, spec->hipc->req_ext_offset, tx->header >> 32);
429 snd_hdac_adsp_writel(adev, spec->hipc->req_offset, tx->header & UINT_MAX);
430}
431
432static int avs_dsp_do_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
433 struct avs_ipc_msg *reply, int timeout, const char *name)
434{
435 struct avs_ipc *ipc = adev->ipc;
436 int ret;
437
438 if (!ipc->ready)
439 return -EPERM;
440
441 mutex_lock(&ipc->msg_mutex);
442
443 spin_lock(lock: &ipc->rx_lock);
444 avs_ipc_msg_init(ipc, reply);
445 avs_dsp_send_tx(adev, tx: request, read_fwregs: true);
446 spin_unlock(lock: &ipc->rx_lock);
447
448 ret = avs_ipc_wait_busy_completion(ipc, timeout);
449 if (ret) {
450 if (ret == -ETIMEDOUT) {
451 union avs_notify_msg msg = AVS_NOTIFICATION(EXCEPTION_CAUGHT);
452
453 /* Same treatment as on exception, just stack_dump=0. */
454 avs_dsp_exception_caught(adev, msg: &msg);
455 }
456 goto exit;
457 }
458
459 ret = ipc->rx.rsp.status;
460 /*
461 * If IPC channel is blocked e.g.: due to ongoing recovery,
462 * -EPERM error code is expected and thus it's not an actual error.
463 *
464 * Unsupported IPCs are of no harm either.
465 */
466 if (ret == -EPERM || ret == AVS_IPC_NOT_SUPPORTED)
467 dev_dbg(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
468 name, request->glb.primary, request->glb.ext.val, ret);
469 else if (ret)
470 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
471 name, request->glb.primary, request->glb.ext.val, ret);
472
473 if (reply) {
474 reply->header = ipc->rx.header;
475 reply->size = ipc->rx.size;
476 if (reply->data && ipc->rx.size)
477 memcpy(reply->data, ipc->rx.data, reply->size);
478 }
479
480exit:
481 mutex_unlock(lock: &ipc->msg_mutex);
482 return ret;
483}
484
485static int avs_dsp_send_msg_sequence(struct avs_dev *adev, struct avs_ipc_msg *request,
486 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
487 bool schedule_d0ix, const char *name)
488{
489 int ret;
490
491 trace_avs_d0ix(op: "wake", proceed: wake_d0i0, header: request->header);
492 if (wake_d0i0) {
493 ret = avs_dsp_wake_d0i0(adev, tx: request);
494 if (ret)
495 return ret;
496 }
497
498 ret = avs_dsp_do_send_msg(adev, request, reply, timeout, name);
499 if (ret)
500 return ret;
501
502 trace_avs_d0ix(op: "schedule", proceed: schedule_d0ix, header: request->header);
503 if (schedule_d0ix)
504 avs_dsp_schedule_d0ix(adev, tx: request);
505
506 return 0;
507}
508
509int avs_dsp_send_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
510 struct avs_ipc_msg *reply, int timeout, const char *name)
511{
512 bool wake_d0i0 = avs_dsp_op(adev, d0ix_toggle, request, true);
513 bool schedule_d0ix = avs_dsp_op(adev, d0ix_toggle, request, false);
514
515 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix,
516 name);
517}
518
519int avs_dsp_send_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
520 struct avs_ipc_msg *reply, const char *name)
521{
522 return avs_dsp_send_msg_timeout(adev, request, reply, timeout: adev->ipc->default_timeout_ms, name);
523}
524
525int avs_dsp_send_pm_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request,
526 struct avs_ipc_msg *reply, int timeout, bool wake_d0i0,
527 const char *name)
528{
529 return avs_dsp_send_msg_sequence(adev, request, reply, timeout, wake_d0i0, schedule_d0ix: false, name);
530}
531
532int avs_dsp_send_pm_msg(struct avs_dev *adev, struct avs_ipc_msg *request,
533 struct avs_ipc_msg *reply, bool wake_d0i0, const char *name)
534{
535 return avs_dsp_send_pm_msg_timeout(adev, request, reply, timeout: adev->ipc->default_timeout_ms,
536 wake_d0i0, name);
537}
538
539static int avs_dsp_do_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
540 const char *name)
541{
542 struct avs_ipc *ipc = adev->ipc;
543 int ret;
544
545 mutex_lock(&ipc->msg_mutex);
546
547 spin_lock(lock: &ipc->rx_lock);
548 avs_ipc_msg_init(ipc, NULL);
549 /*
550 * with hw still stalled, memory windows may not be
551 * configured properly so avoid accessing SRAM
552 */
553 avs_dsp_send_tx(adev, tx: request, read_fwregs: false);
554 spin_unlock(lock: &ipc->rx_lock);
555
556 /* ROM messages must be sent before main core is unstalled */
557 ret = avs_dsp_op(adev, stall, AVS_MAIN_CORE_MASK, false);
558 if (!ret) {
559 ret = wait_for_completion_timeout(x: &ipc->done_completion, timeout: msecs_to_jiffies(m: timeout));
560 ret = ret ? 0 : -ETIMEDOUT;
561 }
562 if (ret)
563 dev_err(adev->dev, "%s (0x%08x 0x%08x) failed: %d\n",
564 name, request->glb.primary, request->glb.ext.val, ret);
565
566 mutex_unlock(lock: &ipc->msg_mutex);
567
568 return ret;
569}
570
571int avs_dsp_send_rom_msg_timeout(struct avs_dev *adev, struct avs_ipc_msg *request, int timeout,
572 const char *name)
573{
574 return avs_dsp_do_send_rom_msg(adev, request, timeout, name);
575}
576
577int avs_dsp_send_rom_msg(struct avs_dev *adev, struct avs_ipc_msg *request, const char *name)
578{
579 return avs_dsp_send_rom_msg_timeout(adev, request, timeout: adev->ipc->default_timeout_ms, name);
580}
581
582void avs_dsp_interrupt_control(struct avs_dev *adev, bool enable)
583{
584 const struct avs_spec *const spec = adev->spec;
585 u32 value, mask;
586
587 /*
588 * No particular bit setting order. All of these are required
589 * to have a functional SW <-> FW communication.
590 */
591 value = enable ? AVS_ADSP_ADSPIC_IPC : 0;
592 snd_hdac_adsp_updatel(adev, AVS_ADSP_REG_ADSPIC, AVS_ADSP_ADSPIC_IPC, value);
593
594 mask = AVS_ADSP_HIPCCTL_DONE | AVS_ADSP_HIPCCTL_BUSY;
595 value = enable ? mask : 0;
596 snd_hdac_adsp_updatel(adev, spec->hipc->ctl_offset, mask, value);
597}
598
599int avs_ipc_init(struct avs_ipc *ipc, struct device *dev)
600{
601 ipc->rx.data = devm_kzalloc(dev, AVS_MAILBOX_SIZE, GFP_KERNEL);
602 if (!ipc->rx.data)
603 return -ENOMEM;
604
605 ipc->dev = dev;
606 ipc->ready = false;
607 ipc->default_timeout_ms = AVS_IPC_TIMEOUT_MS;
608 INIT_WORK(&ipc->recovery_work, avs_dsp_recovery_work);
609 INIT_DELAYED_WORK(&ipc->d0ix_work, avs_dsp_d0ix_work);
610 init_completion(x: &ipc->done_completion);
611 init_completion(x: &ipc->busy_completion);
612 spin_lock_init(&ipc->rx_lock);
613 mutex_init(&ipc->msg_mutex);
614
615 return 0;
616}
617
618void avs_ipc_block(struct avs_ipc *ipc)
619{
620 ipc->ready = false;
621 cancel_work_sync(work: &ipc->recovery_work);
622 cancel_delayed_work_sync(dwork: &ipc->d0ix_work);
623 ipc->in_d0ix = false;
624}
625

source code of linux/sound/soc/intel/avs/ipc.c