1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Atmel MultiMedia Card Interface driver
4	*
5	* Copyright (C) 2004-2008 Atmel Corporation
6	*/
7	#include <linux/blkdev.h>
8	#include <linux/clk.h>
9	#include <linux/debugfs.h>
10	#include <linux/device.h>
11	#include <linux/dmaengine.h>
12	#include <linux/dma-mapping.h>
13	#include <linux/err.h>
14	#include <linux/init.h>
15	#include <linux/interrupt.h>
16	#include <linux/io.h>
17	#include <linux/ioport.h>
18	#include <linux/module.h>
19	#include <linux/of.h>
20	#include <linux/irq.h>
21	#include <linux/gpio/consumer.h>
22	#include <linux/platform_device.h>
23	#include <linux/scatterlist.h>
24	#include <linux/seq_file.h>
25	#include <linux/slab.h>
26	#include <linux/stat.h>
27	#include <linux/types.h>
28
29	#include <linux/mmc/host.h>
30	#include <linux/mmc/sdio.h>
31
32	#include <linux/atmel_pdc.h>
33	#include <linux/pm.h>
34	#include <linux/pm_runtime.h>
35	#include <linux/pinctrl/consumer.h>
36
37	#include <asm/cacheflush.h>
38	#include <asm/io.h>
39	#include <asm/unaligned.h>
40
41	#define ATMCI_MAX_NR_SLOTS 2
42
43	/*
44	* Superset of MCI IP registers integrated in Atmel AT91 Processor
45	* Registers and bitfields marked with [2] are only available in MCI2
46	*/
47
48	/* MCI Register Definitions */
49	#define ATMCI_CR 0x0000 /* Control */
50	#define ATMCI_CR_MCIEN BIT(0) /* MCI Enable */
51	#define ATMCI_CR_MCIDIS BIT(1) /* MCI Disable */
52	#define ATMCI_CR_PWSEN BIT(2) /* Power Save Enable */
53	#define ATMCI_CR_PWSDIS BIT(3) /* Power Save Disable */
54	#define ATMCI_CR_SWRST BIT(7) /* Software Reset */
55	#define ATMCI_MR 0x0004 /* Mode */
56	#define ATMCI_MR_CLKDIV(x) ((x) << 0) /* Clock Divider */
57	#define ATMCI_MR_PWSDIV(x) ((x) << 8) /* Power Saving Divider */
58	#define ATMCI_MR_RDPROOF BIT(11) /* Read Proof */
59	#define ATMCI_MR_WRPROOF BIT(12) /* Write Proof */
60	#define ATMCI_MR_PDCFBYTE BIT(13) /* Force Byte Transfer */
61	#define ATMCI_MR_PDCPADV BIT(14) /* Padding Value */
62	#define ATMCI_MR_PDCMODE BIT(15) /* PDC-oriented Mode */
63	#define ATMCI_MR_CLKODD(x) ((x) << 16) /* LSB of Clock Divider */
64	#define ATMCI_DTOR 0x0008 /* Data Timeout */
65	#define ATMCI_DTOCYC(x) ((x) << 0) /* Data Timeout Cycles */
66	#define ATMCI_DTOMUL(x) ((x) << 4) /* Data Timeout Multiplier */
67	#define ATMCI_SDCR 0x000c /* SD Card / SDIO */
68	#define ATMCI_SDCSEL_SLOT_A (0 << 0) /* Select SD slot A */
69	#define ATMCI_SDCSEL_SLOT_B (1 << 0) /* Select SD slot A */
70	#define ATMCI_SDCSEL_MASK (3 << 0)
71	#define ATMCI_SDCBUS_1BIT (0 << 6) /* 1-bit data bus */
72	#define ATMCI_SDCBUS_4BIT (2 << 6) /* 4-bit data bus */
73	#define ATMCI_SDCBUS_8BIT (3 << 6) /* 8-bit data bus[2] */
74	#define ATMCI_SDCBUS_MASK (3 << 6)
75	#define ATMCI_ARGR 0x0010 /* Command Argument */
76	#define ATMCI_CMDR 0x0014 /* Command */
77	#define ATMCI_CMDR_CMDNB(x) ((x) << 0) /* Command Opcode */
78	#define ATMCI_CMDR_RSPTYP_NONE (0 << 6) /* No response */
79	#define ATMCI_CMDR_RSPTYP_48BIT (1 << 6) /* 48-bit response */
80	#define ATMCI_CMDR_RSPTYP_136BIT (2 << 6) /* 136-bit response */
81	#define ATMCI_CMDR_SPCMD_INIT (1 << 8) /* Initialization command */
82	#define ATMCI_CMDR_SPCMD_SYNC (2 << 8) /* Synchronized command */
83	#define ATMCI_CMDR_SPCMD_INT (4 << 8) /* Interrupt command */
84	#define ATMCI_CMDR_SPCMD_INTRESP (5 << 8) /* Interrupt response */
85	#define ATMCI_CMDR_OPDCMD (1 << 11) /* Open Drain */
86	#define ATMCI_CMDR_MAXLAT_5CYC (0 << 12) /* Max latency 5 cycles */
87	#define ATMCI_CMDR_MAXLAT_64CYC (1 << 12) /* Max latency 64 cycles */
88	#define ATMCI_CMDR_START_XFER (1 << 16) /* Start data transfer */
89	#define ATMCI_CMDR_STOP_XFER (2 << 16) /* Stop data transfer */
90	#define ATMCI_CMDR_TRDIR_WRITE (0 << 18) /* Write data */
91	#define ATMCI_CMDR_TRDIR_READ (1 << 18) /* Read data */
92	#define ATMCI_CMDR_BLOCK (0 << 19) /* Single-block transfer */
93	#define ATMCI_CMDR_MULTI_BLOCK (1 << 19) /* Multi-block transfer */
94	#define ATMCI_CMDR_STREAM (2 << 19) /* MMC Stream transfer */
95	#define ATMCI_CMDR_SDIO_BYTE (4 << 19) /* SDIO Byte transfer */
96	#define ATMCI_CMDR_SDIO_BLOCK (5 << 19) /* SDIO Block transfer */
97	#define ATMCI_CMDR_SDIO_SUSPEND (1 << 24) /* SDIO Suspend Command */
98	#define ATMCI_CMDR_SDIO_RESUME (2 << 24) /* SDIO Resume Command */
99	#define ATMCI_BLKR 0x0018 /* Block */
100	#define ATMCI_BCNT(x) ((x) << 0) /* Data Block Count */
101	#define ATMCI_BLKLEN(x) ((x) << 16) /* Data Block Length */
102	#define ATMCI_CSTOR 0x001c /* Completion Signal Timeout[2] */
103	#define ATMCI_CSTOCYC(x) ((x) << 0) /* CST cycles */
104	#define ATMCI_CSTOMUL(x) ((x) << 4) /* CST multiplier */
105	#define ATMCI_RSPR 0x0020 /* Response 0 */
106	#define ATMCI_RSPR1 0x0024 /* Response 1 */
107	#define ATMCI_RSPR2 0x0028 /* Response 2 */
108	#define ATMCI_RSPR3 0x002c /* Response 3 */
109	#define ATMCI_RDR 0x0030 /* Receive Data */
110	#define ATMCI_TDR 0x0034 /* Transmit Data */
111	#define ATMCI_SR 0x0040 /* Status */
112	#define ATMCI_IER 0x0044 /* Interrupt Enable */
113	#define ATMCI_IDR 0x0048 /* Interrupt Disable */
114	#define ATMCI_IMR 0x004c /* Interrupt Mask */
115	#define ATMCI_CMDRDY BIT(0) /* Command Ready */
116	#define ATMCI_RXRDY BIT(1) /* Receiver Ready */
117	#define ATMCI_TXRDY BIT(2) /* Transmitter Ready */
118	#define ATMCI_BLKE BIT(3) /* Data Block Ended */
119	#define ATMCI_DTIP BIT(4) /* Data Transfer In Progress */
120	#define ATMCI_NOTBUSY BIT(5) /* Data Not Busy */
121	#define ATMCI_ENDRX BIT(6) /* End of RX Buffer */
122	#define ATMCI_ENDTX BIT(7) /* End of TX Buffer */
123	#define ATMCI_SDIOIRQA BIT(8) /* SDIO IRQ in slot A */
124	#define ATMCI_SDIOIRQB BIT(9) /* SDIO IRQ in slot B */
125	#define ATMCI_SDIOWAIT BIT(12) /* SDIO Read Wait Operation Status */
126	#define ATMCI_CSRCV BIT(13) /* CE-ATA Completion Signal Received */
127	#define ATMCI_RXBUFF BIT(14) /* RX Buffer Full */
128	#define ATMCI_TXBUFE BIT(15) /* TX Buffer Empty */
129	#define ATMCI_RINDE BIT(16) /* Response Index Error */
130	#define ATMCI_RDIRE BIT(17) /* Response Direction Error */
131	#define ATMCI_RCRCE BIT(18) /* Response CRC Error */
132	#define ATMCI_RENDE BIT(19) /* Response End Bit Error */
133	#define ATMCI_RTOE BIT(20) /* Response Time-Out Error */
134	#define ATMCI_DCRCE BIT(21) /* Data CRC Error */
135	#define ATMCI_DTOE BIT(22) /* Data Time-Out Error */
136	#define ATMCI_CSTOE BIT(23) /* Completion Signal Time-out Error */
137	#define ATMCI_BLKOVRE BIT(24) /* DMA Block Overrun Error */
138	#define ATMCI_DMADONE BIT(25) /* DMA Transfer Done */
139	#define ATMCI_FIFOEMPTY BIT(26) /* FIFO Empty Flag */
140	#define ATMCI_XFRDONE BIT(27) /* Transfer Done Flag */
141	#define ATMCI_ACKRCV BIT(28) /* Boot Operation Acknowledge Received */
142	#define ATMCI_ACKRCVE BIT(29) /* Boot Operation Acknowledge Error */
143	#define ATMCI_OVRE BIT(30) /* RX Overrun Error */
144	#define ATMCI_UNRE BIT(31) /* TX Underrun Error */
145	#define ATMCI_DMA 0x0050 /* DMA Configuration[2] */
146	#define ATMCI_DMA_OFFSET(x) ((x) << 0) /* DMA Write Buffer Offset */
147	#define ATMCI_DMA_CHKSIZE(x) ((x) << 4) /* DMA Channel Read and Write Chunk Size */
148	#define ATMCI_DMAEN BIT(8) /* DMA Hardware Handshaking Enable */
149	#define ATMCI_CFG 0x0054 /* Configuration[2] */
150	#define ATMCI_CFG_FIFOMODE_1DATA BIT(0) /* MCI Internal FIFO control mode */
151	#define ATMCI_CFG_FERRCTRL_COR BIT(4) /* Flow Error flag reset control mode */
152	#define ATMCI_CFG_HSMODE BIT(8) /* High Speed Mode */
153	#define ATMCI_CFG_LSYNC BIT(12) /* Synchronize on the last block */
154	#define ATMCI_WPMR 0x00e4 /* Write Protection Mode[2] */
155	#define ATMCI_WP_EN BIT(0) /* WP Enable */
156	#define ATMCI_WP_KEY (0x4d4349 << 8) /* WP Key */
157	#define ATMCI_WPSR 0x00e8 /* Write Protection Status[2] */
158	#define ATMCI_GET_WP_VS(x) ((x) & 0x0f)
159	#define ATMCI_GET_WP_VSRC(x) (((x) >> 8) & 0xffff)
160	#define ATMCI_VERSION 0x00FC /* Version */
161	#define ATMCI_FIFO_APERTURE 0x0200 /* FIFO Aperture[2] */
162
163	/* This is not including the FIFO Aperture on MCI2 */
164	#define ATMCI_REGS_SIZE 0x100
165
166	/* Register access macros */
167	#define atmci_readl(port, reg) \
168	__raw_readl((port)->regs + reg)
169	#define atmci_writel(port, reg, value) \
170	__raw_writel((value), (port)->regs + reg)
171
172	#define ATMCI_CMD_TIMEOUT_MS 2000
173	#define AUTOSUSPEND_DELAY 50
174
175	#define ATMCI_DATA_ERROR_FLAGS (ATMCI_DCRCE | ATMCI_DTOE | ATMCI_OVRE | ATMCI_UNRE)
176	#define ATMCI_DMA_THRESHOLD 16
177
178	enum {
179	EVENT_CMD_RDY = 0,
180	EVENT_XFER_COMPLETE,
181	EVENT_NOTBUSY,
182	EVENT_DATA_ERROR,
183	};
184
185	enum atmel_mci_state {
186	STATE_IDLE = 0,
187	STATE_SENDING_CMD,
188	STATE_DATA_XFER,
189	STATE_WAITING_NOTBUSY,
190	STATE_SENDING_STOP,
191	STATE_END_REQUEST,
192	};
193
194	enum atmci_xfer_dir {
195	XFER_RECEIVE = 0,
196	XFER_TRANSMIT,
197	};
198
199	enum atmci_pdc_buf {
200	PDC_FIRST_BUF = 0,
201	PDC_SECOND_BUF,
202	};
203
204	/**
205	* struct mci_slot_pdata - board-specific per-slot configuration
206	* @bus_width: Number of data lines wired up the slot
207	* @detect_pin: GPIO pin wired to the card detect switch
208	* @wp_pin: GPIO pin wired to the write protect sensor
209	* @non_removable: The slot is not removable, only detect once
210	*
211	* If a given slot is not present on the board, @bus_width should be
212	* set to 0. The other fields are ignored in this case.
213	*
214	* Any pins that aren't available should be set to a negative value.
215	*
216	* Note that support for multiple slots is experimental -- some cards
217	* might get upset if we don't get the clock management exactly right.
218	* But in most cases, it should work just fine.
219	*/
220	struct mci_slot_pdata {
221	unsigned int bus_width;
222	struct gpio_desc *detect_pin;
223	struct gpio_desc *wp_pin;
224	bool non_removable;
225	};
226
227	/**
228	* struct mci_platform_data - board-specific MMC/SDcard configuration
229	* @dma_slave: DMA slave interface to use in data transfers.
230	* @dma_filter: Filtering function to filter the DMA channel
231	* @slot: Per-slot configuration data.
232	*/
233	struct mci_platform_data {
234	void *dma_slave;
235	dma_filter_fn dma_filter;
236	struct mci_slot_pdata slot[ATMCI_MAX_NR_SLOTS];
237	};
238
239	struct atmel_mci_caps {
240	bool has_dma_conf_reg;
241	bool has_pdc;
242	bool has_cfg_reg;
243	bool has_cstor_reg;
244	bool has_highspeed;
245	bool has_rwproof;
246	bool has_odd_clk_div;
247	bool has_bad_data_ordering;
248	bool need_reset_after_xfer;
249	bool need_blksz_mul_4;
250	bool need_notbusy_for_read_ops;
251	};
252
253	struct atmel_mci_dma {
254	struct dma_chan *chan;
255	struct dma_async_tx_descriptor *data_desc;
256	};
257
258	/**
259	* struct atmel_mci - MMC controller state shared between all slots
260	* @lock: Spinlock protecting the queue and associated data.
261	* @regs: Pointer to MMIO registers.
262	* @sg: Scatterlist entry currently being processed by PIO or PDC code.
263	* @sg_len: Size of the scatterlist
264	* @pio_offset: Offset into the current scatterlist entry.
265	* @buffer: Buffer used if we don't have the r/w proof capability. We
266	* don't have the time to switch pdc buffers so we have to use only
267	* one buffer for the full transaction.
268	* @buf_size: size of the buffer.
269	* @buf_phys_addr: buffer address needed for pdc.
270	* @cur_slot: The slot which is currently using the controller.
271	* @mrq: The request currently being processed on @cur_slot,
272	* or NULL if the controller is idle.
273	* @cmd: The command currently being sent to the card, or NULL.
274	* @data: The data currently being transferred, or NULL if no data
275	* transfer is in progress.
276	* @data_size: just data->blocks * data->blksz.
277	* @dma: DMA client state.
278	* @data_chan: DMA channel being used for the current data transfer.
279	* @dma_conf: Configuration for the DMA slave
280	* @cmd_status: Snapshot of SR taken upon completion of the current
281	* command. Only valid when EVENT_CMD_COMPLETE is pending.
282	* @data_status: Snapshot of SR taken upon completion of the current
283	* data transfer. Only valid when EVENT_DATA_COMPLETE or
284	* EVENT_DATA_ERROR is pending.
285	* @stop_cmdr: Value to be loaded into CMDR when the stop command is
286	* to be sent.
287	* @tasklet: Tasklet running the request state machine.
288	* @pending_events: Bitmask of events flagged by the interrupt handler
289	* to be processed by the tasklet.
290	* @completed_events: Bitmask of events which the state machine has
291	* processed.
292	* @state: Tasklet state.
293	* @queue: List of slots waiting for access to the controller.
294	* @need_clock_update: Update the clock rate before the next request.
295	* @need_reset: Reset controller before next request.
296	* @timer: Timer to balance the data timeout error flag which cannot rise.
297	* @mode_reg: Value of the MR register.
298	* @cfg_reg: Value of the CFG register.
299	* @bus_hz: The rate of @mck in Hz. This forms the basis for MMC bus
300	* rate and timeout calculations.
301	* @mapbase: Physical address of the MMIO registers.
302	* @mck: The peripheral bus clock hooked up to the MMC controller.
303	* @pdev: Platform device associated with the MMC controller.
304	* @slot: Slots sharing this MMC controller.
305	* @caps: MCI capabilities depending on MCI version.
306	* @prepare_data: function to setup MCI before data transfer which
307	* depends on MCI capabilities.
308	* @submit_data: function to start data transfer which depends on MCI
309	* capabilities.
310	* @stop_transfer: function to stop data transfer which depends on MCI
311	* capabilities.
312	*
313	* Locking
314	* =======
315	*
316	* @lock is a softirq-safe spinlock protecting @queue as well as
317	* @cur_slot, @mrq and @state. These must always be updated
318	* at the same time while holding @lock.
319	*
320	* @lock also protects mode_reg and need_clock_update since these are
321	* used to synchronize mode register updates with the queue
322	* processing.
323	*
324	* The @mrq field of struct atmel_mci_slot is also protected by @lock,
325	* and must always be written at the same time as the slot is added to
326	* @queue.
327	*
328	* @pending_events and @completed_events are accessed using atomic bit
329	* operations, so they don't need any locking.
330	*
331	* None of the fields touched by the interrupt handler need any
332	* locking. However, ordering is important: Before EVENT_DATA_ERROR or
333	* EVENT_DATA_COMPLETE is set in @pending_events, all data-related
334	* interrupts must be disabled and @data_status updated with a
335	* snapshot of SR. Similarly, before EVENT_CMD_COMPLETE is set, the
336	* CMDRDY interrupt must be disabled and @cmd_status updated with a
337	* snapshot of SR, and before EVENT_XFER_COMPLETE can be set, the
338	* bytes_xfered field of @data must be written. This is ensured by
339	* using barriers.
340	*/
341	struct atmel_mci {
342	spinlock_t lock;
343	void __iomem *regs;
344
345	struct scatterlist *sg;
346	unsigned int sg_len;
347	unsigned int pio_offset;
348	unsigned int *buffer;
349	unsigned int buf_size;
350	dma_addr_t buf_phys_addr;
351
352	struct atmel_mci_slot *cur_slot;
353	struct mmc_request *mrq;
354	struct mmc_command *cmd;
355	struct mmc_data *data;
356	unsigned int data_size;
357
358	struct atmel_mci_dma dma;
359	struct dma_chan *data_chan;
360	struct dma_slave_config dma_conf;
361
362	u32 cmd_status;
363	u32 data_status;
364	u32 stop_cmdr;
365
366	struct tasklet_struct tasklet;
367	unsigned long pending_events;
368	unsigned long completed_events;
369	enum atmel_mci_state state;
370	struct list_head queue;
371
372	bool need_clock_update;
373	bool need_reset;
374	struct timer_list timer;
375	u32 mode_reg;
376	u32 cfg_reg;
377	unsigned long bus_hz;
378	unsigned long mapbase;
379	struct clk *mck;
380	struct platform_device *pdev;
381
382	struct atmel_mci_slot *slot[ATMCI_MAX_NR_SLOTS];
383
384	struct atmel_mci_caps caps;
385
386	u32 (*prepare_data)(struct atmel_mci *host, struct mmc_data *data);
387	void (*submit_data)(struct atmel_mci *host, struct mmc_data *data);
388	void (*stop_transfer)(struct atmel_mci *host);
389	};
390
391	/**
392	* struct atmel_mci_slot - MMC slot state
393	* @mmc: The mmc_host representing this slot.
394	* @host: The MMC controller this slot is using.
395	* @sdc_reg: Value of SDCR to be written before using this slot.
396	* @sdio_irq: SDIO irq mask for this slot.
397	* @mrq: mmc_request currently being processed or waiting to be
398	* processed, or NULL when the slot is idle.
399	* @queue_node: List node for placing this node in the @queue list of
400	* &struct atmel_mci.
401	* @clock: Clock rate configured by set_ios(). Protected by host->lock.
402	* @flags: Random state bits associated with the slot.
403	* @detect_pin: GPIO pin used for card detection, or negative if not
404	* available.
405	* @wp_pin: GPIO pin used for card write protect sending, or negative
406	* if not available.
407	* @detect_timer: Timer used for debouncing @detect_pin interrupts.
408	*/
409	struct atmel_mci_slot {
410	struct mmc_host *mmc;
411	struct atmel_mci *host;
412
413	u32 sdc_reg;
414	u32 sdio_irq;
415
416	struct mmc_request *mrq;
417	struct list_head queue_node;
418
419	unsigned int clock;
420	unsigned long flags;
421	#define ATMCI_CARD_PRESENT 0
422	#define ATMCI_CARD_NEED_INIT 1
423	#define ATMCI_SHUTDOWN 2
424
425	struct gpio_desc *detect_pin;
426	struct gpio_desc *wp_pin;
427
428	struct timer_list detect_timer;
429	};
430
431	#define atmci_test_and_clear_pending(host, event) \
432	test_and_clear_bit(event, &host->pending_events)
433	#define atmci_set_completed(host, event) \
434	set_bit(event, &host->completed_events)
435	#define atmci_set_pending(host, event) \
436	set_bit(event, &host->pending_events)
437
438	/*
439	* The debugfs stuff below is mostly optimized away when
440	* CONFIG_DEBUG_FS is not set.
441	*/
442	static int atmci_req_show(struct seq_file *s, void *v)
443	{
444	struct atmel_mci_slot *slot = s->private;
445	struct mmc_request *mrq;
446	struct mmc_command *cmd;
447	struct mmc_command *stop;
448	struct mmc_data *data;
449
450	/* Make sure we get a consistent snapshot */
451	spin_lock_bh(lock: &slot->host->lock);
452	mrq = slot->mrq;
453
454	if (mrq) {
455	cmd = mrq->cmd;
456	data = mrq->data;
457	stop = mrq->stop;
458
459	if (cmd)
460	seq_printf(m: s,
461	fmt: "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
462	cmd->opcode, cmd->arg, cmd->flags,
463	cmd->resp[0], cmd->resp[1], cmd->resp[2],
464	cmd->resp[3], cmd->error);
465	if (data)
466	seq_printf(m: s, fmt: "DATA %u / %u * %u flg %x err %d\n",
467	data->bytes_xfered, data->blocks,
468	data->blksz, data->flags, data->error);
469	if (stop)
470	seq_printf(m: s,
471	fmt: "CMD%u(0x%x) flg %x rsp %x %x %x %x err %d\n",
472	stop->opcode, stop->arg, stop->flags,
473	stop->resp[0], stop->resp[1], stop->resp[2],
474	stop->resp[3], stop->error);
475	}
476
477	spin_unlock_bh(lock: &slot->host->lock);
478
479	return 0;
480	}
481
482	DEFINE_SHOW_ATTRIBUTE(atmci_req);
483
484	static void atmci_show_status_reg(struct seq_file *s,
485	const char *regname, u32 value)
486	{
487	static const char *sr_bit[] = {
488	[0] = "CMDRDY",
489	[1] = "RXRDY",
490	[2] = "TXRDY",
491	[3] = "BLKE",
492	[4] = "DTIP",
493	[5] = "NOTBUSY",
494	[6] = "ENDRX",
495	[7] = "ENDTX",
496	[8] = "SDIOIRQA",
497	[9] = "SDIOIRQB",
498	[12] = "SDIOWAIT",
499	[14] = "RXBUFF",
500	[15] = "TXBUFE",
501	[16] = "RINDE",
502	[17] = "RDIRE",
503	[18] = "RCRCE",
504	[19] = "RENDE",
505	[20] = "RTOE",
506	[21] = "DCRCE",
507	[22] = "DTOE",
508	[23] = "CSTOE",
509	[24] = "BLKOVRE",
510	[25] = "DMADONE",
511	[26] = "FIFOEMPTY",
512	[27] = "XFRDONE",
513	[30] = "OVRE",
514	[31] = "UNRE",
515	};
516	unsigned int i;
517
518	seq_printf(m: s, fmt: "%s:\t0x%08x", regname, value);
519	for (i = 0; i < ARRAY_SIZE(sr_bit); i++) {
520	if (value & (1 << i)) {
521	if (sr_bit[i])
522	seq_printf(m: s, fmt: " %s", sr_bit[i]);
523	else
524	seq_puts(m: s, s: " UNKNOWN");
525	}
526	}
527	seq_putc(m: s, c: '\n');
528	}
529
530	static int atmci_regs_show(struct seq_file *s, void *v)
531	{
532	struct atmel_mci *host = s->private;
533	u32 *buf;
534	int ret = 0;
535
536
537	buf = kmalloc(ATMCI_REGS_SIZE, GFP_KERNEL);
538	if (!buf)
539	return -ENOMEM;
540
541	pm_runtime_get_sync(dev: &host->pdev->dev);
542
543	/*
544	* Grab a more or less consistent snapshot. Note that we're
545	* not disabling interrupts, so IMR and SR may not be
546	* consistent.
547	*/
548	spin_lock_bh(lock: &host->lock);
549	memcpy_fromio(buf, host->regs, ATMCI_REGS_SIZE);
550	spin_unlock_bh(lock: &host->lock);
551
552	pm_runtime_mark_last_busy(dev: &host->pdev->dev);
553	pm_runtime_put_autosuspend(dev: &host->pdev->dev);
554
555	seq_printf(m: s, fmt: "MR:\t0x%08x%s%s ",
556	buf[ATMCI_MR / 4],
557	buf[ATMCI_MR / 4] & ATMCI_MR_RDPROOF ? " RDPROOF" : "",
558	buf[ATMCI_MR / 4] & ATMCI_MR_WRPROOF ? " WRPROOF" : "");
559	if (host->caps.has_odd_clk_div)
560	seq_printf(m: s, fmt: "{CLKDIV,CLKODD}=%u\n",
561	((buf[ATMCI_MR / 4] & 0xff) << 1)
562	| ((buf[ATMCI_MR / 4] >> 16) & 1));
563	else
564	seq_printf(m: s, fmt: "CLKDIV=%u\n",
565	(buf[ATMCI_MR / 4] & 0xff));
566	seq_printf(m: s, fmt: "DTOR:\t0x%08x\n", buf[ATMCI_DTOR / 4]);
567	seq_printf(m: s, fmt: "SDCR:\t0x%08x\n", buf[ATMCI_SDCR / 4]);
568	seq_printf(m: s, fmt: "ARGR:\t0x%08x\n", buf[ATMCI_ARGR / 4]);
569	seq_printf(m: s, fmt: "BLKR:\t0x%08x BCNT=%u BLKLEN=%u\n",
570	buf[ATMCI_BLKR / 4],
571	buf[ATMCI_BLKR / 4] & 0xffff,
572	(buf[ATMCI_BLKR / 4] >> 16) & 0xffff);
573	if (host->caps.has_cstor_reg)
574	seq_printf(m: s, fmt: "CSTOR:\t0x%08x\n", buf[ATMCI_CSTOR / 4]);
575
576	/* Don't read RSPR and RDR; it will consume the data there */
577
578	atmci_show_status_reg(s, regname: "SR", value: buf[ATMCI_SR / 4]);
579	atmci_show_status_reg(s, regname: "IMR", value: buf[ATMCI_IMR / 4]);
580
581	if (host->caps.has_dma_conf_reg) {
582	u32 val;
583
584	val = buf[ATMCI_DMA / 4];
585	seq_printf(m: s, fmt: "DMA:\t0x%08x OFFSET=%u CHKSIZE=%u%s\n",
586	val, val & 3,
587	((val >> 4) & 3) ?
588	1 << (((val >> 4) & 3) + 1) : 1,
589	val & ATMCI_DMAEN ? " DMAEN" : "");
590	}
591	if (host->caps.has_cfg_reg) {
592	u32 val;
593
594	val = buf[ATMCI_CFG / 4];
595	seq_printf(m: s, fmt: "CFG:\t0x%08x%s%s%s%s\n",
596	val,
597	val & ATMCI_CFG_FIFOMODE_1DATA ? " FIFOMODE_ONE_DATA" : "",
598	val & ATMCI_CFG_FERRCTRL_COR ? " FERRCTRL_CLEAR_ON_READ" : "",
599	val & ATMCI_CFG_HSMODE ? " HSMODE" : "",
600	val & ATMCI_CFG_LSYNC ? " LSYNC" : "");
601	}
602
603	kfree(objp: buf);
604
605	return ret;
606	}
607
608	DEFINE_SHOW_ATTRIBUTE(atmci_regs);
609
610	static void atmci_init_debugfs(struct atmel_mci_slot *slot)
611	{
612	struct mmc_host *mmc = slot->mmc;
613	struct atmel_mci *host = slot->host;
614	struct dentry *root;
615
616	root = mmc->debugfs_root;
617	if (!root)
618	return;
619
620	debugfs_create_file(name: "regs", S_IRUSR, parent: root, data: host, fops: &atmci_regs_fops);
621	debugfs_create_file(name: "req", S_IRUSR, parent: root, data: slot, fops: &atmci_req_fops);
622	debugfs_create_u32(name: "state", S_IRUSR, parent: root, value: &host->state);
623	debugfs_create_xul(name: "pending_events", S_IRUSR, parent: root,
624	value: &host->pending_events);
625	debugfs_create_xul(name: "completed_events", S_IRUSR, parent: root,
626	value: &host->completed_events);
627	}
628
629	#if defined(CONFIG_OF)
630	static const struct of_device_id atmci_dt_ids[] = {
631	{ .compatible = "atmel,hsmci" },
632	{ /* sentinel */ }
633	};
634
635	MODULE_DEVICE_TABLE(of, atmci_dt_ids);
636
637	static struct mci_platform_data*
638	atmci_of_init(struct platform_device *pdev)
639	{
640	struct device_node *np = pdev->dev.of_node;
641	struct device_node *cnp;
642	struct mci_platform_data *pdata;
643	u32 slot_id;
644	int err;
645
646	if (!np) {
647	dev_err(&pdev->dev, "device node not found\n");
648	return ERR_PTR(error: -EINVAL);
649	}
650
651	pdata = devm_kzalloc(dev: &pdev->dev, size: sizeof(*pdata), GFP_KERNEL);
652	if (!pdata)
653	return ERR_PTR(error: -ENOMEM);
654
655	for_each_child_of_node(np, cnp) {
656	if (of_property_read_u32(np: cnp, propname: "reg", out_value: &slot_id)) {
657	dev_warn(&pdev->dev, "reg property is missing for %pOF\n",
658	cnp);
659	continue;
660	}
661
662	if (slot_id >= ATMCI_MAX_NR_SLOTS) {
663	dev_warn(&pdev->dev, "can't have more than %d slots\n",
664	ATMCI_MAX_NR_SLOTS);
665	of_node_put(node: cnp);
666	break;
667	}
668
669	if (of_property_read_u32(np: cnp, propname: "bus-width",
670	out_value: &pdata->slot[slot_id].bus_width))
671	pdata->slot[slot_id].bus_width = 1;
672
673	pdata->slot[slot_id].detect_pin =
674	devm_fwnode_gpiod_get(dev: &pdev->dev, of_fwnode_handle(cnp),
675	con_id: "cd", flags: GPIOD_IN, label: "cd-gpios");
676	err = PTR_ERR_OR_ZERO(ptr: pdata->slot[slot_id].detect_pin);
677	if (err) {
678	if (err != -ENOENT) {
679	of_node_put(node: cnp);
680	return ERR_PTR(error: err);
681	}
682	pdata->slot[slot_id].detect_pin = NULL;
683	}
684
685	pdata->slot[slot_id].non_removable =
686	of_property_read_bool(np: cnp, propname: "non-removable");
687
688	pdata->slot[slot_id].wp_pin =
689	devm_fwnode_gpiod_get(dev: &pdev->dev, of_fwnode_handle(cnp),
690	con_id: "wp", flags: GPIOD_IN, label: "wp-gpios");
691	err = PTR_ERR_OR_ZERO(ptr: pdata->slot[slot_id].wp_pin);
692	if (err) {
693	if (err != -ENOENT) {
694	of_node_put(node: cnp);
695	return ERR_PTR(error: err);
696	}
697	pdata->slot[slot_id].wp_pin = NULL;
698	}
699	}
700
701	return pdata;
702	}
703	#else /* CONFIG_OF */
704	static inline struct mci_platform_data*
705	atmci_of_init(struct platform_device *dev)
706	{
707	return ERR_PTR(-EINVAL);
708	}
709	#endif
710
711	static inline unsigned int atmci_get_version(struct atmel_mci *host)
712	{
713	return atmci_readl(host, ATMCI_VERSION) & 0x00000fff;
714	}
715
716	/*
717	* Fix sconfig's burst size according to atmel MCI. We need to convert them as:
718	* 1 -> 0, 4 -> 1, 8 -> 2, 16 -> 3.
719	* With version 0x600, we need to convert them as: 1 -> 0, 2 -> 1, 4 -> 2,
720	* 8 -> 3, 16 -> 4.
721	*
722	* This can be done by finding most significant bit set.
723	*/
724	static inline unsigned int atmci_convert_chksize(struct atmel_mci *host,
725	unsigned int maxburst)
726	{
727	unsigned int version = atmci_get_version(host);
728	unsigned int offset = 2;
729
730	if (version >= 0x600)
731	offset = 1;
732
733	if (maxburst > 1)
734	return fls(x: maxburst) - offset;
735	else
736	return 0;
737	}
738
739	static void atmci_timeout_timer(struct timer_list *t)
740	{
741	struct atmel_mci *host;
742
743	host = from_timer(host, t, timer);
744
745	dev_dbg(&host->pdev->dev, "software timeout\n");
746
747	if (host->mrq->cmd->data) {
748	host->mrq->cmd->data->error = -ETIMEDOUT;
749	host->data = NULL;
750	/*
751	* With some SDIO modules, sometimes DMA transfer hangs. If
752	* stop_transfer() is not called then the DMA request is not
753	* removed, following ones are queued and never computed.
754	*/
755	if (host->state == STATE_DATA_XFER)
756	host->stop_transfer(host);
757	} else {
758	host->mrq->cmd->error = -ETIMEDOUT;
759	host->cmd = NULL;
760	}
761	host->need_reset = 1;
762	host->state = STATE_END_REQUEST;
763	smp_wmb();
764	tasklet_schedule(t: &host->tasklet);
765	}
766
767	static inline unsigned int atmci_ns_to_clocks(struct atmel_mci *host,
768	unsigned int ns)
769	{
770	/*
771	* It is easier here to use us instead of ns for the timeout,
772	* it prevents from overflows during calculation.
773	*/
774	unsigned int us = DIV_ROUND_UP(ns, 1000);
775
776	/* Maximum clock frequency is host->bus_hz/2 */
777	return us * (DIV_ROUND_UP(host->bus_hz, 2000000));
778	}
779
780	static void atmci_set_timeout(struct atmel_mci *host,
781	struct atmel_mci_slot *slot, struct mmc_data *data)
782	{
783	static unsigned dtomul_to_shift[] = {
784	0, 4, 7, 8, 10, 12, 16, 20
785	};
786	unsigned timeout;
787	unsigned dtocyc;
788	unsigned dtomul;
789
790	timeout = atmci_ns_to_clocks(host, ns: data->timeout_ns)
791	+ data->timeout_clks;
792
793	for (dtomul = 0; dtomul < 8; dtomul++) {
794	unsigned shift = dtomul_to_shift[dtomul];
795	dtocyc = (timeout + (1 << shift) - 1) >> shift;
796	if (dtocyc < 15)
797	break;
798	}
799
800	if (dtomul >= 8) {
801	dtomul = 7;
802	dtocyc = 15;
803	}
804
805	dev_vdbg(&slot->mmc->class_dev, "setting timeout to %u cycles\n",
806	dtocyc << dtomul_to_shift[dtomul]);
807	atmci_writel(host, ATMCI_DTOR, (ATMCI_DTOMUL(dtomul) | ATMCI_DTOCYC(dtocyc)));
808	}
809
810	/*
811	* Return mask with command flags to be enabled for this command.
812	*/
813	static u32 atmci_prepare_command(struct mmc_host *mmc,
814	struct mmc_command *cmd)
815	{
816	struct mmc_data *data;
817	u32 cmdr;
818
819	cmd->error = -EINPROGRESS;
820
821	cmdr = ATMCI_CMDR_CMDNB(cmd->opcode);
822
823	if (cmd->flags & MMC_RSP_PRESENT) {
824	if (cmd->flags & MMC_RSP_136)
825	cmdr |= ATMCI_CMDR_RSPTYP_136BIT;
826	else
827	cmdr |= ATMCI_CMDR_RSPTYP_48BIT;
828	}
829
830	/*
831	* This should really be MAXLAT_5 for CMD2 and ACMD41, but
832	* it's too difficult to determine whether this is an ACMD or
833	* not. Better make it 64.
834	*/
835	cmdr |= ATMCI_CMDR_MAXLAT_64CYC;
836
837	if (mmc->ios.bus_mode == MMC_BUSMODE_OPENDRAIN)
838	cmdr |= ATMCI_CMDR_OPDCMD;
839
840	data = cmd->data;
841	if (data) {
842	cmdr |= ATMCI_CMDR_START_XFER;
843
844	if (cmd->opcode == SD_IO_RW_EXTENDED) {
845	cmdr |= ATMCI_CMDR_SDIO_BLOCK;
846	} else {
847	if (data->blocks > 1)
848	cmdr |= ATMCI_CMDR_MULTI_BLOCK;
849	else
850	cmdr |= ATMCI_CMDR_BLOCK;
851	}
852
853	if (data->flags & MMC_DATA_READ)
854	cmdr |= ATMCI_CMDR_TRDIR_READ;
855	}
856
857	return cmdr;
858	}
859
860	static void atmci_send_command(struct atmel_mci *host,
861	struct mmc_command *cmd, u32 cmd_flags)
862	{
863	unsigned int timeout_ms = cmd->busy_timeout ? cmd->busy_timeout :
864	ATMCI_CMD_TIMEOUT_MS;
865
866	WARN_ON(host->cmd);
867	host->cmd = cmd;
868
869	dev_vdbg(&host->pdev->dev,
870	"start command: ARGR=0x%08x CMDR=0x%08x\n",
871	cmd->arg, cmd_flags);
872
873	atmci_writel(host, ATMCI_ARGR, cmd->arg);
874	atmci_writel(host, ATMCI_CMDR, cmd_flags);
875
876	mod_timer(timer: &host->timer, expires: jiffies + msecs_to_jiffies(m: timeout_ms));
877	}
878
879	static void atmci_send_stop_cmd(struct atmel_mci *host, struct mmc_data *data)
880	{
881	dev_dbg(&host->pdev->dev, "send stop command\n");
882	atmci_send_command(host, cmd: data->stop, cmd_flags: host->stop_cmdr);
883	atmci_writel(host, ATMCI_IER, ATMCI_CMDRDY);
884	}
885
886	/*
887	* Configure given PDC buffer taking care of alignement issues.
888	* Update host->data_size and host->sg.
889	*/
890	static void atmci_pdc_set_single_buf(struct atmel_mci *host,
891	enum atmci_xfer_dir dir, enum atmci_pdc_buf buf_nb)
892	{
893	u32 pointer_reg, counter_reg;
894	unsigned int buf_size;
895
896	if (dir == XFER_RECEIVE) {
897	pointer_reg = ATMEL_PDC_RPR;
898	counter_reg = ATMEL_PDC_RCR;
899	} else {
900	pointer_reg = ATMEL_PDC_TPR;
901	counter_reg = ATMEL_PDC_TCR;
902	}
903
904	if (buf_nb == PDC_SECOND_BUF) {
905	pointer_reg += ATMEL_PDC_SCND_BUF_OFF;
906	counter_reg += ATMEL_PDC_SCND_BUF_OFF;
907	}
908
909	if (!host->caps.has_rwproof) {
910	buf_size = host->buf_size;
911	atmci_writel(host, pointer_reg, host->buf_phys_addr);
912	} else {
913	buf_size = sg_dma_len(host->sg);
914	atmci_writel(host, pointer_reg, sg_dma_address(host->sg));
915	}
916
917	if (host->data_size <= buf_size) {
918	if (host->data_size & 0x3) {
919	/* If size is different from modulo 4, transfer bytes */
920	atmci_writel(host, counter_reg, host->data_size);
921	atmci_writel(host, ATMCI_MR, host->mode_reg | ATMCI_MR_PDCFBYTE);
922	} else {
923	/* Else transfer 32-bits words */
924	atmci_writel(host, counter_reg, host->data_size / 4);
925	}
926	host->data_size = 0;
927	} else {
928	/* We assume the size of a page is 32-bits aligned */
929	atmci_writel(host, counter_reg, sg_dma_len(host->sg) / 4);
930	host->data_size -= sg_dma_len(host->sg);
931	if (host->data_size)
932	host->sg = sg_next(host->sg);
933	}
934	}
935
936	/*
937	* Configure PDC buffer according to the data size ie configuring one or two
938	* buffers. Don't use this function if you want to configure only the second
939	* buffer. In this case, use atmci_pdc_set_single_buf.
940	*/
941	static void atmci_pdc_set_both_buf(struct atmel_mci *host, int dir)
942	{
943	atmci_pdc_set_single_buf(host, dir, buf_nb: PDC_FIRST_BUF);
944	if (host->data_size)
945	atmci_pdc_set_single_buf(host, dir, buf_nb: PDC_SECOND_BUF);
946	}
947
948	/*
949	* Unmap sg lists, called when transfer is finished.
950	*/
951	static void atmci_pdc_cleanup(struct atmel_mci *host)
952	{
953	struct mmc_data *data = host->data;
954
955	if (data)
956	dma_unmap_sg(&host->pdev->dev,
957	data->sg, data->sg_len,
958	mmc_get_dma_dir(data));
959	}
960
961	/*
962	* Disable PDC transfers. Update pending flags to EVENT_XFER_COMPLETE after
963	* having received ATMCI_TXBUFE or ATMCI_RXBUFF interrupt. Enable ATMCI_NOTBUSY
964	* interrupt needed for both transfer directions.
965	*/
966	static void atmci_pdc_complete(struct atmel_mci *host)
967	{
968	int transfer_size = host->data->blocks * host->data->blksz;
969	int i;
970
971	atmci_writel(host, ATMEL_PDC_PTCR, ATMEL_PDC_RXTDIS | ATMEL_PDC_TXTDIS);
972
973	if ((!host->caps.has_rwproof)
974	&& (host->data->flags & MMC_DATA_READ)) {
975	if (host->caps.has_bad_data_ordering)
976	for (i = 0; i < transfer_size; i++)
977	host->buffer[i] = swab32(host->buffer[i]);
978	sg_copy_from_buffer(sgl: host->data->sg, nents: host->data->sg_len,
979	buf: host->buffer, buflen: transfer_size);
980	}
981
982	atmci_pdc_cleanup(host);
983
984	dev_dbg(&host->pdev->dev, "(%s) set pending xfer complete\n", __func__);
985	atmci_set_pending(host, EVENT_XFER_COMPLETE);
986	tasklet_schedule(t: &host->tasklet);
987	}
988
989	static void atmci_dma_cleanup(struct atmel_mci *host)
990	{
991	struct mmc_data *data = host->data;
992
993	if (data)
994	dma_unmap_sg(host->dma.chan->device->dev,
995	data->sg, data->sg_len,
996	mmc_get_dma_dir(data));
997	}
998
999	/*
1000	* This function is called by the DMA driver from tasklet context.
1001	*/
1002	static void atmci_dma_complete(void *arg)
1003	{
1004	struct atmel_mci *host = arg;
1005	struct mmc_data *data = host->data;
1006
1007	dev_vdbg(&host->pdev->dev, "DMA complete\n");
1008
1009	if (host->caps.has_dma_conf_reg)
1010	/* Disable DMA hardware handshaking on MCI */
1011	atmci_writel(host, ATMCI_DMA, atmci_readl(host, ATMCI_DMA) & ~ATMCI_DMAEN);
1012
1013	atmci_dma_cleanup(host);
1014
1015	/*
1016	* If the card was removed, data will be NULL. No point trying
1017	* to send the stop command or waiting for NBUSY in this case.
1018	*/
1019	if (data) {
1020	dev_dbg(&host->pdev->dev,
1021	"(%s) set pending xfer complete\n", __func__);
1022	atmci_set_pending(host, EVENT_XFER_COMPLETE);
1023	tasklet_schedule(t: &host->tasklet);
1024
1025	/*
1026	* Regardless of what the documentation says, we have
1027	* to wait for NOTBUSY even after block read
1028	* operations.
1029	*
1030	* When the DMA transfer is complete, the controller
1031	* may still be reading the CRC from the card, i.e.
1032	* the data transfer is still in progress and we
1033	* haven't seen all the potential error bits yet.
1034	*
1035	* The interrupt handler will schedule a different
1036	* tasklet to finish things up when the data transfer
1037	* is completely done.
1038	*
1039	* We may not complete the mmc request here anyway
1040	* because the mmc layer may call back and cause us to
1041	* violate the "don't submit new operations from the
1042	* completion callback" rule of the dma engine
1043	* framework.
1044	*/
1045	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1046	}
1047	}
1048
1049	/*
1050	* Returns a mask of interrupt flags to be enabled after the whole
1051	* request has been prepared.
1052	*/
1053	static u32 atmci_prepare_data(struct atmel_mci *host, struct mmc_data *data)
1054	{
1055	u32 iflags;
1056
1057	data->error = -EINPROGRESS;
1058
1059	host->sg = data->sg;
1060	host->sg_len = data->sg_len;
1061	host->data = data;
1062	host->data_chan = NULL;
1063
1064	iflags = ATMCI_DATA_ERROR_FLAGS;
1065
1066	/*
1067	* Errata: MMC data write operation with less than 12
1068	* bytes is impossible.
1069	*
1070	* Errata: MCI Transmit Data Register (TDR) FIFO
1071	* corruption when length is not multiple of 4.
1072	*/
1073	if (data->blocks * data->blksz < 12
1074	|| (data->blocks * data->blksz) & 3)
1075	host->need_reset = true;
1076
1077	host->pio_offset = 0;
1078	if (data->flags & MMC_DATA_READ)
1079	iflags |= ATMCI_RXRDY;
1080	else
1081	iflags |= ATMCI_TXRDY;
1082
1083	return iflags;
1084	}
1085
1086	/*
1087	* Set interrupt flags and set block length into the MCI mode register even
1088	* if this value is also accessible in the MCI block register. It seems to be
1089	* necessary before the High Speed MCI version. It also map sg and configure
1090	* PDC registers.
1091	*/
1092	static u32
1093	atmci_prepare_data_pdc(struct atmel_mci *host, struct mmc_data *data)
1094	{
1095	u32 iflags, tmp;
1096	int i;
1097
1098	data->error = -EINPROGRESS;
1099
1100	host->data = data;
1101	host->sg = data->sg;
1102	iflags = ATMCI_DATA_ERROR_FLAGS;
1103
1104	/* Enable pdc mode */
1105	atmci_writel(host, ATMCI_MR, host->mode_reg | ATMCI_MR_PDCMODE);
1106
1107	if (data->flags & MMC_DATA_READ)
1108	iflags |= ATMCI_ENDRX | ATMCI_RXBUFF;
1109	else
1110	iflags |= ATMCI_ENDTX | ATMCI_TXBUFE | ATMCI_BLKE;
1111
1112	/* Set BLKLEN */
1113	tmp = atmci_readl(host, ATMCI_MR);
1114	tmp &= 0x0000ffff;
1115	tmp |= ATMCI_BLKLEN(data->blksz);
1116	atmci_writel(host, ATMCI_MR, tmp);
1117
1118	/* Configure PDC */
1119	host->data_size = data->blocks * data->blksz;
1120	dma_map_sg(&host->pdev->dev, data->sg, data->sg_len,
1121	mmc_get_dma_dir(data));
1122
1123	if ((!host->caps.has_rwproof)
1124	&& (host->data->flags & MMC_DATA_WRITE)) {
1125	sg_copy_to_buffer(sgl: host->data->sg, nents: host->data->sg_len,
1126	buf: host->buffer, buflen: host->data_size);
1127	if (host->caps.has_bad_data_ordering)
1128	for (i = 0; i < host->data_size; i++)
1129	host->buffer[i] = swab32(host->buffer[i]);
1130	}
1131
1132	if (host->data_size)
1133	atmci_pdc_set_both_buf(host, dir: data->flags & MMC_DATA_READ ?
1134	XFER_RECEIVE : XFER_TRANSMIT);
1135	return iflags;
1136	}
1137
1138	static u32
1139	atmci_prepare_data_dma(struct atmel_mci *host, struct mmc_data *data)
1140	{
1141	struct dma_chan *chan;
1142	struct dma_async_tx_descriptor *desc;
1143	struct scatterlist *sg;
1144	unsigned int i;
1145	enum dma_transfer_direction slave_dirn;
1146	unsigned int sglen;
1147	u32 maxburst;
1148	u32 iflags;
1149
1150	data->error = -EINPROGRESS;
1151
1152	WARN_ON(host->data);
1153	host->sg = NULL;
1154	host->data = data;
1155
1156	iflags = ATMCI_DATA_ERROR_FLAGS;
1157
1158	/*
1159	* We don't do DMA on "complex" transfers, i.e. with
1160	* non-word-aligned buffers or lengths. Also, we don't bother
1161	* with all the DMA setup overhead for short transfers.
1162	*/
1163	if (data->blocks * data->blksz < ATMCI_DMA_THRESHOLD)
1164	return atmci_prepare_data(host, data);
1165	if (data->blksz & 3)
1166	return atmci_prepare_data(host, data);
1167
1168	for_each_sg(data->sg, sg, data->sg_len, i) {
1169	if (sg->offset & 3 || sg->length & 3)
1170	return atmci_prepare_data(host, data);
1171	}
1172
1173	/* If we don't have a channel, we can't do DMA */
1174	if (!host->dma.chan)
1175	return -ENODEV;
1176
1177	chan = host->dma.chan;
1178	host->data_chan = chan;
1179
1180	if (data->flags & MMC_DATA_READ) {
1181	host->dma_conf.direction = slave_dirn = DMA_DEV_TO_MEM;
1182	maxburst = atmci_convert_chksize(host,
1183	maxburst: host->dma_conf.src_maxburst);
1184	} else {
1185	host->dma_conf.direction = slave_dirn = DMA_MEM_TO_DEV;
1186	maxburst = atmci_convert_chksize(host,
1187	maxburst: host->dma_conf.dst_maxburst);
1188	}
1189
1190	if (host->caps.has_dma_conf_reg)
1191	atmci_writel(host, ATMCI_DMA, ATMCI_DMA_CHKSIZE(maxburst) |
1192	ATMCI_DMAEN);
1193
1194	sglen = dma_map_sg(chan->device->dev, data->sg,
1195	data->sg_len, mmc_get_dma_dir(data));
1196
1197	dmaengine_slave_config(chan, config: &host->dma_conf);
1198	desc = dmaengine_prep_slave_sg(chan,
1199	sgl: data->sg, sg_len: sglen, dir: slave_dirn,
1200	flags: DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
1201	if (!desc)
1202	goto unmap_exit;
1203
1204	host->dma.data_desc = desc;
1205	desc->callback = atmci_dma_complete;
1206	desc->callback_param = host;
1207
1208	return iflags;
1209	unmap_exit:
1210	dma_unmap_sg(chan->device->dev, data->sg, data->sg_len,
1211	mmc_get_dma_dir(data));
1212	return -ENOMEM;
1213	}
1214
1215	static void
1216	atmci_submit_data(struct atmel_mci *host, struct mmc_data *data)
1217	{
1218	return;
1219	}
1220
1221	/*
1222	* Start PDC according to transfer direction.
1223	*/
1224	static void
1225	atmci_submit_data_pdc(struct atmel_mci *host, struct mmc_data *data)
1226	{
1227	if (data->flags & MMC_DATA_READ)
1228	atmci_writel(host, ATMEL_PDC_PTCR, ATMEL_PDC_RXTEN);
1229	else
1230	atmci_writel(host, ATMEL_PDC_PTCR, ATMEL_PDC_TXTEN);
1231	}
1232
1233	static void
1234	atmci_submit_data_dma(struct atmel_mci *host, struct mmc_data *data)
1235	{
1236	struct dma_chan *chan = host->data_chan;
1237	struct dma_async_tx_descriptor *desc = host->dma.data_desc;
1238
1239	if (chan) {
1240	dmaengine_submit(desc);
1241	dma_async_issue_pending(chan);
1242	}
1243	}
1244
1245	static void atmci_stop_transfer(struct atmel_mci *host)
1246	{
1247	dev_dbg(&host->pdev->dev,
1248	"(%s) set pending xfer complete\n", __func__);
1249	atmci_set_pending(host, EVENT_XFER_COMPLETE);
1250	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1251	}
1252
1253	/*
1254	* Stop data transfer because error(s) occurred.
1255	*/
1256	static void atmci_stop_transfer_pdc(struct atmel_mci *host)
1257	{
1258	atmci_writel(host, ATMEL_PDC_PTCR, ATMEL_PDC_RXTDIS | ATMEL_PDC_TXTDIS);
1259	}
1260
1261	static void atmci_stop_transfer_dma(struct atmel_mci *host)
1262	{
1263	struct dma_chan *chan = host->data_chan;
1264
1265	if (chan) {
1266	dmaengine_terminate_all(chan);
1267	atmci_dma_cleanup(host);
1268	} else {
1269	/* Data transfer was stopped by the interrupt handler */
1270	dev_dbg(&host->pdev->dev,
1271	"(%s) set pending xfer complete\n", __func__);
1272	atmci_set_pending(host, EVENT_XFER_COMPLETE);
1273	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1274	}
1275	}
1276
1277	/*
1278	* Start a request: prepare data if needed, prepare the command and activate
1279	* interrupts.
1280	*/
1281	static void atmci_start_request(struct atmel_mci *host,
1282	struct atmel_mci_slot *slot)
1283	{
1284	struct mmc_request *mrq;
1285	struct mmc_command *cmd;
1286	struct mmc_data *data;
1287	u32 iflags;
1288	u32 cmdflags;
1289
1290	mrq = slot->mrq;
1291	host->cur_slot = slot;
1292	host->mrq = mrq;
1293
1294	host->pending_events = 0;
1295	host->completed_events = 0;
1296	host->cmd_status = 0;
1297	host->data_status = 0;
1298
1299	dev_dbg(&host->pdev->dev, "start request: cmd %u\n", mrq->cmd->opcode);
1300
1301	if (host->need_reset || host->caps.need_reset_after_xfer) {
1302	iflags = atmci_readl(host, ATMCI_IMR);
1303	iflags &= (ATMCI_SDIOIRQA | ATMCI_SDIOIRQB);
1304	atmci_writel(host, ATMCI_CR, ATMCI_CR_SWRST);
1305	atmci_writel(host, ATMCI_CR, ATMCI_CR_MCIEN);
1306	atmci_writel(host, ATMCI_MR, host->mode_reg);
1307	if (host->caps.has_cfg_reg)
1308	atmci_writel(host, ATMCI_CFG, host->cfg_reg);
1309	atmci_writel(host, ATMCI_IER, iflags);
1310	host->need_reset = false;
1311	}
1312	atmci_writel(host, ATMCI_SDCR, slot->sdc_reg);
1313
1314	iflags = atmci_readl(host, ATMCI_IMR);
1315	if (iflags & ~(ATMCI_SDIOIRQA | ATMCI_SDIOIRQB))
1316	dev_dbg(&slot->mmc->class_dev, "WARNING: IMR=0x%08x\n",
1317	iflags);
1318
1319	if (unlikely(test_and_clear_bit(ATMCI_CARD_NEED_INIT, &slot->flags))) {
1320	/* Send init sequence (74 clock cycles) */
1321	atmci_writel(host, ATMCI_CMDR, ATMCI_CMDR_SPCMD_INIT);
1322	while (!(atmci_readl(host, ATMCI_SR) & ATMCI_CMDRDY))
1323	cpu_relax();
1324	}
1325	iflags = 0;
1326	data = mrq->data;
1327	if (data) {
1328	atmci_set_timeout(host, slot, data);
1329
1330	/* Must set block count/size before sending command */
1331	atmci_writel(host, ATMCI_BLKR, ATMCI_BCNT(data->blocks)
1332	| ATMCI_BLKLEN(data->blksz));
1333	dev_vdbg(&slot->mmc->class_dev, "BLKR=0x%08x\n",
1334	ATMCI_BCNT(data->blocks) | ATMCI_BLKLEN(data->blksz));
1335
1336	iflags |= host->prepare_data(host, data);
1337	}
1338
1339	iflags |= ATMCI_CMDRDY;
1340	cmd = mrq->cmd;
1341	cmdflags = atmci_prepare_command(mmc: slot->mmc, cmd);
1342
1343	/*
1344	* DMA transfer should be started before sending the command to avoid
1345	* unexpected errors especially for read operations in SDIO mode.
1346	* Unfortunately, in PDC mode, command has to be sent before starting
1347	* the transfer.
1348	*/
1349	if (host->submit_data != &atmci_submit_data_dma)
1350	atmci_send_command(host, cmd, cmd_flags: cmdflags);
1351
1352	if (data)
1353	host->submit_data(host, data);
1354
1355	if (host->submit_data == &atmci_submit_data_dma)
1356	atmci_send_command(host, cmd, cmd_flags: cmdflags);
1357
1358	if (mrq->stop) {
1359	host->stop_cmdr = atmci_prepare_command(mmc: slot->mmc, cmd: mrq->stop);
1360	host->stop_cmdr |= ATMCI_CMDR_STOP_XFER;
1361	if (!(data->flags & MMC_DATA_WRITE))
1362	host->stop_cmdr |= ATMCI_CMDR_TRDIR_READ;
1363	host->stop_cmdr |= ATMCI_CMDR_MULTI_BLOCK;
1364	}
1365
1366	/*
1367	* We could have enabled interrupts earlier, but I suspect
1368	* that would open up a nice can of interesting race
1369	* conditions (e.g. command and data complete, but stop not
1370	* prepared yet.)
1371	*/
1372	atmci_writel(host, ATMCI_IER, iflags);
1373	}
1374
1375	static void atmci_queue_request(struct atmel_mci *host,
1376	struct atmel_mci_slot *slot, struct mmc_request *mrq)
1377	{
1378	dev_vdbg(&slot->mmc->class_dev, "queue request: state=%d\n",
1379	host->state);
1380
1381	spin_lock_bh(lock: &host->lock);
1382	slot->mrq = mrq;
1383	if (host->state == STATE_IDLE) {
1384	host->state = STATE_SENDING_CMD;
1385	atmci_start_request(host, slot);
1386	} else {
1387	dev_dbg(&host->pdev->dev, "queue request\n");
1388	list_add_tail(new: &slot->queue_node, head: &host->queue);
1389	}
1390	spin_unlock_bh(lock: &host->lock);
1391	}
1392
1393	static void atmci_request(struct mmc_host *mmc, struct mmc_request *mrq)
1394	{
1395	struct atmel_mci_slot *slot = mmc_priv(host: mmc);
1396	struct atmel_mci *host = slot->host;
1397	struct mmc_data *data;
1398
1399	WARN_ON(slot->mrq);
1400	dev_dbg(&host->pdev->dev, "MRQ: cmd %u\n", mrq->cmd->opcode);
1401
1402	/*
1403	* We may "know" the card is gone even though there's still an
1404	* electrical connection. If so, we really need to communicate
1405	* this to the MMC core since there won't be any more
1406	* interrupts as the card is completely removed. Otherwise,
1407	* the MMC core might believe the card is still there even
1408	* though the card was just removed very slowly.
1409	*/
1410	if (!test_bit(ATMCI_CARD_PRESENT, &slot->flags)) {
1411	mrq->cmd->error = -ENOMEDIUM;
1412	mmc_request_done(mmc, mrq);
1413	return;
1414	}
1415
1416	/* We don't support multiple blocks of weird lengths. */
1417	data = mrq->data;
1418	if (data && data->blocks > 1 && data->blksz & 3) {
1419	mrq->cmd->error = -EINVAL;
1420	mmc_request_done(mmc, mrq);
1421	}
1422
1423	atmci_queue_request(host, slot, mrq);
1424	}
1425
1426	static void atmci_set_ios(struct mmc_host *mmc, struct mmc_ios *ios)
1427	{
1428	struct atmel_mci_slot *slot = mmc_priv(host: mmc);
1429	struct atmel_mci *host = slot->host;
1430	unsigned int i;
1431
1432	slot->sdc_reg &= ~ATMCI_SDCBUS_MASK;
1433	switch (ios->bus_width) {
1434	case MMC_BUS_WIDTH_1:
1435	slot->sdc_reg |= ATMCI_SDCBUS_1BIT;
1436	break;
1437	case MMC_BUS_WIDTH_4:
1438	slot->sdc_reg |= ATMCI_SDCBUS_4BIT;
1439	break;
1440	case MMC_BUS_WIDTH_8:
1441	slot->sdc_reg |= ATMCI_SDCBUS_8BIT;
1442	break;
1443	}
1444
1445	if (ios->clock) {
1446	unsigned int clock_min = ~0U;
1447	int clkdiv;
1448
1449	spin_lock_bh(lock: &host->lock);
1450	if (!host->mode_reg) {
1451	atmci_writel(host, ATMCI_CR, ATMCI_CR_SWRST);
1452	atmci_writel(host, ATMCI_CR, ATMCI_CR_MCIEN);
1453	if (host->caps.has_cfg_reg)
1454	atmci_writel(host, ATMCI_CFG, host->cfg_reg);
1455	}
1456
1457	/*
1458	* Use mirror of ios->clock to prevent race with mmc
1459	* core ios update when finding the minimum.
1460	*/
1461	slot->clock = ios->clock;
1462	for (i = 0; i < ATMCI_MAX_NR_SLOTS; i++) {
1463	if (host->slot[i] && host->slot[i]->clock
1464	&& host->slot[i]->clock < clock_min)
1465	clock_min = host->slot[i]->clock;
1466	}
1467
1468	/* Calculate clock divider */
1469	if (host->caps.has_odd_clk_div) {
1470	clkdiv = DIV_ROUND_UP(host->bus_hz, clock_min) - 2;
1471	if (clkdiv < 0) {
1472	dev_warn(&mmc->class_dev,
1473	"clock %u too fast; using %lu\n",
1474	clock_min, host->bus_hz / 2);
1475	clkdiv = 0;
1476	} else if (clkdiv > 511) {
1477	dev_warn(&mmc->class_dev,
1478	"clock %u too slow; using %lu\n",
1479	clock_min, host->bus_hz / (511 + 2));
1480	clkdiv = 511;
1481	}
1482	host->mode_reg = ATMCI_MR_CLKDIV(clkdiv >> 1)
1483	| ATMCI_MR_CLKODD(clkdiv & 1);
1484	} else {
1485	clkdiv = DIV_ROUND_UP(host->bus_hz, 2 * clock_min) - 1;
1486	if (clkdiv > 255) {
1487	dev_warn(&mmc->class_dev,
1488	"clock %u too slow; using %lu\n",
1489	clock_min, host->bus_hz / (2 * 256));
1490	clkdiv = 255;
1491	}
1492	host->mode_reg = ATMCI_MR_CLKDIV(clkdiv);
1493	}
1494
1495	/*
1496	* WRPROOF and RDPROOF prevent overruns/underruns by
1497	* stopping the clock when the FIFO is full/empty.
1498	* This state is not expected to last for long.
1499	*/
1500	if (host->caps.has_rwproof)
1501	host->mode_reg |= (ATMCI_MR_WRPROOF | ATMCI_MR_RDPROOF);
1502
1503	if (host->caps.has_cfg_reg) {
1504	/* setup High Speed mode in relation with card capacity */
1505	if (ios->timing == MMC_TIMING_SD_HS)
1506	host->cfg_reg |= ATMCI_CFG_HSMODE;
1507	else
1508	host->cfg_reg &= ~ATMCI_CFG_HSMODE;
1509	}
1510
1511	if (list_empty(head: &host->queue)) {
1512	atmci_writel(host, ATMCI_MR, host->mode_reg);
1513	if (host->caps.has_cfg_reg)
1514	atmci_writel(host, ATMCI_CFG, host->cfg_reg);
1515	} else {
1516	host->need_clock_update = true;
1517	}
1518
1519	spin_unlock_bh(lock: &host->lock);
1520	} else {
1521	bool any_slot_active = false;
1522
1523	spin_lock_bh(lock: &host->lock);
1524	slot->clock = 0;
1525	for (i = 0; i < ATMCI_MAX_NR_SLOTS; i++) {
1526	if (host->slot[i] && host->slot[i]->clock) {
1527	any_slot_active = true;
1528	break;
1529	}
1530	}
1531	if (!any_slot_active) {
1532	atmci_writel(host, ATMCI_CR, ATMCI_CR_MCIDIS);
1533	if (host->mode_reg) {
1534	atmci_readl(host, ATMCI_MR);
1535	}
1536	host->mode_reg = 0;
1537	}
1538	spin_unlock_bh(lock: &host->lock);
1539	}
1540
1541	switch (ios->power_mode) {
1542	case MMC_POWER_OFF:
1543	if (!IS_ERR(ptr: mmc->supply.vmmc))
1544	mmc_regulator_set_ocr(mmc, supply: mmc->supply.vmmc, vdd_bit: 0);
1545	break;
1546	case MMC_POWER_UP:
1547	set_bit(ATMCI_CARD_NEED_INIT, addr: &slot->flags);
1548	if (!IS_ERR(ptr: mmc->supply.vmmc))
1549	mmc_regulator_set_ocr(mmc, supply: mmc->supply.vmmc, vdd_bit: ios->vdd);
1550	break;
1551	default:
1552	break;
1553	}
1554	}
1555
1556	static int atmci_get_ro(struct mmc_host *mmc)
1557	{
1558	int read_only = -ENOSYS;
1559	struct atmel_mci_slot *slot = mmc_priv(host: mmc);
1560
1561	if (slot->wp_pin) {
1562	read_only = gpiod_get_value(desc: slot->wp_pin);
1563	dev_dbg(&mmc->class_dev, "card is %s\n",
1564	read_only ? "read-only" : "read-write");
1565	}
1566
1567	return read_only;
1568	}
1569
1570	static int atmci_get_cd(struct mmc_host *mmc)
1571	{
1572	int present = -ENOSYS;
1573	struct atmel_mci_slot *slot = mmc_priv(host: mmc);
1574
1575	if (slot->detect_pin) {
1576	present = gpiod_get_value_cansleep(desc: slot->detect_pin);
1577	dev_dbg(&mmc->class_dev, "card is %spresent\n",
1578	present ? "" : "not ");
1579	}
1580
1581	return present;
1582	}
1583
1584	static void atmci_enable_sdio_irq(struct mmc_host *mmc, int enable)
1585	{
1586	struct atmel_mci_slot *slot = mmc_priv(host: mmc);
1587	struct atmel_mci *host = slot->host;
1588
1589	if (enable)
1590	atmci_writel(host, ATMCI_IER, slot->sdio_irq);
1591	else
1592	atmci_writel(host, ATMCI_IDR, slot->sdio_irq);
1593	}
1594
1595	static const struct mmc_host_ops atmci_ops = {
1596	.request = atmci_request,
1597	.set_ios = atmci_set_ios,
1598	.get_ro = atmci_get_ro,
1599	.get_cd = atmci_get_cd,
1600	.enable_sdio_irq = atmci_enable_sdio_irq,
1601	};
1602
1603	/* Called with host->lock held */
1604	static void atmci_request_end(struct atmel_mci *host, struct mmc_request *mrq)
1605	__releases(&host->lock)
1606	__acquires(&host->lock)
1607	{
1608	struct atmel_mci_slot *slot = NULL;
1609	struct mmc_host *prev_mmc = host->cur_slot->mmc;
1610
1611	WARN_ON(host->cmd || host->data);
1612
1613	del_timer(timer: &host->timer);
1614
1615	/*
1616	* Update the MMC clock rate if necessary. This may be
1617	* necessary if set_ios() is called when a different slot is
1618	* busy transferring data.
1619	*/
1620	if (host->need_clock_update) {
1621	atmci_writel(host, ATMCI_MR, host->mode_reg);
1622	if (host->caps.has_cfg_reg)
1623	atmci_writel(host, ATMCI_CFG, host->cfg_reg);
1624	}
1625
1626	host->cur_slot->mrq = NULL;
1627	host->mrq = NULL;
1628	if (!list_empty(head: &host->queue)) {
1629	slot = list_entry(host->queue.next,
1630	struct atmel_mci_slot, queue_node);
1631	list_del(entry: &slot->queue_node);
1632	dev_vdbg(&host->pdev->dev, "list not empty: %s is next\n",
1633	mmc_hostname(slot->mmc));
1634	host->state = STATE_SENDING_CMD;
1635	atmci_start_request(host, slot);
1636	} else {
1637	dev_vdbg(&host->pdev->dev, "list empty\n");
1638	host->state = STATE_IDLE;
1639	}
1640
1641	spin_unlock(lock: &host->lock);
1642	mmc_request_done(prev_mmc, mrq);
1643	spin_lock(lock: &host->lock);
1644	}
1645
1646	static void atmci_command_complete(struct atmel_mci *host,
1647	struct mmc_command *cmd)
1648	{
1649	u32 status = host->cmd_status;
1650
1651	/* Read the response from the card (up to 16 bytes) */
1652	cmd->resp[0] = atmci_readl(host, ATMCI_RSPR);
1653	cmd->resp[1] = atmci_readl(host, ATMCI_RSPR);
1654	cmd->resp[2] = atmci_readl(host, ATMCI_RSPR);
1655	cmd->resp[3] = atmci_readl(host, ATMCI_RSPR);
1656
1657	if (status & ATMCI_RTOE)
1658	cmd->error = -ETIMEDOUT;
1659	else if ((cmd->flags & MMC_RSP_CRC) && (status & ATMCI_RCRCE))
1660	cmd->error = -EILSEQ;
1661	else if (status & (ATMCI_RINDE | ATMCI_RDIRE | ATMCI_RENDE))
1662	cmd->error = -EIO;
1663	else if (host->mrq->data && (host->mrq->data->blksz & 3)) {
1664	if (host->caps.need_blksz_mul_4) {
1665	cmd->error = -EINVAL;
1666	host->need_reset = 1;
1667	}
1668	} else
1669	cmd->error = 0;
1670	}
1671
1672	static void atmci_detect_change(struct timer_list *t)
1673	{
1674	struct atmel_mci_slot *slot = from_timer(slot, t, detect_timer);
1675	bool present;
1676	bool present_old;
1677
1678	/*
1679	* atmci_cleanup_slot() sets the ATMCI_SHUTDOWN flag before
1680	* freeing the interrupt. We must not re-enable the interrupt
1681	* if it has been freed, and if we're shutting down, it
1682	* doesn't really matter whether the card is present or not.
1683	*/
1684	smp_rmb();
1685	if (test_bit(ATMCI_SHUTDOWN, &slot->flags))
1686	return;
1687
1688	enable_irq(irq: gpiod_to_irq(desc: slot->detect_pin));
1689	present = gpiod_get_value_cansleep(desc: slot->detect_pin);
1690	present_old = test_bit(ATMCI_CARD_PRESENT, &slot->flags);
1691
1692	dev_vdbg(&slot->mmc->class_dev, "detect change: %d (was %d)\n",
1693	present, present_old);
1694
1695	if (present != present_old) {
1696	struct atmel_mci *host = slot->host;
1697	struct mmc_request *mrq;
1698
1699	dev_dbg(&slot->mmc->class_dev, "card %s\n",
1700	present ? "inserted" : "removed");
1701
1702	spin_lock(lock: &host->lock);
1703
1704	if (!present)
1705	clear_bit(ATMCI_CARD_PRESENT, addr: &slot->flags);
1706	else
1707	set_bit(ATMCI_CARD_PRESENT, addr: &slot->flags);
1708
1709	/* Clean up queue if present */
1710	mrq = slot->mrq;
1711	if (mrq) {
1712	if (mrq == host->mrq) {
1713	/*
1714	* Reset controller to terminate any ongoing
1715	* commands or data transfers.
1716	*/
1717	atmci_writel(host, ATMCI_CR, ATMCI_CR_SWRST);
1718	atmci_writel(host, ATMCI_CR, ATMCI_CR_MCIEN);
1719	atmci_writel(host, ATMCI_MR, host->mode_reg);
1720	if (host->caps.has_cfg_reg)
1721	atmci_writel(host, ATMCI_CFG, host->cfg_reg);
1722
1723	host->data = NULL;
1724	host->cmd = NULL;
1725
1726	switch (host->state) {
1727	case STATE_IDLE:
1728	break;
1729	case STATE_SENDING_CMD:
1730	mrq->cmd->error = -ENOMEDIUM;
1731	if (mrq->data)
1732	host->stop_transfer(host);
1733	break;
1734	case STATE_DATA_XFER:
1735	mrq->data->error = -ENOMEDIUM;
1736	host->stop_transfer(host);
1737	break;
1738	case STATE_WAITING_NOTBUSY:
1739	mrq->data->error = -ENOMEDIUM;
1740	break;
1741	case STATE_SENDING_STOP:
1742	mrq->stop->error = -ENOMEDIUM;
1743	break;
1744	case STATE_END_REQUEST:
1745	break;
1746	}
1747
1748	atmci_request_end(host, mrq);
1749	} else {
1750	list_del(entry: &slot->queue_node);
1751	mrq->cmd->error = -ENOMEDIUM;
1752	if (mrq->data)
1753	mrq->data->error = -ENOMEDIUM;
1754	if (mrq->stop)
1755	mrq->stop->error = -ENOMEDIUM;
1756
1757	spin_unlock(lock: &host->lock);
1758	mmc_request_done(slot->mmc, mrq);
1759	spin_lock(lock: &host->lock);
1760	}
1761	}
1762	spin_unlock(lock: &host->lock);
1763
1764	mmc_detect_change(slot->mmc, delay: 0);
1765	}
1766	}
1767
1768	static void atmci_tasklet_func(struct tasklet_struct *t)
1769	{
1770	struct atmel_mci *host = from_tasklet(host, t, tasklet);
1771	struct mmc_request *mrq = host->mrq;
1772	struct mmc_data *data = host->data;
1773	enum atmel_mci_state state = host->state;
1774	enum atmel_mci_state prev_state;
1775	u32 status;
1776
1777	spin_lock(lock: &host->lock);
1778
1779	state = host->state;
1780
1781	dev_vdbg(&host->pdev->dev,
1782	"tasklet: state %u pending/completed/mask %lx/%lx/%x\n",
1783	state, host->pending_events, host->completed_events,
1784	atmci_readl(host, ATMCI_IMR));
1785
1786	do {
1787	prev_state = state;
1788	dev_dbg(&host->pdev->dev, "FSM: state=%d\n", state);
1789
1790	switch (state) {
1791	case STATE_IDLE:
1792	break;
1793
1794	case STATE_SENDING_CMD:
1795	/*
1796	* Command has been sent, we are waiting for command
1797	* ready. Then we have three next states possible:
1798	* END_REQUEST by default, WAITING_NOTBUSY if it's a
1799	* command needing it or DATA_XFER if there is data.
1800	*/
1801	dev_dbg(&host->pdev->dev, "FSM: cmd ready?\n");
1802	if (!atmci_test_and_clear_pending(host,
1803	EVENT_CMD_RDY))
1804	break;
1805
1806	dev_dbg(&host->pdev->dev, "set completed cmd ready\n");
1807	host->cmd = NULL;
1808	atmci_set_completed(host, EVENT_CMD_RDY);
1809	atmci_command_complete(host, cmd: mrq->cmd);
1810	if (mrq->data) {
1811	dev_dbg(&host->pdev->dev,
1812	"command with data transfer");
1813	/*
1814	* If there is a command error don't start
1815	* data transfer.
1816	*/
1817	if (mrq->cmd->error) {
1818	host->stop_transfer(host);
1819	host->data = NULL;
1820	atmci_writel(host, ATMCI_IDR,
1821	ATMCI_TXRDY | ATMCI_RXRDY
1822	| ATMCI_DATA_ERROR_FLAGS);
1823	state = STATE_END_REQUEST;
1824	} else
1825	state = STATE_DATA_XFER;
1826	} else if ((!mrq->data) && (mrq->cmd->flags & MMC_RSP_BUSY)) {
1827	dev_dbg(&host->pdev->dev,
1828	"command response need waiting notbusy");
1829	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1830	state = STATE_WAITING_NOTBUSY;
1831	} else
1832	state = STATE_END_REQUEST;
1833
1834	break;
1835
1836	case STATE_DATA_XFER:
1837	if (atmci_test_and_clear_pending(host,
1838	EVENT_DATA_ERROR)) {
1839	dev_dbg(&host->pdev->dev, "set completed data error\n");
1840	atmci_set_completed(host, EVENT_DATA_ERROR);
1841	state = STATE_END_REQUEST;
1842	break;
1843	}
1844
1845	/*
1846	* A data transfer is in progress. The event expected
1847	* to move to the next state depends of data transfer
1848	* type (PDC or DMA). Once transfer done we can move
1849	* to the next step which is WAITING_NOTBUSY in write
1850	* case and directly SENDING_STOP in read case.
1851	*/
1852	dev_dbg(&host->pdev->dev, "FSM: xfer complete?\n");
1853	if (!atmci_test_and_clear_pending(host,
1854	EVENT_XFER_COMPLETE))
1855	break;
1856
1857	dev_dbg(&host->pdev->dev,
1858	"(%s) set completed xfer complete\n",
1859	__func__);
1860	atmci_set_completed(host, EVENT_XFER_COMPLETE);
1861
1862	if (host->caps.need_notbusy_for_read_ops ||
1863	(host->data->flags & MMC_DATA_WRITE)) {
1864	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1865	state = STATE_WAITING_NOTBUSY;
1866	} else if (host->mrq->stop) {
1867	atmci_send_stop_cmd(host, data);
1868	state = STATE_SENDING_STOP;
1869	} else {
1870	host->data = NULL;
1871	data->bytes_xfered = data->blocks * data->blksz;
1872	data->error = 0;
1873	state = STATE_END_REQUEST;
1874	}
1875	break;
1876
1877	case STATE_WAITING_NOTBUSY:
1878	/*
1879	* We can be in the state for two reasons: a command
1880	* requiring waiting not busy signal (stop command
1881	* included) or a write operation. In the latest case,
1882	* we need to send a stop command.
1883	*/
1884	dev_dbg(&host->pdev->dev, "FSM: not busy?\n");
1885	if (!atmci_test_and_clear_pending(host,
1886	EVENT_NOTBUSY))
1887	break;
1888
1889	dev_dbg(&host->pdev->dev, "set completed not busy\n");
1890	atmci_set_completed(host, EVENT_NOTBUSY);
1891
1892	if (host->data) {
1893	/*
1894	* For some commands such as CMD53, even if
1895	* there is data transfer, there is no stop
1896	* command to send.
1897	*/
1898	if (host->mrq->stop) {
1899	atmci_send_stop_cmd(host, data);
1900	state = STATE_SENDING_STOP;
1901	} else {
1902	host->data = NULL;
1903	data->bytes_xfered = data->blocks
1904	* data->blksz;
1905	data->error = 0;
1906	state = STATE_END_REQUEST;
1907	}
1908	} else
1909	state = STATE_END_REQUEST;
1910	break;
1911
1912	case STATE_SENDING_STOP:
1913	/*
1914	* In this state, it is important to set host->data to
1915	* NULL (which is tested in the waiting notbusy state)
1916	* in order to go to the end request state instead of
1917	* sending stop again.
1918	*/
1919	dev_dbg(&host->pdev->dev, "FSM: cmd ready?\n");
1920	if (!atmci_test_and_clear_pending(host,
1921	EVENT_CMD_RDY))
1922	break;
1923
1924	dev_dbg(&host->pdev->dev, "FSM: cmd ready\n");
1925	host->cmd = NULL;
1926	data->bytes_xfered = data->blocks * data->blksz;
1927	data->error = 0;
1928	atmci_command_complete(host, cmd: mrq->stop);
1929	if (mrq->stop->error) {
1930	host->stop_transfer(host);
1931	atmci_writel(host, ATMCI_IDR,
1932	ATMCI_TXRDY | ATMCI_RXRDY
1933	| ATMCI_DATA_ERROR_FLAGS);
1934	state = STATE_END_REQUEST;
1935	} else {
1936	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
1937	state = STATE_WAITING_NOTBUSY;
1938	}
1939	host->data = NULL;
1940	break;
1941
1942	case STATE_END_REQUEST:
1943	atmci_writel(host, ATMCI_IDR, ATMCI_TXRDY | ATMCI_RXRDY
1944	| ATMCI_DATA_ERROR_FLAGS);
1945	status = host->data_status;
1946	if (unlikely(status)) {
1947	host->stop_transfer(host);
1948	host->data = NULL;
1949	if (data) {
1950	if (status & ATMCI_DTOE) {
1951	data->error = -ETIMEDOUT;
1952	} else if (status & ATMCI_DCRCE) {
1953	data->error = -EILSEQ;
1954	} else {
1955	data->error = -EIO;
1956	}
1957	}
1958	}
1959
1960	atmci_request_end(host, mrq: host->mrq);
1961	goto unlock; /* atmci_request_end() sets host->state */
1962	break;
1963	}
1964	} while (state != prev_state);
1965
1966	host->state = state;
1967
1968	unlock:
1969	spin_unlock(lock: &host->lock);
1970	}
1971
1972	static void atmci_read_data_pio(struct atmel_mci *host)
1973	{
1974	struct scatterlist *sg = host->sg;
1975	unsigned int offset = host->pio_offset;
1976	struct mmc_data *data = host->data;
1977	u32 value;
1978	u32 status;
1979	unsigned int nbytes = 0;
1980
1981	do {
1982	value = atmci_readl(host, ATMCI_RDR);
1983	if (likely(offset + 4 <= sg->length)) {
1984	sg_pcopy_from_buffer(sgl: sg, nents: 1, buf: &value, buflen: sizeof(u32), skip: offset);
1985
1986	offset += 4;
1987	nbytes += 4;
1988
1989	if (offset == sg->length) {
1990	flush_dcache_page(page: sg_page(sg));
1991	host->sg = sg = sg_next(sg);
1992	host->sg_len--;
1993	if (!sg || !host->sg_len)
1994	goto done;
1995
1996	offset = 0;
1997	}
1998	} else {
1999	unsigned int remaining = sg->length - offset;
2000
2001	sg_pcopy_from_buffer(sgl: sg, nents: 1, buf: &value, buflen: remaining, skip: offset);
2002	nbytes += remaining;
2003
2004	flush_dcache_page(page: sg_page(sg));
2005	host->sg = sg = sg_next(sg);
2006	host->sg_len--;
2007	if (!sg || !host->sg_len)
2008	goto done;
2009
2010	offset = 4 - remaining;
2011	sg_pcopy_from_buffer(sgl: sg, nents: 1, buf: (u8 *)&value + remaining,
2012	buflen: offset, skip: 0);
2013	nbytes += offset;
2014	}
2015
2016	status = atmci_readl(host, ATMCI_SR);
2017	if (status & ATMCI_DATA_ERROR_FLAGS) {
2018	atmci_writel(host, ATMCI_IDR, (ATMCI_NOTBUSY | ATMCI_RXRDY
2019	| ATMCI_DATA_ERROR_FLAGS));
2020	host->data_status = status;
2021	data->bytes_xfered += nbytes;
2022	return;
2023	}
2024	} while (status & ATMCI_RXRDY);
2025
2026	host->pio_offset = offset;
2027	data->bytes_xfered += nbytes;
2028
2029	return;
2030
2031	done:
2032	atmci_writel(host, ATMCI_IDR, ATMCI_RXRDY);
2033	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
2034	data->bytes_xfered += nbytes;
2035	smp_wmb();
2036	atmci_set_pending(host, EVENT_XFER_COMPLETE);
2037	}
2038
2039	static void atmci_write_data_pio(struct atmel_mci *host)
2040	{
2041	struct scatterlist *sg = host->sg;
2042	unsigned int offset = host->pio_offset;
2043	struct mmc_data *data = host->data;
2044	u32 value;
2045	u32 status;
2046	unsigned int nbytes = 0;
2047
2048	do {
2049	if (likely(offset + 4 <= sg->length)) {
2050	sg_pcopy_to_buffer(sgl: sg, nents: 1, buf: &value, buflen: sizeof(u32), skip: offset);
2051	atmci_writel(host, ATMCI_TDR, value);
2052
2053	offset += 4;
2054	nbytes += 4;
2055	if (offset == sg->length) {
2056	host->sg = sg = sg_next(sg);
2057	host->sg_len--;
2058	if (!sg || !host->sg_len)
2059	goto done;
2060
2061	offset = 0;
2062	}
2063	} else {
2064	unsigned int remaining = sg->length - offset;
2065
2066	value = 0;
2067	sg_pcopy_to_buffer(sgl: sg, nents: 1, buf: &value, buflen: remaining, skip: offset);
2068	nbytes += remaining;
2069
2070	host->sg = sg = sg_next(sg);
2071	host->sg_len--;
2072	if (!sg || !host->sg_len) {
2073	atmci_writel(host, ATMCI_TDR, value);
2074	goto done;
2075	}
2076
2077	offset = 4 - remaining;
2078	sg_pcopy_to_buffer(sgl: sg, nents: 1, buf: (u8 *)&value + remaining,
2079	buflen: offset, skip: 0);
2080	atmci_writel(host, ATMCI_TDR, value);
2081	nbytes += offset;
2082	}
2083
2084	status = atmci_readl(host, ATMCI_SR);
2085	if (status & ATMCI_DATA_ERROR_FLAGS) {
2086	atmci_writel(host, ATMCI_IDR, (ATMCI_NOTBUSY | ATMCI_TXRDY
2087	| ATMCI_DATA_ERROR_FLAGS));
2088	host->data_status = status;
2089	data->bytes_xfered += nbytes;
2090	return;
2091	}
2092	} while (status & ATMCI_TXRDY);
2093
2094	host->pio_offset = offset;
2095	data->bytes_xfered += nbytes;
2096
2097	return;
2098
2099	done:
2100	atmci_writel(host, ATMCI_IDR, ATMCI_TXRDY);
2101	atmci_writel(host, ATMCI_IER, ATMCI_NOTBUSY);
2102	data->bytes_xfered += nbytes;
2103	smp_wmb();
2104	atmci_set_pending(host, EVENT_XFER_COMPLETE);
2105	}
2106
2107	static void atmci_sdio_interrupt(struct atmel_mci *host, u32 status)
2108	{
2109	int i;
2110
2111	for (i = 0; i < ATMCI_MAX_NR_SLOTS; i++) {
2112	struct atmel_mci_slot *slot = host->slot[i];
2113	if (slot && (status & slot->sdio_irq)) {
2114	mmc_signal_sdio_irq(host: slot->mmc);
2115	}
2116	}
2117	}
2118
2119
2120	static irqreturn_t atmci_interrupt(int irq, void *dev_id)
2121	{
2122	struct atmel_mci *host = dev_id;
2123	u32 status, mask, pending;
2124	unsigned int pass_count = 0;
2125
2126	do {
2127	status = atmci_readl(host, ATMCI_SR);
2128	mask = atmci_readl(host, ATMCI_IMR);
2129	pending = status & mask;
2130	if (!pending)
2131	break;
2132
2133	if (pending & ATMCI_DATA_ERROR_FLAGS) {
2134	dev_dbg(&host->pdev->dev, "IRQ: data error\n");
2135	atmci_writel(host, ATMCI_IDR, ATMCI_DATA_ERROR_FLAGS
2136	| ATMCI_RXRDY | ATMCI_TXRDY
2137	| ATMCI_ENDRX | ATMCI_ENDTX
2138	| ATMCI_RXBUFF | ATMCI_TXBUFE);
2139
2140	host->data_status = status;
2141	dev_dbg(&host->pdev->dev, "set pending data error\n");
2142	smp_wmb();
2143	atmci_set_pending(host, EVENT_DATA_ERROR);
2144	tasklet_schedule(t: &host->tasklet);
2145	}
2146
2147	if (pending & ATMCI_TXBUFE) {
2148	dev_dbg(&host->pdev->dev, "IRQ: tx buffer empty\n");
2149	atmci_writel(host, ATMCI_IDR, ATMCI_TXBUFE);
2150	atmci_writel(host, ATMCI_IDR, ATMCI_ENDTX);
2151	/*
2152	* We can receive this interruption before having configured
2153	* the second pdc buffer, so we need to reconfigure first and
2154	* second buffers again
2155	*/
2156	if (host->data_size) {
2157	atmci_pdc_set_both_buf(host, dir: XFER_TRANSMIT);
2158	atmci_writel(host, ATMCI_IER, ATMCI_ENDTX);
2159	atmci_writel(host, ATMCI_IER, ATMCI_TXBUFE);
2160	} else {
2161	atmci_pdc_complete(host);
2162	}
2163	} else if (pending & ATMCI_ENDTX) {
2164	dev_dbg(&host->pdev->dev, "IRQ: end of tx buffer\n");
2165	atmci_writel(host, ATMCI_IDR, ATMCI_ENDTX);
2166
2167	if (host->data_size) {
2168	atmci_pdc_set_single_buf(host,
2169	dir: XFER_TRANSMIT, buf_nb: PDC_SECOND_BUF);
2170	atmci_writel(host, ATMCI_IER, ATMCI_ENDTX);
2171	}
2172	}
2173
2174	if (pending & ATMCI_RXBUFF) {
2175	dev_dbg(&host->pdev->dev, "IRQ: rx buffer full\n");
2176	atmci_writel(host, ATMCI_IDR, ATMCI_RXBUFF);
2177	atmci_writel(host, ATMCI_IDR, ATMCI_ENDRX);
2178	/*
2179	* We can receive this interruption before having configured
2180	* the second pdc buffer, so we need to reconfigure first and
2181	* second buffers again
2182	*/
2183	if (host->data_size) {
2184	atmci_pdc_set_both_buf(host, dir: XFER_RECEIVE);
2185	atmci_writel(host, ATMCI_IER, ATMCI_ENDRX);
2186	atmci_writel(host, ATMCI_IER, ATMCI_RXBUFF);
2187	} else {
2188	atmci_pdc_complete(host);
2189	}
2190	} else if (pending & ATMCI_ENDRX) {
2191	dev_dbg(&host->pdev->dev, "IRQ: end of rx buffer\n");
2192	atmci_writel(host, ATMCI_IDR, ATMCI_ENDRX);
2193
2194	if (host->data_size) {
2195	atmci_pdc_set_single_buf(host,
2196	dir: XFER_RECEIVE, buf_nb: PDC_SECOND_BUF);
2197	atmci_writel(host, ATMCI_IER, ATMCI_ENDRX);
2198	}
2199	}
2200
2201	/*
2202	* First mci IPs, so mainly the ones having pdc, have some
2203	* issues with the notbusy signal. You can't get it after
2204	* data transmission if you have not sent a stop command.
2205	* The appropriate workaround is to use the BLKE signal.
2206	*/
2207	if (pending & ATMCI_BLKE) {
2208	dev_dbg(&host->pdev->dev, "IRQ: blke\n");
2209	atmci_writel(host, ATMCI_IDR, ATMCI_BLKE);
2210	smp_wmb();
2211	dev_dbg(&host->pdev->dev, "set pending notbusy\n");
2212	atmci_set_pending(host, EVENT_NOTBUSY);
2213	tasklet_schedule(t: &host->tasklet);
2214	}
2215
2216	if (pending & ATMCI_NOTBUSY) {
2217	dev_dbg(&host->pdev->dev, "IRQ: not_busy\n");
2218	atmci_writel(host, ATMCI_IDR, ATMCI_NOTBUSY);
2219	smp_wmb();
2220	dev_dbg(&host->pdev->dev, "set pending notbusy\n");
2221	atmci_set_pending(host, EVENT_NOTBUSY);
2222	tasklet_schedule(t: &host->tasklet);
2223	}
2224
2225	if (pending & ATMCI_RXRDY)
2226	atmci_read_data_pio(host);
2227	if (pending & ATMCI_TXRDY)
2228	atmci_write_data_pio(host);
2229
2230	if (pending & ATMCI_CMDRDY) {
2231	dev_dbg(&host->pdev->dev, "IRQ: cmd ready\n");
2232	atmci_writel(host, ATMCI_IDR, ATMCI_CMDRDY);
2233	host->cmd_status = status;
2234	smp_wmb();
2235	dev_dbg(&host->pdev->dev, "set pending cmd rdy\n");
2236	atmci_set_pending(host, EVENT_CMD_RDY);
2237	tasklet_schedule(t: &host->tasklet);
2238	}
2239
2240	if (pending & (ATMCI_SDIOIRQA | ATMCI_SDIOIRQB))
2241	atmci_sdio_interrupt(host, status);
2242
2243	} while (pass_count++ < 5);
2244
2245	return pass_count ? IRQ_HANDLED : IRQ_NONE;
2246	}
2247
2248	static irqreturn_t atmci_detect_interrupt(int irq, void *dev_id)
2249	{
2250	struct atmel_mci_slot *slot = dev_id;
2251
2252	/*
2253	* Disable interrupts until the pin has stabilized and check
2254	* the state then. Use mod_timer() since we may be in the
2255	* middle of the timer routine when this interrupt triggers.
2256	*/
2257	disable_irq_nosync(irq);
2258	mod_timer(timer: &slot->detect_timer, expires: jiffies + msecs_to_jiffies(m: 20));
2259
2260	return IRQ_HANDLED;
2261	}
2262
2263	static int atmci_init_slot(struct atmel_mci *host,
2264	struct mci_slot_pdata *slot_data, unsigned int id,
2265	u32 sdc_reg, u32 sdio_irq)
2266	{
2267	struct mmc_host *mmc;
2268	struct atmel_mci_slot *slot;
2269	int ret;
2270
2271	mmc = mmc_alloc_host(extra: sizeof(struct atmel_mci_slot), &host->pdev->dev);
2272	if (!mmc)
2273	return -ENOMEM;
2274
2275	slot = mmc_priv(host: mmc);
2276	slot->mmc = mmc;
2277	slot->host = host;
2278	slot->detect_pin = slot_data->detect_pin;
2279	slot->wp_pin = slot_data->wp_pin;
2280	slot->sdc_reg = sdc_reg;
2281	slot->sdio_irq = sdio_irq;
2282
2283	dev_dbg(&mmc->class_dev,
2284	"slot[%u]: bus_width=%u, detect_pin=%d, "
2285	"detect_is_active_high=%s, wp_pin=%d\n",
2286	id, slot_data->bus_width, desc_to_gpio(slot_data->detect_pin),
2287	!gpiod_is_active_low(slot_data->detect_pin) ? "true" : "false",
2288	desc_to_gpio(slot_data->wp_pin));
2289
2290	mmc->ops = &atmci_ops;
2291	mmc->f_min = DIV_ROUND_UP(host->bus_hz, 512);
2292	mmc->f_max = host->bus_hz / 2;
2293	mmc->ocr_avail = MMC_VDD_32_33 | MMC_VDD_33_34;
2294	if (sdio_irq)
2295	mmc->caps |= MMC_CAP_SDIO_IRQ;
2296	if (host->caps.has_highspeed)
2297	mmc->caps |= MMC_CAP_SD_HIGHSPEED;
2298	/*
2299	* Without the read/write proof capability, it is strongly suggested to
2300	* use only one bit for data to prevent fifo underruns and overruns
2301	* which will corrupt data.
2302	*/
2303	if ((slot_data->bus_width >= 4) && host->caps.has_rwproof) {
2304	mmc->caps |= MMC_CAP_4_BIT_DATA;
2305	if (slot_data->bus_width >= 8)
2306	mmc->caps |= MMC_CAP_8_BIT_DATA;
2307	}
2308
2309	if (atmci_get_version(host) < 0x200) {
2310	mmc->max_segs = 256;
2311	mmc->max_blk_size = 4095;
2312	mmc->max_blk_count = 256;
2313	mmc->max_req_size = mmc->max_blk_size * mmc->max_blk_count;
2314	mmc->max_seg_size = mmc->max_blk_size * mmc->max_segs;
2315	} else {
2316	mmc->max_segs = 64;
2317	mmc->max_req_size = 32768 * 512;
2318	mmc->max_blk_size = 32768;
2319	mmc->max_blk_count = 512;
2320	}
2321
2322	/* Assume card is present initially */
2323	set_bit(ATMCI_CARD_PRESENT, addr: &slot->flags);
2324	if (slot->detect_pin) {
2325	if (!gpiod_get_value_cansleep(desc: slot->detect_pin))
2326	clear_bit(ATMCI_CARD_PRESENT, addr: &slot->flags);
2327	} else {
2328	dev_dbg(&mmc->class_dev, "no detect pin available\n");
2329	}
2330
2331	if (!slot->detect_pin) {
2332	if (slot_data->non_removable)
2333	mmc->caps |= MMC_CAP_NONREMOVABLE;
2334	else
2335	mmc->caps |= MMC_CAP_NEEDS_POLL;
2336	}
2337
2338	if (!slot->wp_pin)
2339	dev_dbg(&mmc->class_dev, "no WP pin available\n");
2340
2341	host->slot[id] = slot;
2342	mmc_regulator_get_supply(mmc);
2343	ret = mmc_add_host(mmc);
2344	if (ret) {
2345	mmc_free_host(mmc);
2346	return ret;
2347	}
2348
2349	if (slot->detect_pin) {
2350	timer_setup(&slot->detect_timer, atmci_detect_change, 0);
2351
2352	ret = request_irq(irq: gpiod_to_irq(desc: slot->detect_pin),
2353	handler: atmci_detect_interrupt,
2354	IRQF_TRIGGER_FALLING | IRQF_TRIGGER_RISING,
2355	name: "mmc-detect", dev: slot);
2356	if (ret) {
2357	dev_dbg(&mmc->class_dev,
2358	"could not request IRQ %d for detect pin\n",
2359	gpiod_to_irq(slot->detect_pin));
2360	slot->detect_pin = NULL;
2361	}
2362	}
2363
2364	atmci_init_debugfs(slot);
2365
2366	return 0;
2367	}
2368
2369	static void atmci_cleanup_slot(struct atmel_mci_slot *slot,
2370	unsigned int id)
2371	{
2372	/* Debugfs stuff is cleaned up by mmc core */
2373
2374	set_bit(ATMCI_SHUTDOWN, addr: &slot->flags);
2375	smp_wmb();
2376
2377	mmc_remove_host(slot->mmc);
2378
2379	if (slot->detect_pin) {
2380	free_irq(gpiod_to_irq(desc: slot->detect_pin), slot);
2381	del_timer_sync(timer: &slot->detect_timer);
2382	}
2383
2384	slot->host->slot[id] = NULL;
2385	mmc_free_host(slot->mmc);
2386	}
2387
2388	static int atmci_configure_dma(struct atmel_mci *host)
2389	{
2390	host->dma.chan = dma_request_chan(dev: &host->pdev->dev, name: "rxtx");
2391
2392	if (PTR_ERR(ptr: host->dma.chan) == -ENODEV) {
2393	struct mci_platform_data *pdata = host->pdev->dev.platform_data;
2394	dma_cap_mask_t mask;
2395
2396	if (!pdata || !pdata->dma_filter)
2397	return -ENODEV;
2398
2399	dma_cap_zero(mask);
2400	dma_cap_set(DMA_SLAVE, mask);
2401
2402	host->dma.chan = dma_request_channel(mask, pdata->dma_filter,
2403	pdata->dma_slave);
2404	if (!host->dma.chan)
2405	host->dma.chan = ERR_PTR(error: -ENODEV);
2406	}
2407
2408	if (IS_ERR(ptr: host->dma.chan))
2409	return PTR_ERR(ptr: host->dma.chan);
2410
2411	dev_info(&host->pdev->dev, "using %s for DMA transfers\n",
2412	dma_chan_name(host->dma.chan));
2413
2414	host->dma_conf.src_addr = host->mapbase + ATMCI_RDR;
2415	host->dma_conf.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2416	host->dma_conf.src_maxburst = 1;
2417	host->dma_conf.dst_addr = host->mapbase + ATMCI_TDR;
2418	host->dma_conf.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2419	host->dma_conf.dst_maxburst = 1;
2420	host->dma_conf.device_fc = false;
2421
2422	return 0;
2423	}
2424
2425	/*
2426	* HSMCI (High Speed MCI) module is not fully compatible with MCI module.
2427	* HSMCI provides DMA support and a new config register but no more supports
2428	* PDC.
2429	*/
2430	static void atmci_get_cap(struct atmel_mci *host)
2431	{
2432	unsigned int version;
2433
2434	version = atmci_get_version(host);
2435	dev_info(&host->pdev->dev,
2436	"version: 0x%x\n", version);
2437
2438	host->caps.has_dma_conf_reg = false;
2439	host->caps.has_pdc = true;
2440	host->caps.has_cfg_reg = false;
2441	host->caps.has_cstor_reg = false;
2442	host->caps.has_highspeed = false;
2443	host->caps.has_rwproof = false;
2444	host->caps.has_odd_clk_div = false;
2445	host->caps.has_bad_data_ordering = true;
2446	host->caps.need_reset_after_xfer = true;
2447	host->caps.need_blksz_mul_4 = true;
2448	host->caps.need_notbusy_for_read_ops = false;
2449
2450	/* keep only major version number */
2451	switch (version & 0xf00) {
2452	case 0x600:
2453	case 0x500:
2454	host->caps.has_odd_clk_div = true;
2455	fallthrough;
2456	case 0x400:
2457	case 0x300:
2458	host->caps.has_dma_conf_reg = true;
2459	host->caps.has_pdc = false;
2460	host->caps.has_cfg_reg = true;
2461	host->caps.has_cstor_reg = true;
2462	host->caps.has_highspeed = true;
2463	fallthrough;
2464	case 0x200:
2465	host->caps.has_rwproof = true;
2466	host->caps.need_blksz_mul_4 = false;
2467	host->caps.need_notbusy_for_read_ops = true;
2468	fallthrough;
2469	case 0x100:
2470	host->caps.has_bad_data_ordering = false;
2471	host->caps.need_reset_after_xfer = false;
2472	fallthrough;
2473	case 0x0:
2474	break;
2475	default:
2476	host->caps.has_pdc = false;
2477	dev_warn(&host->pdev->dev,
2478	"Unmanaged mci version, set minimum capabilities\n");
2479	break;
2480	}
2481	}
2482
2483	static int atmci_probe(struct platform_device *pdev)
2484	{
2485	struct mci_platform_data *pdata;
2486	struct atmel_mci *host;
2487	struct resource *regs;
2488	unsigned int nr_slots;
2489	int irq;
2490	int ret, i;
2491
2492	regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2493	if (!regs)
2494	return -ENXIO;
2495	pdata = pdev->dev.platform_data;
2496	if (!pdata) {
2497	pdata = atmci_of_init(pdev);
2498	if (IS_ERR(ptr: pdata)) {
2499	dev_err(&pdev->dev, "platform data not available\n");
2500	return PTR_ERR(ptr: pdata);
2501	}
2502	}
2503
2504	irq = platform_get_irq(pdev, 0);
2505	if (irq < 0)
2506	return irq;
2507
2508	host = devm_kzalloc(dev: &pdev->dev, size: sizeof(*host), GFP_KERNEL);
2509	if (!host)
2510	return -ENOMEM;
2511
2512	host->pdev = pdev;
2513	spin_lock_init(&host->lock);
2514	INIT_LIST_HEAD(list: &host->queue);
2515
2516	host->mck = devm_clk_get(dev: &pdev->dev, id: "mci_clk");
2517	if (IS_ERR(ptr: host->mck))
2518	return PTR_ERR(ptr: host->mck);
2519
2520	host->regs = devm_ioremap(dev: &pdev->dev, offset: regs->start, size: resource_size(res: regs));
2521	if (!host->regs)
2522	return -ENOMEM;
2523
2524	ret = clk_prepare_enable(clk: host->mck);
2525	if (ret)
2526	return ret;
2527
2528	atmci_writel(host, ATMCI_CR, ATMCI_CR_SWRST);
2529	host->bus_hz = clk_get_rate(clk: host->mck);
2530
2531	host->mapbase = regs->start;
2532
2533	tasklet_setup(t: &host->tasklet, callback: atmci_tasklet_func);
2534
2535	ret = request_irq(irq, handler: atmci_interrupt, flags: 0, name: dev_name(dev: &pdev->dev), dev: host);
2536	if (ret) {
2537	clk_disable_unprepare(clk: host->mck);
2538	return ret;
2539	}
2540
2541	/* Get MCI capabilities and set operations according to it */
2542	atmci_get_cap(host);
2543	ret = atmci_configure_dma(host);
2544	if (ret == -EPROBE_DEFER)
2545	goto err_dma_probe_defer;
2546	if (ret == 0) {
2547	host->prepare_data = &atmci_prepare_data_dma;
2548	host->submit_data = &atmci_submit_data_dma;
2549	host->stop_transfer = &atmci_stop_transfer_dma;
2550	} else if (host->caps.has_pdc) {
2551	dev_info(&pdev->dev, "using PDC\n");
2552	host->prepare_data = &atmci_prepare_data_pdc;
2553	host->submit_data = &atmci_submit_data_pdc;
2554	host->stop_transfer = &atmci_stop_transfer_pdc;
2555	} else {
2556	dev_info(&pdev->dev, "using PIO\n");
2557	host->prepare_data = &atmci_prepare_data;
2558	host->submit_data = &atmci_submit_data;
2559	host->stop_transfer = &atmci_stop_transfer;
2560	}
2561
2562	platform_set_drvdata(pdev, data: host);
2563
2564	timer_setup(&host->timer, atmci_timeout_timer, 0);
2565
2566	pm_runtime_get_noresume(dev: &pdev->dev);
2567	pm_runtime_set_active(dev: &pdev->dev);
2568	pm_runtime_set_autosuspend_delay(dev: &pdev->dev, AUTOSUSPEND_DELAY);
2569	pm_runtime_use_autosuspend(dev: &pdev->dev);
2570	pm_runtime_enable(dev: &pdev->dev);
2571
2572	/* We need at least one slot to succeed */
2573	nr_slots = 0;
2574	ret = -ENODEV;
2575	if (pdata->slot[0].bus_width) {
2576	ret = atmci_init_slot(host, slot_data: &pdata->slot[0],
2577	id: 0, ATMCI_SDCSEL_SLOT_A, ATMCI_SDIOIRQA);
2578	if (!ret) {
2579	nr_slots++;
2580	host->buf_size = host->slot[0]->mmc->max_req_size;
2581	}
2582	}
2583	if (pdata->slot[1].bus_width) {
2584	ret = atmci_init_slot(host, slot_data: &pdata->slot[1],
2585	id: 1, ATMCI_SDCSEL_SLOT_B, ATMCI_SDIOIRQB);
2586	if (!ret) {
2587	nr_slots++;
2588	if (host->slot[1]->mmc->max_req_size > host->buf_size)
2589	host->buf_size =
2590	host->slot[1]->mmc->max_req_size;
2591	}
2592	}
2593
2594	if (!nr_slots) {
2595	dev_err(&pdev->dev, "init failed: no slot defined\n");
2596	goto err_init_slot;
2597	}
2598
2599	if (!host->caps.has_rwproof) {
2600	host->buffer = dma_alloc_coherent(dev: &pdev->dev, size: host->buf_size,
2601	dma_handle: &host->buf_phys_addr,
2602	GFP_KERNEL);
2603	if (!host->buffer) {
2604	ret = -ENOMEM;
2605	dev_err(&pdev->dev, "buffer allocation failed\n");
2606	goto err_dma_alloc;
2607	}
2608	}
2609
2610	dev_info(&pdev->dev,
2611	"Atmel MCI controller at 0x%08lx irq %d, %u slots\n",
2612	host->mapbase, irq, nr_slots);
2613
2614	pm_runtime_mark_last_busy(dev: &host->pdev->dev);
2615	pm_runtime_put_autosuspend(dev: &pdev->dev);
2616
2617	return 0;
2618
2619	err_dma_alloc:
2620	for (i = 0; i < ATMCI_MAX_NR_SLOTS; i++) {
2621	if (host->slot[i])
2622	atmci_cleanup_slot(slot: host->slot[i], id: i);
2623	}
2624	err_init_slot:
2625	clk_disable_unprepare(clk: host->mck);
2626
2627	pm_runtime_disable(dev: &pdev->dev);
2628	pm_runtime_put_noidle(dev: &pdev->dev);
2629
2630	del_timer_sync(timer: &host->timer);
2631	if (!IS_ERR(ptr: host->dma.chan))
2632	dma_release_channel(chan: host->dma.chan);
2633	err_dma_probe_defer:
2634	free_irq(irq, host);
2635	return ret;
2636	}
2637
2638	static void atmci_remove(struct platform_device *pdev)
2639	{
2640	struct atmel_mci *host = platform_get_drvdata(pdev);
2641	unsigned int i;
2642
2643	pm_runtime_get_sync(dev: &pdev->dev);
2644
2645	if (host->buffer)
2646	dma_free_coherent(dev: &pdev->dev, size: host->buf_size,
2647	cpu_addr: host->buffer, dma_handle: host->buf_phys_addr);
2648
2649	for (i = 0; i < ATMCI_MAX_NR_SLOTS; i++) {
2650	if (host->slot[i])
2651	atmci_cleanup_slot(slot: host->slot[i], id: i);
2652	}
2653
2654	atmci_writel(host, ATMCI_IDR, ~0UL);
2655	atmci_writel(host, ATMCI_CR, ATMCI_CR_MCIDIS);
2656	atmci_readl(host, ATMCI_SR);
2657
2658	del_timer_sync(timer: &host->timer);
2659	if (!IS_ERR(ptr: host->dma.chan))
2660	dma_release_channel(chan: host->dma.chan);
2661
2662	free_irq(platform_get_irq(pdev, 0), host);
2663
2664	clk_disable_unprepare(clk: host->mck);
2665
2666	pm_runtime_disable(dev: &pdev->dev);
2667	pm_runtime_put_noidle(dev: &pdev->dev);
2668	}
2669
2670	#ifdef CONFIG_PM
2671	static int atmci_runtime_suspend(struct device *dev)
2672	{
2673	struct atmel_mci *host = dev_get_drvdata(dev);
2674
2675	clk_disable_unprepare(clk: host->mck);
2676
2677	pinctrl_pm_select_sleep_state(dev);
2678
2679	return 0;
2680	}
2681
2682	static int atmci_runtime_resume(struct device *dev)
2683	{
2684	struct atmel_mci *host = dev_get_drvdata(dev);
2685
2686	pinctrl_select_default_state(dev);
2687
2688	return clk_prepare_enable(clk: host->mck);
2689	}
2690	#endif
2691
2692	static const struct dev_pm_ops atmci_dev_pm_ops = {
2693	SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend,
2694	pm_runtime_force_resume)
2695	SET_RUNTIME_PM_OPS(atmci_runtime_suspend, atmci_runtime_resume, NULL)
2696	};
2697
2698	static struct platform_driver atmci_driver = {
2699	.probe = atmci_probe,
2700	.remove_new = atmci_remove,
2701	.driver = {
2702	.name = "atmel_mci",
2703	.probe_type = PROBE_PREFER_ASYNCHRONOUS,
2704	.of_match_table = of_match_ptr(atmci_dt_ids),
2705	.pm = &atmci_dev_pm_ops,
2706	},
2707	};
2708	module_platform_driver(atmci_driver);
2709
2710	MODULE_DESCRIPTION("Atmel Multimedia Card Interface driver");
2711	MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
2712	MODULE_LICENSE("GPL v2");
2713