1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Copyright (C) 2013 Boris BREZILLON <b.brezillon.dev@gmail.com>
4	*
5	* Derived from:
6	* https://github.com/yuq/sunxi-nfc-mtd
7	* Copyright (C) 2013 Qiang Yu <yuq825@gmail.com>
8	*
9	* https://github.com/hno/Allwinner-Info
10	* Copyright (C) 2013 Henrik Nordström <Henrik Nordström>
11	*
12	* Copyright (C) 2013 Dmitriy B. <rzk333@gmail.com>
13	* Copyright (C) 2013 Sergey Lapin <slapin@ossfans.org>
14	*/
15
16	#include <linux/dma-mapping.h>
17	#include <linux/slab.h>
18	#include <linux/module.h>
19	#include <linux/moduleparam.h>
20	#include <linux/platform_device.h>
21	#include <linux/of.h>
22	#include <linux/mtd/mtd.h>
23	#include <linux/mtd/rawnand.h>
24	#include <linux/mtd/partitions.h>
25	#include <linux/clk.h>
26	#include <linux/delay.h>
27	#include <linux/dmaengine.h>
28	#include <linux/interrupt.h>
29	#include <linux/iopoll.h>
30	#include <linux/reset.h>
31
32	#define NFC_REG_CTL 0x0000
33	#define NFC_REG_ST 0x0004
34	#define NFC_REG_INT 0x0008
35	#define NFC_REG_TIMING_CTL 0x000C
36	#define NFC_REG_TIMING_CFG 0x0010
37	#define NFC_REG_ADDR_LOW 0x0014
38	#define NFC_REG_ADDR_HIGH 0x0018
39	#define NFC_REG_SECTOR_NUM 0x001C
40	#define NFC_REG_CNT 0x0020
41	#define NFC_REG_CMD 0x0024
42	#define NFC_REG_RCMD_SET 0x0028
43	#define NFC_REG_WCMD_SET 0x002C
44	#define NFC_REG_A10_IO_DATA 0x0030
45	#define NFC_REG_A23_IO_DATA 0x0300
46	#define NFC_REG_ECC_CTL 0x0034
47	#define NFC_REG_ECC_ST 0x0038
48	#define NFC_REG_DEBUG 0x003C
49	#define NFC_REG_ECC_ERR_CNT(x) ((0x0040 + (x)) & ~0x3)
50	#define NFC_REG_USER_DATA(x) (0x0050 + ((x) * 4))
51	#define NFC_REG_SPARE_AREA 0x00A0
52	#define NFC_REG_PAT_ID 0x00A4
53	#define NFC_REG_MDMA_ADDR 0x00C0
54	#define NFC_REG_MDMA_CNT 0x00C4
55	#define NFC_RAM0_BASE 0x0400
56	#define NFC_RAM1_BASE 0x0800
57
58	/* define bit use in NFC_CTL */
59	#define NFC_EN BIT(0)
60	#define NFC_RESET BIT(1)
61	#define NFC_BUS_WIDTH_MSK BIT(2)
62	#define NFC_BUS_WIDTH_8 (0 << 2)
63	#define NFC_BUS_WIDTH_16 (1 << 2)
64	#define NFC_RB_SEL_MSK BIT(3)
65	#define NFC_RB_SEL(x) ((x) << 3)
66	#define NFC_CE_SEL_MSK GENMASK(26, 24)
67	#define NFC_CE_SEL(x) ((x) << 24)
68	#define NFC_CE_CTL BIT(6)
69	#define NFC_PAGE_SHIFT_MSK GENMASK(11, 8)
70	#define NFC_PAGE_SHIFT(x) (((x) < 10 ? 0 : (x) - 10) << 8)
71	#define NFC_SAM BIT(12)
72	#define NFC_RAM_METHOD BIT(14)
73	#define NFC_DMA_TYPE_NORMAL BIT(15)
74	#define NFC_DEBUG_CTL BIT(31)
75
76	/* define bit use in NFC_ST */
77	#define NFC_RB_B2R BIT(0)
78	#define NFC_CMD_INT_FLAG BIT(1)
79	#define NFC_DMA_INT_FLAG BIT(2)
80	#define NFC_CMD_FIFO_STATUS BIT(3)
81	#define NFC_STA BIT(4)
82	#define NFC_NATCH_INT_FLAG BIT(5)
83	#define NFC_RB_STATE(x) BIT(x + 8)
84
85	/* define bit use in NFC_INT */
86	#define NFC_B2R_INT_ENABLE BIT(0)
87	#define NFC_CMD_INT_ENABLE BIT(1)
88	#define NFC_DMA_INT_ENABLE BIT(2)
89	#define NFC_INT_MASK (NFC_B2R_INT_ENABLE | \
90	NFC_CMD_INT_ENABLE | \
91	NFC_DMA_INT_ENABLE)
92
93	/* define bit use in NFC_TIMING_CTL */
94	#define NFC_TIMING_CTL_EDO BIT(8)
95
96	/* define NFC_TIMING_CFG register layout */
97	#define NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD) \
98	(((tWB) & 0x3) | (((tADL) & 0x3) << 2) | \
99	(((tWHR) & 0x3) << 4) | (((tRHW) & 0x3) << 6) | \
100	(((tCAD) & 0x7) << 8))
101
102	/* define bit use in NFC_CMD */
103	#define NFC_CMD_LOW_BYTE_MSK GENMASK(7, 0)
104	#define NFC_CMD_HIGH_BYTE_MSK GENMASK(15, 8)
105	#define NFC_CMD(x) (x)
106	#define NFC_ADR_NUM_MSK GENMASK(18, 16)
107	#define NFC_ADR_NUM(x) (((x) - 1) << 16)
108	#define NFC_SEND_ADR BIT(19)
109	#define NFC_ACCESS_DIR BIT(20)
110	#define NFC_DATA_TRANS BIT(21)
111	#define NFC_SEND_CMD1 BIT(22)
112	#define NFC_WAIT_FLAG BIT(23)
113	#define NFC_SEND_CMD2 BIT(24)
114	#define NFC_SEQ BIT(25)
115	#define NFC_DATA_SWAP_METHOD BIT(26)
116	#define NFC_ROW_AUTO_INC BIT(27)
117	#define NFC_SEND_CMD3 BIT(28)
118	#define NFC_SEND_CMD4 BIT(29)
119	#define NFC_CMD_TYPE_MSK GENMASK(31, 30)
120	#define NFC_NORMAL_OP (0 << 30)
121	#define NFC_ECC_OP (1 << 30)
122	#define NFC_PAGE_OP (2U << 30)
123
124	/* define bit use in NFC_RCMD_SET */
125	#define NFC_READ_CMD_MSK GENMASK(7, 0)
126	#define NFC_RND_READ_CMD0_MSK GENMASK(15, 8)
127	#define NFC_RND_READ_CMD1_MSK GENMASK(23, 16)
128
129	/* define bit use in NFC_WCMD_SET */
130	#define NFC_PROGRAM_CMD_MSK GENMASK(7, 0)
131	#define NFC_RND_WRITE_CMD_MSK GENMASK(15, 8)
132	#define NFC_READ_CMD0_MSK GENMASK(23, 16)
133	#define NFC_READ_CMD1_MSK GENMASK(31, 24)
134
135	/* define bit use in NFC_ECC_CTL */
136	#define NFC_ECC_EN BIT(0)
137	#define NFC_ECC_PIPELINE BIT(3)
138	#define NFC_ECC_EXCEPTION BIT(4)
139	#define NFC_ECC_BLOCK_SIZE_MSK BIT(5)
140	#define NFC_ECC_BLOCK_512 BIT(5)
141	#define NFC_RANDOM_EN BIT(9)
142	#define NFC_RANDOM_DIRECTION BIT(10)
143	#define NFC_ECC_MODE_MSK GENMASK(15, 12)
144	#define NFC_ECC_MODE(x) ((x) << 12)
145	#define NFC_RANDOM_SEED_MSK GENMASK(30, 16)
146	#define NFC_RANDOM_SEED(x) ((x) << 16)
147
148	/* define bit use in NFC_ECC_ST */
149	#define NFC_ECC_ERR(x) BIT(x)
150	#define NFC_ECC_ERR_MSK GENMASK(15, 0)
151	#define NFC_ECC_PAT_FOUND(x) BIT(x + 16)
152	#define NFC_ECC_ERR_CNT(b, x) (((x) >> (((b) % 4) * 8)) & 0xff)
153
154	#define NFC_DEFAULT_TIMEOUT_MS 1000
155
156	#define NFC_SRAM_SIZE 1024
157
158	#define NFC_MAX_CS 7
159
160	/**
161	* struct sunxi_nand_chip_sel - stores information related to NAND Chip Select
162	*
163	* @cs: the NAND CS id used to communicate with a NAND Chip
164	* @rb: the Ready/Busy pin ID. -1 means no R/B pin connected to the NFC
165	*/
166	struct sunxi_nand_chip_sel {
167	u8 cs;
168	s8 rb;
169	};
170
171	/**
172	* struct sunxi_nand_hw_ecc - stores information related to HW ECC support
173	*
174	* @ecc_ctl: ECC_CTL register value for this NAND chip
175	*/
176	struct sunxi_nand_hw_ecc {
177	u32 ecc_ctl;
178	};
179
180	/**
181	* struct sunxi_nand_chip - stores NAND chip device related information
182	*
183	* @node: used to store NAND chips into a list
184	* @nand: base NAND chip structure
185	* @ecc: ECC controller structure
186	* @clk_rate: clk_rate required for this NAND chip
187	* @timing_cfg: TIMING_CFG register value for this NAND chip
188	* @timing_ctl: TIMING_CTL register value for this NAND chip
189	* @nsels: number of CS lines required by the NAND chip
190	* @sels: array of CS lines descriptions
191	*/
192	struct sunxi_nand_chip {
193	struct list_head node;
194	struct nand_chip nand;
195	struct sunxi_nand_hw_ecc ecc;
196	unsigned long clk_rate;
197	u32 timing_cfg;
198	u32 timing_ctl;
199	int nsels;
200	struct sunxi_nand_chip_sel sels[] __counted_by(nsels);
201	};
202
203	static inline struct sunxi_nand_chip *to_sunxi_nand(struct nand_chip *nand)
204	{
205	return container_of(nand, struct sunxi_nand_chip, nand);
206	}
207
208	/*
209	* NAND Controller capabilities structure: stores NAND controller capabilities
210	* for distinction between compatible strings.
211	*
212	* @has_mdma: Use mbus dma mode, otherwise general dma
213	* through MBUS on A23/A33 needs extra configuration.
214	* @reg_io_data: I/O data register
215	* @dma_maxburst: DMA maxburst
216	*/
217	struct sunxi_nfc_caps {
218	bool has_mdma;
219	unsigned int reg_io_data;
220	unsigned int dma_maxburst;
221	};
222
223	/**
224	* struct sunxi_nfc - stores sunxi NAND controller information
225	*
226	* @controller: base controller structure
227	* @dev: parent device (used to print error messages)
228	* @regs: NAND controller registers
229	* @ahb_clk: NAND controller AHB clock
230	* @mod_clk: NAND controller mod clock
231	* @reset: NAND controller reset line
232	* @assigned_cs: bitmask describing already assigned CS lines
233	* @clk_rate: NAND controller current clock rate
234	* @chips: a list containing all the NAND chips attached to this NAND
235	* controller
236	* @complete: a completion object used to wait for NAND controller events
237	* @dmac: the DMA channel attached to the NAND controller
238	* @caps: NAND Controller capabilities
239	*/
240	struct sunxi_nfc {
241	struct nand_controller controller;
242	struct device *dev;
243	void __iomem *regs;
244	struct clk *ahb_clk;
245	struct clk *mod_clk;
246	struct reset_control *reset;
247	unsigned long assigned_cs;
248	unsigned long clk_rate;
249	struct list_head chips;
250	struct completion complete;
251	struct dma_chan *dmac;
252	const struct sunxi_nfc_caps *caps;
253	};
254
255	static inline struct sunxi_nfc *to_sunxi_nfc(struct nand_controller *ctrl)
256	{
257	return container_of(ctrl, struct sunxi_nfc, controller);
258	}
259
260	static irqreturn_t sunxi_nfc_interrupt(int irq, void *dev_id)
261	{
262	struct sunxi_nfc *nfc = dev_id;
263	u32 st = readl(addr: nfc->regs + NFC_REG_ST);
264	u32 ien = readl(addr: nfc->regs + NFC_REG_INT);
265
266	if (!(ien & st))
267	return IRQ_NONE;
268
269	if ((ien & st) == ien)
270	complete(&nfc->complete);
271
272	writel(val: st & NFC_INT_MASK, addr: nfc->regs + NFC_REG_ST);
273	writel(val: ~st & ien & NFC_INT_MASK, addr: nfc->regs + NFC_REG_INT);
274
275	return IRQ_HANDLED;
276	}
277
278	static int sunxi_nfc_wait_events(struct sunxi_nfc *nfc, u32 events,
279	bool use_polling, unsigned int timeout_ms)
280	{
281	int ret;
282
283	if (events & ~NFC_INT_MASK)
284	return -EINVAL;
285
286	if (!timeout_ms)
287	timeout_ms = NFC_DEFAULT_TIMEOUT_MS;
288
289	if (!use_polling) {
290	init_completion(x: &nfc->complete);
291
292	writel(val: events, addr: nfc->regs + NFC_REG_INT);
293
294	ret = wait_for_completion_timeout(x: &nfc->complete,
295	timeout: msecs_to_jiffies(m: timeout_ms));
296	if (!ret)
297	ret = -ETIMEDOUT;
298	else
299	ret = 0;
300
301	writel(val: 0, addr: nfc->regs + NFC_REG_INT);
302	} else {
303	u32 status;
304
305	ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
306	(status & events) == events, 1,
307	timeout_ms * 1000);
308	}
309
310	writel(val: events & NFC_INT_MASK, addr: nfc->regs + NFC_REG_ST);
311
312	if (ret)
313	dev_err(nfc->dev, "wait interrupt timedout\n");
314
315	return ret;
316	}
317
318	static int sunxi_nfc_wait_cmd_fifo_empty(struct sunxi_nfc *nfc)
319	{
320	u32 status;
321	int ret;
322
323	ret = readl_poll_timeout(nfc->regs + NFC_REG_ST, status,
324	!(status & NFC_CMD_FIFO_STATUS), 1,
325	NFC_DEFAULT_TIMEOUT_MS * 1000);
326	if (ret)
327	dev_err(nfc->dev, "wait for empty cmd FIFO timedout\n");
328
329	return ret;
330	}
331
332	static int sunxi_nfc_rst(struct sunxi_nfc *nfc)
333	{
334	u32 ctl;
335	int ret;
336
337	writel(val: 0, addr: nfc->regs + NFC_REG_ECC_CTL);
338	writel(NFC_RESET, addr: nfc->regs + NFC_REG_CTL);
339
340	ret = readl_poll_timeout(nfc->regs + NFC_REG_CTL, ctl,
341	!(ctl & NFC_RESET), 1,
342	NFC_DEFAULT_TIMEOUT_MS * 1000);
343	if (ret)
344	dev_err(nfc->dev, "wait for NAND controller reset timedout\n");
345
346	return ret;
347	}
348
349	static int sunxi_nfc_dma_op_prepare(struct sunxi_nfc *nfc, const void *buf,
350	int chunksize, int nchunks,
351	enum dma_data_direction ddir,
352	struct scatterlist *sg)
353	{
354	struct dma_async_tx_descriptor *dmad;
355	enum dma_transfer_direction tdir;
356	dma_cookie_t dmat;
357	int ret;
358
359	if (ddir == DMA_FROM_DEVICE)
360	tdir = DMA_DEV_TO_MEM;
361	else
362	tdir = DMA_MEM_TO_DEV;
363
364	sg_init_one(sg, buf, nchunks * chunksize);
365	ret = dma_map_sg(nfc->dev, sg, 1, ddir);
366	if (!ret)
367	return -ENOMEM;
368
369	if (!nfc->caps->has_mdma) {
370	dmad = dmaengine_prep_slave_sg(chan: nfc->dmac, sgl: sg, sg_len: 1, dir: tdir, flags: DMA_CTRL_ACK);
371	if (!dmad) {
372	ret = -EINVAL;
373	goto err_unmap_buf;
374	}
375	}
376
377	writel(readl(addr: nfc->regs + NFC_REG_CTL) | NFC_RAM_METHOD,
378	addr: nfc->regs + NFC_REG_CTL);
379	writel(val: nchunks, addr: nfc->regs + NFC_REG_SECTOR_NUM);
380	writel(val: chunksize, addr: nfc->regs + NFC_REG_CNT);
381
382	if (nfc->caps->has_mdma) {
383	writel(readl(addr: nfc->regs + NFC_REG_CTL) & ~NFC_DMA_TYPE_NORMAL,
384	addr: nfc->regs + NFC_REG_CTL);
385	writel(val: chunksize * nchunks, addr: nfc->regs + NFC_REG_MDMA_CNT);
386	writel(sg_dma_address(sg), addr: nfc->regs + NFC_REG_MDMA_ADDR);
387	} else {
388	dmat = dmaengine_submit(desc: dmad);
389
390	ret = dma_submit_error(cookie: dmat);
391	if (ret)
392	goto err_clr_dma_flag;
393	}
394
395	return 0;
396
397	err_clr_dma_flag:
398	writel(readl(addr: nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
399	addr: nfc->regs + NFC_REG_CTL);
400
401	err_unmap_buf:
402	dma_unmap_sg(nfc->dev, sg, 1, ddir);
403	return ret;
404	}
405
406	static void sunxi_nfc_dma_op_cleanup(struct sunxi_nfc *nfc,
407	enum dma_data_direction ddir,
408	struct scatterlist *sg)
409	{
410	dma_unmap_sg(nfc->dev, sg, 1, ddir);
411	writel(readl(addr: nfc->regs + NFC_REG_CTL) & ~NFC_RAM_METHOD,
412	addr: nfc->regs + NFC_REG_CTL);
413	}
414
415	static void sunxi_nfc_select_chip(struct nand_chip *nand, unsigned int cs)
416	{
417	struct mtd_info *mtd = nand_to_mtd(chip: nand);
418	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
419	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: sunxi_nand->nand.controller);
420	struct sunxi_nand_chip_sel *sel;
421	u32 ctl;
422
423	if (cs >= sunxi_nand->nsels)
424	return;
425
426	ctl = readl(addr: nfc->regs + NFC_REG_CTL) &
427	~(NFC_PAGE_SHIFT_MSK | NFC_CE_SEL_MSK | NFC_RB_SEL_MSK | NFC_EN);
428
429	sel = &sunxi_nand->sels[cs];
430	ctl |= NFC_CE_SEL(sel->cs) | NFC_EN | NFC_PAGE_SHIFT(nand->page_shift);
431	if (sel->rb >= 0)
432	ctl |= NFC_RB_SEL(sel->rb);
433
434	writel(val: mtd->writesize, addr: nfc->regs + NFC_REG_SPARE_AREA);
435
436	if (nfc->clk_rate != sunxi_nand->clk_rate) {
437	clk_set_rate(clk: nfc->mod_clk, rate: sunxi_nand->clk_rate);
438	nfc->clk_rate = sunxi_nand->clk_rate;
439	}
440
441	writel(val: sunxi_nand->timing_ctl, addr: nfc->regs + NFC_REG_TIMING_CTL);
442	writel(val: sunxi_nand->timing_cfg, addr: nfc->regs + NFC_REG_TIMING_CFG);
443	writel(val: ctl, addr: nfc->regs + NFC_REG_CTL);
444	}
445
446	static void sunxi_nfc_read_buf(struct nand_chip *nand, uint8_t *buf, int len)
447	{
448	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
449	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: sunxi_nand->nand.controller);
450	int ret;
451	int cnt;
452	int offs = 0;
453	u32 tmp;
454
455	while (len > offs) {
456	bool poll = false;
457
458	cnt = min(len - offs, NFC_SRAM_SIZE);
459
460	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
461	if (ret)
462	break;
463
464	writel(val: cnt, addr: nfc->regs + NFC_REG_CNT);
465	tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
466	writel(val: tmp, addr: nfc->regs + NFC_REG_CMD);
467
468	/* Arbitrary limit for polling mode */
469	if (cnt < 64)
470	poll = true;
471
472	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, use_polling: poll, timeout_ms: 0);
473	if (ret)
474	break;
475
476	if (buf)
477	memcpy_fromio(buf + offs, nfc->regs + NFC_RAM0_BASE,
478	cnt);
479	offs += cnt;
480	}
481	}
482
483	static void sunxi_nfc_write_buf(struct nand_chip *nand, const uint8_t *buf,
484	int len)
485	{
486	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
487	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: sunxi_nand->nand.controller);
488	int ret;
489	int cnt;
490	int offs = 0;
491	u32 tmp;
492
493	while (len > offs) {
494	bool poll = false;
495
496	cnt = min(len - offs, NFC_SRAM_SIZE);
497
498	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
499	if (ret)
500	break;
501
502	writel(val: cnt, addr: nfc->regs + NFC_REG_CNT);
503	memcpy_toio(nfc->regs + NFC_RAM0_BASE, buf + offs, cnt);
504	tmp = NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
505	NFC_ACCESS_DIR;
506	writel(val: tmp, addr: nfc->regs + NFC_REG_CMD);
507
508	/* Arbitrary limit for polling mode */
509	if (cnt < 64)
510	poll = true;
511
512	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, use_polling: poll, timeout_ms: 0);
513	if (ret)
514	break;
515
516	offs += cnt;
517	}
518	}
519
520	/* These seed values have been extracted from Allwinner's BSP */
521	static const u16 sunxi_nfc_randomizer_page_seeds[] = {
522	0x2b75, 0x0bd0, 0x5ca3, 0x62d1, 0x1c93, 0x07e9, 0x2162, 0x3a72,
523	0x0d67, 0x67f9, 0x1be7, 0x077d, 0x032f, 0x0dac, 0x2716, 0x2436,
524	0x7922, 0x1510, 0x3860, 0x5287, 0x480f, 0x4252, 0x1789, 0x5a2d,
525	0x2a49, 0x5e10, 0x437f, 0x4b4e, 0x2f45, 0x216e, 0x5cb7, 0x7130,
526	0x2a3f, 0x60e4, 0x4dc9, 0x0ef0, 0x0f52, 0x1bb9, 0x6211, 0x7a56,
527	0x226d, 0x4ea7, 0x6f36, 0x3692, 0x38bf, 0x0c62, 0x05eb, 0x4c55,
528	0x60f4, 0x728c, 0x3b6f, 0x2037, 0x7f69, 0x0936, 0x651a, 0x4ceb,
529	0x6218, 0x79f3, 0x383f, 0x18d9, 0x4f05, 0x5c82, 0x2912, 0x6f17,
530	0x6856, 0x5938, 0x1007, 0x61ab, 0x3e7f, 0x57c2, 0x542f, 0x4f62,
531	0x7454, 0x2eac, 0x7739, 0x42d4, 0x2f90, 0x435a, 0x2e52, 0x2064,
532	0x637c, 0x66ad, 0x2c90, 0x0bad, 0x759c, 0x0029, 0x0986, 0x7126,
533	0x1ca7, 0x1605, 0x386a, 0x27f5, 0x1380, 0x6d75, 0x24c3, 0x0f8e,
534	0x2b7a, 0x1418, 0x1fd1, 0x7dc1, 0x2d8e, 0x43af, 0x2267, 0x7da3,
535	0x4e3d, 0x1338, 0x50db, 0x454d, 0x764d, 0x40a3, 0x42e6, 0x262b,
536	0x2d2e, 0x1aea, 0x2e17, 0x173d, 0x3a6e, 0x71bf, 0x25f9, 0x0a5d,
537	0x7c57, 0x0fbe, 0x46ce, 0x4939, 0x6b17, 0x37bb, 0x3e91, 0x76db,
538	};
539
540	/*
541	* sunxi_nfc_randomizer_ecc512_seeds and sunxi_nfc_randomizer_ecc1024_seeds
542	* have been generated using
543	* sunxi_nfc_randomizer_step(seed, (step_size * 8) + 15), which is what
544	* the randomizer engine does internally before de/scrambling OOB data.
545	*
546	* Those tables are statically defined to avoid calculating randomizer state
547	* at runtime.
548	*/
549	static const u16 sunxi_nfc_randomizer_ecc512_seeds[] = {
550	0x3346, 0x367f, 0x1f18, 0x769a, 0x4f64, 0x068c, 0x2ef1, 0x6b64,
551	0x28a9, 0x15d7, 0x30f8, 0x3659, 0x53db, 0x7c5f, 0x71d4, 0x4409,
552	0x26eb, 0x03cc, 0x655d, 0x47d4, 0x4daa, 0x0877, 0x712d, 0x3617,
553	0x3264, 0x49aa, 0x7f9e, 0x588e, 0x4fbc, 0x7176, 0x7f91, 0x6c6d,
554	0x4b95, 0x5fb7, 0x3844, 0x4037, 0x0184, 0x081b, 0x0ee8, 0x5b91,
555	0x293d, 0x1f71, 0x0e6f, 0x402b, 0x5122, 0x1e52, 0x22be, 0x3d2d,
556	0x75bc, 0x7c60, 0x6291, 0x1a2f, 0x61d4, 0x74aa, 0x4140, 0x29ab,
557	0x472d, 0x2852, 0x017e, 0x15e8, 0x5ec2, 0x17cf, 0x7d0f, 0x06b8,
558	0x117a, 0x6b94, 0x789b, 0x3126, 0x6ac5, 0x5be7, 0x150f, 0x51f8,
559	0x7889, 0x0aa5, 0x663d, 0x77e8, 0x0b87, 0x3dcb, 0x360d, 0x218b,
560	0x512f, 0x7dc9, 0x6a4d, 0x630a, 0x3547, 0x1dd2, 0x5aea, 0x69a5,
561	0x7bfa, 0x5e4f, 0x1519, 0x6430, 0x3a0e, 0x5eb3, 0x5425, 0x0c7a,
562	0x5540, 0x3670, 0x63c1, 0x31e9, 0x5a39, 0x2de7, 0x5979, 0x2891,
563	0x1562, 0x014b, 0x5b05, 0x2756, 0x5a34, 0x13aa, 0x6cb5, 0x2c36,
564	0x5e72, 0x1306, 0x0861, 0x15ef, 0x1ee8, 0x5a37, 0x7ac4, 0x45dd,
565	0x44c4, 0x7266, 0x2f41, 0x3ccc, 0x045e, 0x7d40, 0x7c66, 0x0fa0,
566	};
567
568	static const u16 sunxi_nfc_randomizer_ecc1024_seeds[] = {
569	0x2cf5, 0x35f1, 0x63a4, 0x5274, 0x2bd2, 0x778b, 0x7285, 0x32b6,
570	0x6a5c, 0x70d6, 0x757d, 0x6769, 0x5375, 0x1e81, 0x0cf3, 0x3982,
571	0x6787, 0x042a, 0x6c49, 0x1925, 0x56a8, 0x40a9, 0x063e, 0x7bd9,
572	0x4dbf, 0x55ec, 0x672e, 0x7334, 0x5185, 0x4d00, 0x232a, 0x7e07,
573	0x445d, 0x6b92, 0x528f, 0x4255, 0x53ba, 0x7d82, 0x2a2e, 0x3a4e,
574	0x75eb, 0x450c, 0x6844, 0x1b5d, 0x581a, 0x4cc6, 0x0379, 0x37b2,
575	0x419f, 0x0e92, 0x6b27, 0x5624, 0x01e3, 0x07c1, 0x44a5, 0x130c,
576	0x13e8, 0x5910, 0x0876, 0x60c5, 0x54e3, 0x5b7f, 0x2269, 0x509f,
577	0x7665, 0x36fd, 0x3e9a, 0x0579, 0x6295, 0x14ef, 0x0a81, 0x1bcc,
578	0x4b16, 0x64db, 0x0514, 0x4f07, 0x0591, 0x3576, 0x6853, 0x0d9e,
579	0x259f, 0x38b7, 0x64fb, 0x3094, 0x4693, 0x6ddd, 0x29bb, 0x0bc8,
580	0x3f47, 0x490e, 0x0c0e, 0x7933, 0x3c9e, 0x5840, 0x398d, 0x3e68,
581	0x4af1, 0x71f5, 0x57cf, 0x1121, 0x64eb, 0x3579, 0x15ac, 0x584d,
582	0x5f2a, 0x47e2, 0x6528, 0x6eac, 0x196e, 0x6b96, 0x0450, 0x0179,
583	0x609c, 0x06e1, 0x4626, 0x42c7, 0x273e, 0x486f, 0x0705, 0x1601,
584	0x145b, 0x407e, 0x062b, 0x57a5, 0x53f9, 0x5659, 0x4410, 0x3ccd,
585	};
586
587	static u16 sunxi_nfc_randomizer_step(u16 state, int count)
588	{
589	state &= 0x7fff;
590
591	/*
592	* This loop is just a simple implementation of a Fibonacci LFSR using
593	* the x16 + x15 + 1 polynomial.
594	*/
595	while (count--)
596	state = ((state >> 1) |
597	(((state ^ (state >> 1)) & 1) << 14)) & 0x7fff;
598
599	return state;
600	}
601
602	static u16 sunxi_nfc_randomizer_state(struct nand_chip *nand, int page,
603	bool ecc)
604	{
605	struct mtd_info *mtd = nand_to_mtd(chip: nand);
606	const u16 *seeds = sunxi_nfc_randomizer_page_seeds;
607	int mod = mtd_div_by_ws(sz: mtd->erasesize, mtd);
608
609	if (mod > ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds))
610	mod = ARRAY_SIZE(sunxi_nfc_randomizer_page_seeds);
611
612	if (ecc) {
613	if (mtd->ecc_step_size == 512)
614	seeds = sunxi_nfc_randomizer_ecc512_seeds;
615	else
616	seeds = sunxi_nfc_randomizer_ecc1024_seeds;
617	}
618
619	return seeds[page % mod];
620	}
621
622	static void sunxi_nfc_randomizer_config(struct nand_chip *nand, int page,
623	bool ecc)
624	{
625	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
626	u32 ecc_ctl = readl(addr: nfc->regs + NFC_REG_ECC_CTL);
627	u16 state;
628
629	if (!(nand->options & NAND_NEED_SCRAMBLING))
630	return;
631
632	ecc_ctl = readl(addr: nfc->regs + NFC_REG_ECC_CTL);
633	state = sunxi_nfc_randomizer_state(nand, page, ecc);
634	ecc_ctl = readl(addr: nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_SEED_MSK;
635	writel(val: ecc_ctl | NFC_RANDOM_SEED(state), addr: nfc->regs + NFC_REG_ECC_CTL);
636	}
637
638	static void sunxi_nfc_randomizer_enable(struct nand_chip *nand)
639	{
640	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
641
642	if (!(nand->options & NAND_NEED_SCRAMBLING))
643	return;
644
645	writel(readl(addr: nfc->regs + NFC_REG_ECC_CTL) | NFC_RANDOM_EN,
646	addr: nfc->regs + NFC_REG_ECC_CTL);
647	}
648
649	static void sunxi_nfc_randomizer_disable(struct nand_chip *nand)
650	{
651	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
652
653	if (!(nand->options & NAND_NEED_SCRAMBLING))
654	return;
655
656	writel(readl(addr: nfc->regs + NFC_REG_ECC_CTL) & ~NFC_RANDOM_EN,
657	addr: nfc->regs + NFC_REG_ECC_CTL);
658	}
659
660	static void sunxi_nfc_randomize_bbm(struct nand_chip *nand, int page, u8 *bbm)
661	{
662	u16 state = sunxi_nfc_randomizer_state(nand, page, ecc: true);
663
664	bbm[0] ^= state;
665	bbm[1] ^= sunxi_nfc_randomizer_step(state, count: 8);
666	}
667
668	static void sunxi_nfc_randomizer_write_buf(struct nand_chip *nand,
669	const uint8_t *buf, int len,
670	bool ecc, int page)
671	{
672	sunxi_nfc_randomizer_config(nand, page, ecc);
673	sunxi_nfc_randomizer_enable(nand);
674	sunxi_nfc_write_buf(nand, buf, len);
675	sunxi_nfc_randomizer_disable(nand);
676	}
677
678	static void sunxi_nfc_randomizer_read_buf(struct nand_chip *nand, uint8_t *buf,
679	int len, bool ecc, int page)
680	{
681	sunxi_nfc_randomizer_config(nand, page, ecc);
682	sunxi_nfc_randomizer_enable(nand);
683	sunxi_nfc_read_buf(nand, buf, len);
684	sunxi_nfc_randomizer_disable(nand);
685	}
686
687	static void sunxi_nfc_hw_ecc_enable(struct nand_chip *nand)
688	{
689	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
690	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
691
692	writel(val: sunxi_nand->ecc.ecc_ctl, addr: nfc->regs + NFC_REG_ECC_CTL);
693	}
694
695	static void sunxi_nfc_hw_ecc_disable(struct nand_chip *nand)
696	{
697	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
698
699	writel(val: 0, addr: nfc->regs + NFC_REG_ECC_CTL);
700	}
701
702	static inline void sunxi_nfc_user_data_to_buf(u32 user_data, u8 *buf)
703	{
704	buf[0] = user_data;
705	buf[1] = user_data >> 8;
706	buf[2] = user_data >> 16;
707	buf[3] = user_data >> 24;
708	}
709
710	static inline u32 sunxi_nfc_buf_to_user_data(const u8 *buf)
711	{
712	return buf[0] | (buf[1] << 8) | (buf[2] << 16) | (buf[3] << 24);
713	}
714
715	static void sunxi_nfc_hw_ecc_get_prot_oob_bytes(struct nand_chip *nand, u8 *oob,
716	int step, bool bbm, int page)
717	{
718	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
719
720	sunxi_nfc_user_data_to_buf(readl(addr: nfc->regs + NFC_REG_USER_DATA(step)),
721	buf: oob);
722
723	/* De-randomize the Bad Block Marker. */
724	if (bbm && (nand->options & NAND_NEED_SCRAMBLING))
725	sunxi_nfc_randomize_bbm(nand, page, bbm: oob);
726	}
727
728	static void sunxi_nfc_hw_ecc_set_prot_oob_bytes(struct nand_chip *nand,
729	const u8 *oob, int step,
730	bool bbm, int page)
731	{
732	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
733	u8 user_data[4];
734
735	/* Randomize the Bad Block Marker. */
736	if (bbm && (nand->options & NAND_NEED_SCRAMBLING)) {
737	memcpy(user_data, oob, sizeof(user_data));
738	sunxi_nfc_randomize_bbm(nand, page, bbm: user_data);
739	oob = user_data;
740	}
741
742	writel(val: sunxi_nfc_buf_to_user_data(buf: oob),
743	addr: nfc->regs + NFC_REG_USER_DATA(step));
744	}
745
746	static void sunxi_nfc_hw_ecc_update_stats(struct nand_chip *nand,
747	unsigned int *max_bitflips, int ret)
748	{
749	struct mtd_info *mtd = nand_to_mtd(chip: nand);
750
751	if (ret < 0) {
752	mtd->ecc_stats.failed++;
753	} else {
754	mtd->ecc_stats.corrected += ret;
755	*max_bitflips = max_t(unsigned int, *max_bitflips, ret);
756	}
757	}
758
759	static int sunxi_nfc_hw_ecc_correct(struct nand_chip *nand, u8 *data, u8 *oob,
760	int step, u32 status, bool *erased)
761	{
762	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
763	struct nand_ecc_ctrl *ecc = &nand->ecc;
764	u32 tmp;
765
766	*erased = false;
767
768	if (status & NFC_ECC_ERR(step))
769	return -EBADMSG;
770
771	if (status & NFC_ECC_PAT_FOUND(step)) {
772	u8 pattern;
773
774	if (unlikely(!(readl(nfc->regs + NFC_REG_PAT_ID) & 0x1))) {
775	pattern = 0x0;
776	} else {
777	pattern = 0xff;
778	*erased = true;
779	}
780
781	if (data)
782	memset(data, pattern, ecc->size);
783
784	if (oob)
785	memset(oob, pattern, ecc->bytes + 4);
786
787	return 0;
788	}
789
790	tmp = readl(addr: nfc->regs + NFC_REG_ECC_ERR_CNT(step));
791
792	return NFC_ECC_ERR_CNT(step, tmp);
793	}
794
795	static int sunxi_nfc_hw_ecc_read_chunk(struct nand_chip *nand,
796	u8 *data, int data_off,
797	u8 *oob, int oob_off,
798	int *cur_off,
799	unsigned int *max_bitflips,
800	bool bbm, bool oob_required, int page)
801	{
802	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
803	struct nand_ecc_ctrl *ecc = &nand->ecc;
804	int raw_mode = 0;
805	bool erased;
806	int ret;
807
808	if (*cur_off != data_off)
809	nand_change_read_column_op(chip: nand, offset_in_page: data_off, NULL, len: 0, force_8bit: false);
810
811	sunxi_nfc_randomizer_read_buf(nand, NULL, len: ecc->size, ecc: false, page);
812
813	if (data_off + ecc->size != oob_off)
814	nand_change_read_column_op(chip: nand, offset_in_page: oob_off, NULL, len: 0, force_8bit: false);
815
816	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
817	if (ret)
818	return ret;
819
820	sunxi_nfc_randomizer_enable(nand);
821	writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD | NFC_ECC_OP,
822	addr: nfc->regs + NFC_REG_CMD);
823
824	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, use_polling: false, timeout_ms: 0);
825	sunxi_nfc_randomizer_disable(nand);
826	if (ret)
827	return ret;
828
829	*cur_off = oob_off + ecc->bytes + 4;
830
831	ret = sunxi_nfc_hw_ecc_correct(nand, data, oob: oob_required ? oob : NULL, step: 0,
832	readl(addr: nfc->regs + NFC_REG_ECC_ST),
833	erased: &erased);
834	if (erased)
835	return 1;
836
837	if (ret < 0) {
838	/*
839	* Re-read the data with the randomizer disabled to identify
840	* bitflips in erased pages.
841	*/
842	if (nand->options & NAND_NEED_SCRAMBLING)
843	nand_change_read_column_op(chip: nand, offset_in_page: data_off, buf: data,
844	len: ecc->size, force_8bit: false);
845	else
846	memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE,
847	ecc->size);
848
849	nand_change_read_column_op(chip: nand, offset_in_page: oob_off, buf: oob, len: ecc->bytes + 4,
850	force_8bit: false);
851
852	ret = nand_check_erased_ecc_chunk(data, datalen: ecc->size,
853	ecc: oob, ecclen: ecc->bytes + 4,
854	NULL, extraooblen: 0, threshold: ecc->strength);
855	if (ret >= 0)
856	raw_mode = 1;
857	} else {
858	memcpy_fromio(data, nfc->regs + NFC_RAM0_BASE, ecc->size);
859
860	if (oob_required) {
861	nand_change_read_column_op(chip: nand, offset_in_page: oob_off, NULL, len: 0,
862	force_8bit: false);
863	sunxi_nfc_randomizer_read_buf(nand, buf: oob, len: ecc->bytes + 4,
864	ecc: true, page);
865
866	sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, step: 0,
867	bbm, page);
868	}
869	}
870
871	sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips, ret);
872
873	return raw_mode;
874	}
875
876	static void sunxi_nfc_hw_ecc_read_extra_oob(struct nand_chip *nand,
877	u8 *oob, int *cur_off,
878	bool randomize, int page)
879	{
880	struct mtd_info *mtd = nand_to_mtd(chip: nand);
881	struct nand_ecc_ctrl *ecc = &nand->ecc;
882	int offset = ((ecc->bytes + 4) * ecc->steps);
883	int len = mtd->oobsize - offset;
884
885	if (len <= 0)
886	return;
887
888	if (!cur_off || *cur_off != offset)
889	nand_change_read_column_op(chip: nand, offset_in_page: mtd->writesize, NULL, len: 0,
890	force_8bit: false);
891
892	if (!randomize)
893	sunxi_nfc_read_buf(nand, buf: oob + offset, len);
894	else
895	sunxi_nfc_randomizer_read_buf(nand, buf: oob + offset, len,
896	ecc: false, page);
897
898	if (cur_off)
899	*cur_off = mtd->oobsize + mtd->writesize;
900	}
901
902	static int sunxi_nfc_hw_ecc_read_chunks_dma(struct nand_chip *nand, uint8_t *buf,
903	int oob_required, int page,
904	int nchunks)
905	{
906	bool randomized = nand->options & NAND_NEED_SCRAMBLING;
907	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
908	struct mtd_info *mtd = nand_to_mtd(chip: nand);
909	struct nand_ecc_ctrl *ecc = &nand->ecc;
910	unsigned int max_bitflips = 0;
911	int ret, i, raw_mode = 0;
912	struct scatterlist sg;
913	u32 status, wait;
914
915	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
916	if (ret)
917	return ret;
918
919	ret = sunxi_nfc_dma_op_prepare(nfc, buf, chunksize: ecc->size, nchunks,
920	ddir: DMA_FROM_DEVICE, sg: &sg);
921	if (ret)
922	return ret;
923
924	sunxi_nfc_hw_ecc_enable(nand);
925	sunxi_nfc_randomizer_config(nand, page, ecc: false);
926	sunxi_nfc_randomizer_enable(nand);
927
928	writel(val: (NAND_CMD_RNDOUTSTART << 16) | (NAND_CMD_RNDOUT << 8) |
929	NAND_CMD_READSTART, addr: nfc->regs + NFC_REG_RCMD_SET);
930
931	wait = NFC_CMD_INT_FLAG;
932
933	if (nfc->caps->has_mdma)
934	wait |= NFC_DMA_INT_FLAG;
935	else
936	dma_async_issue_pending(chan: nfc->dmac);
937
938	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD | NFC_DATA_TRANS,
939	addr: nfc->regs + NFC_REG_CMD);
940
941	ret = sunxi_nfc_wait_events(nfc, events: wait, use_polling: false, timeout_ms: 0);
942	if (ret && !nfc->caps->has_mdma)
943	dmaengine_terminate_all(chan: nfc->dmac);
944
945	sunxi_nfc_randomizer_disable(nand);
946	sunxi_nfc_hw_ecc_disable(nand);
947
948	sunxi_nfc_dma_op_cleanup(nfc, ddir: DMA_FROM_DEVICE, sg: &sg);
949
950	if (ret)
951	return ret;
952
953	status = readl(addr: nfc->regs + NFC_REG_ECC_ST);
954
955	for (i = 0; i < nchunks; i++) {
956	int data_off = i * ecc->size;
957	int oob_off = i * (ecc->bytes + 4);
958	u8 *data = buf + data_off;
959	u8 *oob = nand->oob_poi + oob_off;
960	bool erased;
961
962	ret = sunxi_nfc_hw_ecc_correct(nand, data: randomized ? data : NULL,
963	oob: oob_required ? oob : NULL,
964	step: i, status, erased: &erased);
965
966	/* ECC errors are handled in the second loop. */
967	if (ret < 0)
968	continue;
969
970	if (oob_required && !erased) {
971	/* TODO: use DMA to retrieve OOB */
972	nand_change_read_column_op(chip: nand,
973	offset_in_page: mtd->writesize + oob_off,
974	buf: oob, len: ecc->bytes + 4, force_8bit: false);
975
976	sunxi_nfc_hw_ecc_get_prot_oob_bytes(nand, oob, step: i,
977	bbm: !i, page);
978	}
979
980	if (erased)
981	raw_mode = 1;
982
983	sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips: &max_bitflips, ret);
984	}
985
986	if (status & NFC_ECC_ERR_MSK) {
987	for (i = 0; i < nchunks; i++) {
988	int data_off = i * ecc->size;
989	int oob_off = i * (ecc->bytes + 4);
990	u8 *data = buf + data_off;
991	u8 *oob = nand->oob_poi + oob_off;
992
993	if (!(status & NFC_ECC_ERR(i)))
994	continue;
995
996	/*
997	* Re-read the data with the randomizer disabled to
998	* identify bitflips in erased pages.
999	* TODO: use DMA to read page in raw mode
1000	*/
1001	if (randomized)
1002	nand_change_read_column_op(chip: nand, offset_in_page: data_off,
1003	buf: data, len: ecc->size,
1004	force_8bit: false);
1005
1006	/* TODO: use DMA to retrieve OOB */
1007	nand_change_read_column_op(chip: nand,
1008	offset_in_page: mtd->writesize + oob_off,
1009	buf: oob, len: ecc->bytes + 4, force_8bit: false);
1010
1011	ret = nand_check_erased_ecc_chunk(data, datalen: ecc->size,
1012	ecc: oob, ecclen: ecc->bytes + 4,
1013	NULL, extraooblen: 0,
1014	threshold: ecc->strength);
1015	if (ret >= 0)
1016	raw_mode = 1;
1017
1018	sunxi_nfc_hw_ecc_update_stats(nand, max_bitflips: &max_bitflips, ret);
1019	}
1020	}
1021
1022	if (oob_required)
1023	sunxi_nfc_hw_ecc_read_extra_oob(nand, oob: nand->oob_poi,
1024	NULL, randomize: !raw_mode,
1025	page);
1026
1027	return max_bitflips;
1028	}
1029
1030	static int sunxi_nfc_hw_ecc_write_chunk(struct nand_chip *nand,
1031	const u8 *data, int data_off,
1032	const u8 *oob, int oob_off,
1033	int *cur_off, bool bbm,
1034	int page)
1035	{
1036	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
1037	struct nand_ecc_ctrl *ecc = &nand->ecc;
1038	int ret;
1039
1040	if (data_off != *cur_off)
1041	nand_change_write_column_op(chip: nand, offset_in_page: data_off, NULL, len: 0, force_8bit: false);
1042
1043	sunxi_nfc_randomizer_write_buf(nand, buf: data, len: ecc->size, ecc: false, page);
1044
1045	if (data_off + ecc->size != oob_off)
1046	nand_change_write_column_op(chip: nand, offset_in_page: oob_off, NULL, len: 0, force_8bit: false);
1047
1048	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1049	if (ret)
1050	return ret;
1051
1052	sunxi_nfc_randomizer_enable(nand);
1053	sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, step: 0, bbm, page);
1054
1055	writel(NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD |
1056	NFC_ACCESS_DIR | NFC_ECC_OP,
1057	addr: nfc->regs + NFC_REG_CMD);
1058
1059	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG, use_polling: false, timeout_ms: 0);
1060	sunxi_nfc_randomizer_disable(nand);
1061	if (ret)
1062	return ret;
1063
1064	*cur_off = oob_off + ecc->bytes + 4;
1065
1066	return 0;
1067	}
1068
1069	static void sunxi_nfc_hw_ecc_write_extra_oob(struct nand_chip *nand,
1070	u8 *oob, int *cur_off,
1071	int page)
1072	{
1073	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1074	struct nand_ecc_ctrl *ecc = &nand->ecc;
1075	int offset = ((ecc->bytes + 4) * ecc->steps);
1076	int len = mtd->oobsize - offset;
1077
1078	if (len <= 0)
1079	return;
1080
1081	if (!cur_off || *cur_off != offset)
1082	nand_change_write_column_op(chip: nand, offset_in_page: offset + mtd->writesize,
1083	NULL, len: 0, force_8bit: false);
1084
1085	sunxi_nfc_randomizer_write_buf(nand, buf: oob + offset, len, ecc: false, page);
1086
1087	if (cur_off)
1088	*cur_off = mtd->oobsize + mtd->writesize;
1089	}
1090
1091	static int sunxi_nfc_hw_ecc_read_page(struct nand_chip *nand, uint8_t *buf,
1092	int oob_required, int page)
1093	{
1094	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1095	struct nand_ecc_ctrl *ecc = &nand->ecc;
1096	unsigned int max_bitflips = 0;
1097	int ret, i, cur_off = 0;
1098	bool raw_mode = false;
1099
1100	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1101
1102	nand_read_page_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1103
1104	sunxi_nfc_hw_ecc_enable(nand);
1105
1106	for (i = 0; i < ecc->steps; i++) {
1107	int data_off = i * ecc->size;
1108	int oob_off = i * (ecc->bytes + 4);
1109	u8 *data = buf + data_off;
1110	u8 *oob = nand->oob_poi + oob_off;
1111
1112	ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off, oob,
1113	oob_off: oob_off + mtd->writesize,
1114	cur_off: &cur_off, max_bitflips: &max_bitflips,
1115	bbm: !i, oob_required, page);
1116	if (ret < 0)
1117	return ret;
1118	else if (ret)
1119	raw_mode = true;
1120	}
1121
1122	if (oob_required)
1123	sunxi_nfc_hw_ecc_read_extra_oob(nand, oob: nand->oob_poi, cur_off: &cur_off,
1124	randomize: !raw_mode, page);
1125
1126	sunxi_nfc_hw_ecc_disable(nand);
1127
1128	return max_bitflips;
1129	}
1130
1131	static int sunxi_nfc_hw_ecc_read_page_dma(struct nand_chip *nand, u8 *buf,
1132	int oob_required, int page)
1133	{
1134	int ret;
1135
1136	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1137
1138	nand_read_page_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1139
1140	ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required, page,
1141	nchunks: nand->ecc.steps);
1142	if (ret >= 0)
1143	return ret;
1144
1145	/* Fallback to PIO mode */
1146	return sunxi_nfc_hw_ecc_read_page(nand, buf, oob_required, page);
1147	}
1148
1149	static int sunxi_nfc_hw_ecc_read_subpage(struct nand_chip *nand,
1150	u32 data_offs, u32 readlen,
1151	u8 *bufpoi, int page)
1152	{
1153	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1154	struct nand_ecc_ctrl *ecc = &nand->ecc;
1155	int ret, i, cur_off = 0;
1156	unsigned int max_bitflips = 0;
1157
1158	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1159
1160	nand_read_page_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1161
1162	sunxi_nfc_hw_ecc_enable(nand);
1163
1164	for (i = data_offs / ecc->size;
1165	i < DIV_ROUND_UP(data_offs + readlen, ecc->size); i++) {
1166	int data_off = i * ecc->size;
1167	int oob_off = i * (ecc->bytes + 4);
1168	u8 *data = bufpoi + data_off;
1169	u8 *oob = nand->oob_poi + oob_off;
1170
1171	ret = sunxi_nfc_hw_ecc_read_chunk(nand, data, data_off,
1172	oob,
1173	oob_off: oob_off + mtd->writesize,
1174	cur_off: &cur_off, max_bitflips: &max_bitflips, bbm: !i,
1175	oob_required: false, page);
1176	if (ret < 0)
1177	return ret;
1178	}
1179
1180	sunxi_nfc_hw_ecc_disable(nand);
1181
1182	return max_bitflips;
1183	}
1184
1185	static int sunxi_nfc_hw_ecc_read_subpage_dma(struct nand_chip *nand,
1186	u32 data_offs, u32 readlen,
1187	u8 *buf, int page)
1188	{
1189	int nchunks = DIV_ROUND_UP(data_offs + readlen, nand->ecc.size);
1190	int ret;
1191
1192	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1193
1194	nand_read_page_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1195
1196	ret = sunxi_nfc_hw_ecc_read_chunks_dma(nand, buf, oob_required: false, page, nchunks);
1197	if (ret >= 0)
1198	return ret;
1199
1200	/* Fallback to PIO mode */
1201	return sunxi_nfc_hw_ecc_read_subpage(nand, data_offs, readlen,
1202	bufpoi: buf, page);
1203	}
1204
1205	static int sunxi_nfc_hw_ecc_write_page(struct nand_chip *nand,
1206	const uint8_t *buf, int oob_required,
1207	int page)
1208	{
1209	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1210	struct nand_ecc_ctrl *ecc = &nand->ecc;
1211	int ret, i, cur_off = 0;
1212
1213	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1214
1215	nand_prog_page_begin_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1216
1217	sunxi_nfc_hw_ecc_enable(nand);
1218
1219	for (i = 0; i < ecc->steps; i++) {
1220	int data_off = i * ecc->size;
1221	int oob_off = i * (ecc->bytes + 4);
1222	const u8 *data = buf + data_off;
1223	const u8 *oob = nand->oob_poi + oob_off;
1224
1225	ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1226	oob_off: oob_off + mtd->writesize,
1227	cur_off: &cur_off, bbm: !i, page);
1228	if (ret)
1229	return ret;
1230	}
1231
1232	if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1233	sunxi_nfc_hw_ecc_write_extra_oob(nand, oob: nand->oob_poi,
1234	cur_off: &cur_off, page);
1235
1236	sunxi_nfc_hw_ecc_disable(nand);
1237
1238	return nand_prog_page_end_op(chip: nand);
1239	}
1240
1241	static int sunxi_nfc_hw_ecc_write_subpage(struct nand_chip *nand,
1242	u32 data_offs, u32 data_len,
1243	const u8 *buf, int oob_required,
1244	int page)
1245	{
1246	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1247	struct nand_ecc_ctrl *ecc = &nand->ecc;
1248	int ret, i, cur_off = 0;
1249
1250	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1251
1252	nand_prog_page_begin_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1253
1254	sunxi_nfc_hw_ecc_enable(nand);
1255
1256	for (i = data_offs / ecc->size;
1257	i < DIV_ROUND_UP(data_offs + data_len, ecc->size); i++) {
1258	int data_off = i * ecc->size;
1259	int oob_off = i * (ecc->bytes + 4);
1260	const u8 *data = buf + data_off;
1261	const u8 *oob = nand->oob_poi + oob_off;
1262
1263	ret = sunxi_nfc_hw_ecc_write_chunk(nand, data, data_off, oob,
1264	oob_off: oob_off + mtd->writesize,
1265	cur_off: &cur_off, bbm: !i, page);
1266	if (ret)
1267	return ret;
1268	}
1269
1270	sunxi_nfc_hw_ecc_disable(nand);
1271
1272	return nand_prog_page_end_op(chip: nand);
1273	}
1274
1275	static int sunxi_nfc_hw_ecc_write_page_dma(struct nand_chip *nand,
1276	const u8 *buf,
1277	int oob_required,
1278	int page)
1279	{
1280	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
1281	struct nand_ecc_ctrl *ecc = &nand->ecc;
1282	struct scatterlist sg;
1283	u32 wait;
1284	int ret, i;
1285
1286	sunxi_nfc_select_chip(nand, cs: nand->cur_cs);
1287
1288	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1289	if (ret)
1290	return ret;
1291
1292	ret = sunxi_nfc_dma_op_prepare(nfc, buf, chunksize: ecc->size, nchunks: ecc->steps,
1293	ddir: DMA_TO_DEVICE, sg: &sg);
1294	if (ret)
1295	goto pio_fallback;
1296
1297	for (i = 0; i < ecc->steps; i++) {
1298	const u8 *oob = nand->oob_poi + (i * (ecc->bytes + 4));
1299
1300	sunxi_nfc_hw_ecc_set_prot_oob_bytes(nand, oob, step: i, bbm: !i, page);
1301	}
1302
1303	nand_prog_page_begin_op(chip: nand, page, offset_in_page: 0, NULL, len: 0);
1304
1305	sunxi_nfc_hw_ecc_enable(nand);
1306	sunxi_nfc_randomizer_config(nand, page, ecc: false);
1307	sunxi_nfc_randomizer_enable(nand);
1308
1309	writel(val: (NAND_CMD_RNDIN << 8) | NAND_CMD_PAGEPROG,
1310	addr: nfc->regs + NFC_REG_WCMD_SET);
1311
1312	wait = NFC_CMD_INT_FLAG;
1313
1314	if (nfc->caps->has_mdma)
1315	wait |= NFC_DMA_INT_FLAG;
1316	else
1317	dma_async_issue_pending(chan: nfc->dmac);
1318
1319	writel(NFC_PAGE_OP | NFC_DATA_SWAP_METHOD |
1320	NFC_DATA_TRANS | NFC_ACCESS_DIR,
1321	addr: nfc->regs + NFC_REG_CMD);
1322
1323	ret = sunxi_nfc_wait_events(nfc, events: wait, use_polling: false, timeout_ms: 0);
1324	if (ret && !nfc->caps->has_mdma)
1325	dmaengine_terminate_all(chan: nfc->dmac);
1326
1327	sunxi_nfc_randomizer_disable(nand);
1328	sunxi_nfc_hw_ecc_disable(nand);
1329
1330	sunxi_nfc_dma_op_cleanup(nfc, ddir: DMA_TO_DEVICE, sg: &sg);
1331
1332	if (ret)
1333	return ret;
1334
1335	if (oob_required || (nand->options & NAND_NEED_SCRAMBLING))
1336	/* TODO: use DMA to transfer extra OOB bytes ? */
1337	sunxi_nfc_hw_ecc_write_extra_oob(nand, oob: nand->oob_poi,
1338	NULL, page);
1339
1340	return nand_prog_page_end_op(chip: nand);
1341
1342	pio_fallback:
1343	return sunxi_nfc_hw_ecc_write_page(nand, buf, oob_required, page);
1344	}
1345
1346	static int sunxi_nfc_hw_ecc_read_oob(struct nand_chip *nand, int page)
1347	{
1348	u8 *buf = nand_get_data_buf(chip: nand);
1349
1350	return nand->ecc.read_page(nand, buf, 1, page);
1351	}
1352
1353	static int sunxi_nfc_hw_ecc_write_oob(struct nand_chip *nand, int page)
1354	{
1355	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1356	u8 *buf = nand_get_data_buf(chip: nand);
1357	int ret;
1358
1359	memset(buf, 0xff, mtd->writesize);
1360	ret = nand->ecc.write_page(nand, buf, 1, page);
1361	if (ret)
1362	return ret;
1363
1364	/* Send command to program the OOB data */
1365	return nand_prog_page_end_op(chip: nand);
1366	}
1367
1368	static const s32 tWB_lut[] = {6, 12, 16, 20};
1369	static const s32 tRHW_lut[] = {4, 8, 12, 20};
1370
1371	static int _sunxi_nand_lookup_timing(const s32 *lut, int lut_size, u32 duration,
1372	u32 clk_period)
1373	{
1374	u32 clk_cycles = DIV_ROUND_UP(duration, clk_period);
1375	int i;
1376
1377	for (i = 0; i < lut_size; i++) {
1378	if (clk_cycles <= lut[i])
1379	return i;
1380	}
1381
1382	/* Doesn't fit */
1383	return -EINVAL;
1384	}
1385
1386	#define sunxi_nand_lookup_timing(l, p, c) \
1387	_sunxi_nand_lookup_timing(l, ARRAY_SIZE(l), p, c)
1388
1389	static int sunxi_nfc_setup_interface(struct nand_chip *nand, int csline,
1390	const struct nand_interface_config *conf)
1391	{
1392	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1393	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: sunxi_nand->nand.controller);
1394	const struct nand_sdr_timings *timings;
1395	u32 min_clk_period = 0;
1396	s32 tWB, tADL, tWHR, tRHW, tCAD;
1397	long real_clk_rate;
1398
1399	timings = nand_get_sdr_timings(conf);
1400	if (IS_ERR(ptr: timings))
1401	return -ENOTSUPP;
1402
1403	/* T1 <=> tCLS */
1404	if (timings->tCLS_min > min_clk_period)
1405	min_clk_period = timings->tCLS_min;
1406
1407	/* T2 <=> tCLH */
1408	if (timings->tCLH_min > min_clk_period)
1409	min_clk_period = timings->tCLH_min;
1410
1411	/* T3 <=> tCS */
1412	if (timings->tCS_min > min_clk_period)
1413	min_clk_period = timings->tCS_min;
1414
1415	/* T4 <=> tCH */
1416	if (timings->tCH_min > min_clk_period)
1417	min_clk_period = timings->tCH_min;
1418
1419	/* T5 <=> tWP */
1420	if (timings->tWP_min > min_clk_period)
1421	min_clk_period = timings->tWP_min;
1422
1423	/* T6 <=> tWH */
1424	if (timings->tWH_min > min_clk_period)
1425	min_clk_period = timings->tWH_min;
1426
1427	/* T7 <=> tALS */
1428	if (timings->tALS_min > min_clk_period)
1429	min_clk_period = timings->tALS_min;
1430
1431	/* T8 <=> tDS */
1432	if (timings->tDS_min > min_clk_period)
1433	min_clk_period = timings->tDS_min;
1434
1435	/* T9 <=> tDH */
1436	if (timings->tDH_min > min_clk_period)
1437	min_clk_period = timings->tDH_min;
1438
1439	/* T10 <=> tRR */
1440	if (timings->tRR_min > (min_clk_period * 3))
1441	min_clk_period = DIV_ROUND_UP(timings->tRR_min, 3);
1442
1443	/* T11 <=> tALH */
1444	if (timings->tALH_min > min_clk_period)
1445	min_clk_period = timings->tALH_min;
1446
1447	/* T12 <=> tRP */
1448	if (timings->tRP_min > min_clk_period)
1449	min_clk_period = timings->tRP_min;
1450
1451	/* T13 <=> tREH */
1452	if (timings->tREH_min > min_clk_period)
1453	min_clk_period = timings->tREH_min;
1454
1455	/* T14 <=> tRC */
1456	if (timings->tRC_min > (min_clk_period * 2))
1457	min_clk_period = DIV_ROUND_UP(timings->tRC_min, 2);
1458
1459	/* T15 <=> tWC */
1460	if (timings->tWC_min > (min_clk_period * 2))
1461	min_clk_period = DIV_ROUND_UP(timings->tWC_min, 2);
1462
1463	/* T16 - T19 + tCAD */
1464	if (timings->tWB_max > (min_clk_period * 20))
1465	min_clk_period = DIV_ROUND_UP(timings->tWB_max, 20);
1466
1467	if (timings->tADL_min > (min_clk_period * 32))
1468	min_clk_period = DIV_ROUND_UP(timings->tADL_min, 32);
1469
1470	if (timings->tWHR_min > (min_clk_period * 32))
1471	min_clk_period = DIV_ROUND_UP(timings->tWHR_min, 32);
1472
1473	if (timings->tRHW_min > (min_clk_period * 20))
1474	min_clk_period = DIV_ROUND_UP(timings->tRHW_min, 20);
1475
1476	/*
1477	* In non-EDO, tREA should be less than tRP to guarantee that the
1478	* controller does not sample the IO lines too early. Unfortunately,
1479	* the sunxi NAND controller does not allow us to have different
1480	* values for tRP and tREH (tRP = tREH = tRW / 2).
1481	*
1482	* We have 2 options to overcome this limitation:
1483	*
1484	* 1/ Extend tRC to fulfil the tREA <= tRC / 2 constraint
1485	* 2/ Use EDO mode (only works if timings->tRLOH > 0)
1486	*/
1487	if (timings->tREA_max > min_clk_period && !timings->tRLOH_min)
1488	min_clk_period = timings->tREA_max;
1489
1490	tWB = sunxi_nand_lookup_timing(tWB_lut, timings->tWB_max,
1491	min_clk_period);
1492	if (tWB < 0) {
1493	dev_err(nfc->dev, "unsupported tWB\n");
1494	return tWB;
1495	}
1496
1497	tADL = DIV_ROUND_UP(timings->tADL_min, min_clk_period) >> 3;
1498	if (tADL > 3) {
1499	dev_err(nfc->dev, "unsupported tADL\n");
1500	return -EINVAL;
1501	}
1502
1503	tWHR = DIV_ROUND_UP(timings->tWHR_min, min_clk_period) >> 3;
1504	if (tWHR > 3) {
1505	dev_err(nfc->dev, "unsupported tWHR\n");
1506	return -EINVAL;
1507	}
1508
1509	tRHW = sunxi_nand_lookup_timing(tRHW_lut, timings->tRHW_min,
1510	min_clk_period);
1511	if (tRHW < 0) {
1512	dev_err(nfc->dev, "unsupported tRHW\n");
1513	return tRHW;
1514	}
1515
1516	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1517	return 0;
1518
1519	/*
1520	* TODO: according to ONFI specs this value only applies for DDR NAND,
1521	* but Allwinner seems to set this to 0x7. Mimic them for now.
1522	*/
1523	tCAD = 0x7;
1524
1525	/* TODO: A83 has some more bits for CDQSS, CS, CLHZ, CCS, WC */
1526	sunxi_nand->timing_cfg = NFC_TIMING_CFG(tWB, tADL, tWHR, tRHW, tCAD);
1527
1528	/* Convert min_clk_period from picoseconds to nanoseconds */
1529	min_clk_period = DIV_ROUND_UP(min_clk_period, 1000);
1530
1531	/*
1532	* Unlike what is stated in Allwinner datasheet, the clk_rate should
1533	* be set to (1 / min_clk_period), and not (2 / min_clk_period).
1534	* This new formula was verified with a scope and validated by
1535	* Allwinner engineers.
1536	*/
1537	sunxi_nand->clk_rate = NSEC_PER_SEC / min_clk_period;
1538	real_clk_rate = clk_round_rate(clk: nfc->mod_clk, rate: sunxi_nand->clk_rate);
1539	if (real_clk_rate <= 0) {
1540	dev_err(nfc->dev, "Unable to round clk %lu\n",
1541	sunxi_nand->clk_rate);
1542	return -EINVAL;
1543	}
1544
1545	sunxi_nand->timing_ctl = 0;
1546
1547	/*
1548	* ONFI specification 3.1, paragraph 4.15.2 dictates that EDO data
1549	* output cycle timings shall be used if the host drives tRC less than
1550	* 30 ns. We should also use EDO mode if tREA is bigger than tRP.
1551	*/
1552	min_clk_period = NSEC_PER_SEC / real_clk_rate;
1553	if (min_clk_period * 2 < 30 || min_clk_period * 1000 < timings->tREA_max)
1554	sunxi_nand->timing_ctl = NFC_TIMING_CTL_EDO;
1555
1556	return 0;
1557	}
1558
1559	static int sunxi_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
1560	struct mtd_oob_region *oobregion)
1561	{
1562	struct nand_chip *nand = mtd_to_nand(mtd);
1563	struct nand_ecc_ctrl *ecc = &nand->ecc;
1564
1565	if (section >= ecc->steps)
1566	return -ERANGE;
1567
1568	oobregion->offset = section * (ecc->bytes + 4) + 4;
1569	oobregion->length = ecc->bytes;
1570
1571	return 0;
1572	}
1573
1574	static int sunxi_nand_ooblayout_free(struct mtd_info *mtd, int section,
1575	struct mtd_oob_region *oobregion)
1576	{
1577	struct nand_chip *nand = mtd_to_nand(mtd);
1578	struct nand_ecc_ctrl *ecc = &nand->ecc;
1579
1580	if (section > ecc->steps)
1581	return -ERANGE;
1582
1583	/*
1584	* The first 2 bytes are used for BB markers, hence we
1585	* only have 2 bytes available in the first user data
1586	* section.
1587	*/
1588	if (!section && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
1589	oobregion->offset = 2;
1590	oobregion->length = 2;
1591
1592	return 0;
1593	}
1594
1595	/*
1596	* The controller does not provide access to OOB bytes
1597	* past the end of the ECC data.
1598	*/
1599	if (section == ecc->steps && ecc->engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST)
1600	return -ERANGE;
1601
1602	oobregion->offset = section * (ecc->bytes + 4);
1603
1604	if (section < ecc->steps)
1605	oobregion->length = 4;
1606	else
1607	oobregion->length = mtd->oobsize - oobregion->offset;
1608
1609	return 0;
1610	}
1611
1612	static const struct mtd_ooblayout_ops sunxi_nand_ooblayout_ops = {
1613	.ecc = sunxi_nand_ooblayout_ecc,
1614	.free = sunxi_nand_ooblayout_free,
1615	};
1616
1617	static int sunxi_nand_hw_ecc_ctrl_init(struct nand_chip *nand,
1618	struct nand_ecc_ctrl *ecc,
1619	struct device_node *np)
1620	{
1621	static const u8 strengths[] = { 16, 24, 28, 32, 40, 48, 56, 60, 64 };
1622	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1623	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
1624	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1625	struct nand_device *nanddev = mtd_to_nanddev(mtd);
1626	int nsectors;
1627	int i;
1628
1629	if (nanddev->ecc.user_conf.flags & NAND_ECC_MAXIMIZE_STRENGTH) {
1630	int bytes;
1631
1632	ecc->size = 1024;
1633	nsectors = mtd->writesize / ecc->size;
1634
1635	/* Reserve 2 bytes for the BBM */
1636	bytes = (mtd->oobsize - 2) / nsectors;
1637
1638	/* 4 non-ECC bytes are added before each ECC bytes section */
1639	bytes -= 4;
1640
1641	/* and bytes has to be even. */
1642	if (bytes % 2)
1643	bytes--;
1644
1645	ecc->strength = bytes * 8 / fls(x: 8 * ecc->size);
1646
1647	for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1648	if (strengths[i] > ecc->strength)
1649	break;
1650	}
1651
1652	if (!i)
1653	ecc->strength = 0;
1654	else
1655	ecc->strength = strengths[i - 1];
1656	}
1657
1658	if (ecc->size != 512 && ecc->size != 1024)
1659	return -EINVAL;
1660
1661	/* Prefer 1k ECC chunk over 512 ones */
1662	if (ecc->size == 512 && mtd->writesize > 512) {
1663	ecc->size = 1024;
1664	ecc->strength *= 2;
1665	}
1666
1667	/* Add ECC info retrieval from DT */
1668	for (i = 0; i < ARRAY_SIZE(strengths); i++) {
1669	if (ecc->strength <= strengths[i]) {
1670	/*
1671	* Update ecc->strength value with the actual strength
1672	* that will be used by the ECC engine.
1673	*/
1674	ecc->strength = strengths[i];
1675	break;
1676	}
1677	}
1678
1679	if (i >= ARRAY_SIZE(strengths)) {
1680	dev_err(nfc->dev, "unsupported strength\n");
1681	return -ENOTSUPP;
1682	}
1683
1684	/* HW ECC always request ECC bytes for 1024 bytes blocks */
1685	ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * 1024), 8);
1686
1687	/* HW ECC always work with even numbers of ECC bytes */
1688	ecc->bytes = ALIGN(ecc->bytes, 2);
1689
1690	nsectors = mtd->writesize / ecc->size;
1691
1692	if (mtd->oobsize < ((ecc->bytes + 4) * nsectors))
1693	return -EINVAL;
1694
1695	ecc->read_oob = sunxi_nfc_hw_ecc_read_oob;
1696	ecc->write_oob = sunxi_nfc_hw_ecc_write_oob;
1697	mtd_set_ooblayout(mtd, ooblayout: &sunxi_nand_ooblayout_ops);
1698
1699	if (nfc->dmac || nfc->caps->has_mdma) {
1700	ecc->read_page = sunxi_nfc_hw_ecc_read_page_dma;
1701	ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage_dma;
1702	ecc->write_page = sunxi_nfc_hw_ecc_write_page_dma;
1703	nand->options |= NAND_USES_DMA;
1704	} else {
1705	ecc->read_page = sunxi_nfc_hw_ecc_read_page;
1706	ecc->read_subpage = sunxi_nfc_hw_ecc_read_subpage;
1707	ecc->write_page = sunxi_nfc_hw_ecc_write_page;
1708	}
1709
1710	/* TODO: support DMA for raw accesses and subpage write */
1711	ecc->write_subpage = sunxi_nfc_hw_ecc_write_subpage;
1712	ecc->read_oob_raw = nand_read_oob_std;
1713	ecc->write_oob_raw = nand_write_oob_std;
1714
1715	sunxi_nand->ecc.ecc_ctl = NFC_ECC_MODE(i) | NFC_ECC_EXCEPTION |
1716	NFC_ECC_PIPELINE | NFC_ECC_EN;
1717
1718	if (ecc->size == 512)
1719	sunxi_nand->ecc.ecc_ctl |= NFC_ECC_BLOCK_512;
1720
1721	return 0;
1722	}
1723
1724	static int sunxi_nand_attach_chip(struct nand_chip *nand)
1725	{
1726	const struct nand_ecc_props *requirements =
1727	nanddev_get_ecc_requirements(nand: &nand->base);
1728	struct nand_ecc_ctrl *ecc = &nand->ecc;
1729	struct device_node *np = nand_get_flash_node(chip: nand);
1730	int ret;
1731
1732	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1733	nand->bbt_options |= NAND_BBT_NO_OOB;
1734
1735	if (nand->options & NAND_NEED_SCRAMBLING)
1736	nand->options |= NAND_NO_SUBPAGE_WRITE;
1737
1738	nand->options |= NAND_SUBPAGE_READ;
1739
1740	if (!ecc->size) {
1741	ecc->size = requirements->step_size;
1742	ecc->strength = requirements->strength;
1743	}
1744
1745	if (!ecc->size || !ecc->strength)
1746	return -EINVAL;
1747
1748	switch (ecc->engine_type) {
1749	case NAND_ECC_ENGINE_TYPE_ON_HOST:
1750	ret = sunxi_nand_hw_ecc_ctrl_init(nand, ecc, np);
1751	if (ret)
1752	return ret;
1753	break;
1754	case NAND_ECC_ENGINE_TYPE_NONE:
1755	case NAND_ECC_ENGINE_TYPE_SOFT:
1756	break;
1757	default:
1758	return -EINVAL;
1759	}
1760
1761	return 0;
1762	}
1763
1764	static int sunxi_nfc_exec_subop(struct nand_chip *nand,
1765	const struct nand_subop *subop)
1766	{
1767	struct sunxi_nfc *nfc = to_sunxi_nfc(ctrl: nand->controller);
1768	u32 cmd = 0, extcmd = 0, cnt = 0, addrs[2] = { };
1769	unsigned int i, j, remaining, start;
1770	void *inbuf = NULL;
1771	int ret;
1772
1773	for (i = 0; i < subop->ninstrs; i++) {
1774	const struct nand_op_instr *instr = &subop->instrs[i];
1775
1776	switch (instr->type) {
1777	case NAND_OP_CMD_INSTR:
1778	if (cmd & NFC_SEND_CMD1) {
1779	if (WARN_ON(cmd & NFC_SEND_CMD2))
1780	return -EINVAL;
1781
1782	cmd |= NFC_SEND_CMD2;
1783	extcmd |= instr->ctx.cmd.opcode;
1784	} else {
1785	cmd |= NFC_SEND_CMD1 |
1786	NFC_CMD(instr->ctx.cmd.opcode);
1787	}
1788	break;
1789
1790	case NAND_OP_ADDR_INSTR:
1791	remaining = nand_subop_get_num_addr_cyc(subop, op_id: i);
1792	start = nand_subop_get_addr_start_off(subop, op_id: i);
1793	for (j = 0; j < 8 && j + start < remaining; j++) {
1794	u32 addr = instr->ctx.addr.addrs[j + start];
1795
1796	addrs[j / 4] |= addr << (j % 4) * 8;
1797	}
1798
1799	if (j)
1800	cmd |= NFC_SEND_ADR | NFC_ADR_NUM(j);
1801
1802	break;
1803
1804	case NAND_OP_DATA_IN_INSTR:
1805	case NAND_OP_DATA_OUT_INSTR:
1806	start = nand_subop_get_data_start_off(subop, op_id: i);
1807	remaining = nand_subop_get_data_len(subop, op_id: i);
1808	cnt = min_t(u32, remaining, NFC_SRAM_SIZE);
1809	cmd |= NFC_DATA_TRANS | NFC_DATA_SWAP_METHOD;
1810
1811	if (instr->type == NAND_OP_DATA_OUT_INSTR) {
1812	cmd |= NFC_ACCESS_DIR;
1813	memcpy_toio(nfc->regs + NFC_RAM0_BASE,
1814	instr->ctx.data.buf.out + start,
1815	cnt);
1816	} else {
1817	inbuf = instr->ctx.data.buf.in + start;
1818	}
1819
1820	break;
1821
1822	case NAND_OP_WAITRDY_INSTR:
1823	cmd |= NFC_WAIT_FLAG;
1824	break;
1825	}
1826	}
1827
1828	ret = sunxi_nfc_wait_cmd_fifo_empty(nfc);
1829	if (ret)
1830	return ret;
1831
1832	if (cmd & NFC_SEND_ADR) {
1833	writel(val: addrs[0], addr: nfc->regs + NFC_REG_ADDR_LOW);
1834	writel(val: addrs[1], addr: nfc->regs + NFC_REG_ADDR_HIGH);
1835	}
1836
1837	if (cmd & NFC_SEND_CMD2)
1838	writel(val: extcmd,
1839	addr: nfc->regs +
1840	(cmd & NFC_ACCESS_DIR ?
1841	NFC_REG_WCMD_SET : NFC_REG_RCMD_SET));
1842
1843	if (cmd & NFC_DATA_TRANS)
1844	writel(val: cnt, addr: nfc->regs + NFC_REG_CNT);
1845
1846	writel(val: cmd, addr: nfc->regs + NFC_REG_CMD);
1847
1848	ret = sunxi_nfc_wait_events(nfc, NFC_CMD_INT_FLAG,
1849	use_polling: !(cmd & NFC_WAIT_FLAG) && cnt < 64,
1850	timeout_ms: 0);
1851	if (ret)
1852	return ret;
1853
1854	if (inbuf)
1855	memcpy_fromio(inbuf, nfc->regs + NFC_RAM0_BASE, cnt);
1856
1857	return 0;
1858	}
1859
1860	static int sunxi_nfc_soft_waitrdy(struct nand_chip *nand,
1861	const struct nand_subop *subop)
1862	{
1863	return nand_soft_waitrdy(chip: nand,
1864	timeout_ms: subop->instrs[0].ctx.waitrdy.timeout_ms);
1865	}
1866
1867	static const struct nand_op_parser sunxi_nfc_op_parser = NAND_OP_PARSER(
1868	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1869	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1870	NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1871	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1872	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
1873	NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1874	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1875	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1876	NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1877	NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1878	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1879	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
1880	);
1881
1882	static const struct nand_op_parser sunxi_nfc_norb_op_parser = NAND_OP_PARSER(
1883	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1884	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1885	NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1886	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1887	NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, 1024)),
1888	NAND_OP_PARSER_PATTERN(sunxi_nfc_exec_subop,
1889	NAND_OP_PARSER_PAT_CMD_ELEM(true),
1890	NAND_OP_PARSER_PAT_ADDR_ELEM(true, 8),
1891	NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, 1024),
1892	NAND_OP_PARSER_PAT_CMD_ELEM(true)),
1893	NAND_OP_PARSER_PATTERN(sunxi_nfc_soft_waitrdy,
1894	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
1895	);
1896
1897	static int sunxi_nfc_exec_op(struct nand_chip *nand,
1898	const struct nand_operation *op, bool check_only)
1899	{
1900	struct sunxi_nand_chip *sunxi_nand = to_sunxi_nand(nand);
1901	const struct nand_op_parser *parser;
1902
1903	if (!check_only)
1904	sunxi_nfc_select_chip(nand, cs: op->cs);
1905
1906	if (sunxi_nand->sels[op->cs].rb >= 0)
1907	parser = &sunxi_nfc_op_parser;
1908	else
1909	parser = &sunxi_nfc_norb_op_parser;
1910
1911	return nand_op_parser_exec_op(chip: nand, parser, op, check_only);
1912	}
1913
1914	static const struct nand_controller_ops sunxi_nand_controller_ops = {
1915	.attach_chip = sunxi_nand_attach_chip,
1916	.setup_interface = sunxi_nfc_setup_interface,
1917	.exec_op = sunxi_nfc_exec_op,
1918	};
1919
1920	static void sunxi_nand_chips_cleanup(struct sunxi_nfc *nfc)
1921	{
1922	struct sunxi_nand_chip *sunxi_nand;
1923	struct nand_chip *chip;
1924	int ret;
1925
1926	while (!list_empty(head: &nfc->chips)) {
1927	sunxi_nand = list_first_entry(&nfc->chips,
1928	struct sunxi_nand_chip,
1929	node);
1930	chip = &sunxi_nand->nand;
1931	ret = mtd_device_unregister(master: nand_to_mtd(chip));
1932	WARN_ON(ret);
1933	nand_cleanup(chip);
1934	list_del(entry: &sunxi_nand->node);
1935	}
1936	}
1937
1938	static int sunxi_nand_chip_init(struct device *dev, struct sunxi_nfc *nfc,
1939	struct device_node *np)
1940	{
1941	struct sunxi_nand_chip *sunxi_nand;
1942	struct mtd_info *mtd;
1943	struct nand_chip *nand;
1944	int nsels;
1945	int ret;
1946	int i;
1947	u32 tmp;
1948
1949	if (!of_get_property(node: np, name: "reg", lenp: &nsels))
1950	return -EINVAL;
1951
1952	nsels /= sizeof(u32);
1953	if (!nsels) {
1954	dev_err(dev, "invalid reg property size\n");
1955	return -EINVAL;
1956	}
1957
1958	sunxi_nand = devm_kzalloc(dev, struct_size(sunxi_nand, sels, nsels),
1959	GFP_KERNEL);
1960	if (!sunxi_nand)
1961	return -ENOMEM;
1962
1963	sunxi_nand->nsels = nsels;
1964
1965	for (i = 0; i < nsels; i++) {
1966	ret = of_property_read_u32_index(np, propname: "reg", index: i, out_value: &tmp);
1967	if (ret) {
1968	dev_err(dev, "could not retrieve reg property: %d\n",
1969	ret);
1970	return ret;
1971	}
1972
1973	if (tmp > NFC_MAX_CS) {
1974	dev_err(dev,
1975	"invalid reg value: %u (max CS = 7)\n",
1976	tmp);
1977	return -EINVAL;
1978	}
1979
1980	if (test_and_set_bit(nr: tmp, addr: &nfc->assigned_cs)) {
1981	dev_err(dev, "CS %d already assigned\n", tmp);
1982	return -EINVAL;
1983	}
1984
1985	sunxi_nand->sels[i].cs = tmp;
1986
1987	if (!of_property_read_u32_index(np, propname: "allwinner,rb", index: i, out_value: &tmp) &&
1988	tmp < 2)
1989	sunxi_nand->sels[i].rb = tmp;
1990	else
1991	sunxi_nand->sels[i].rb = -1;
1992	}
1993
1994	nand = &sunxi_nand->nand;
1995	/* Default tR value specified in the ONFI spec (chapter 4.15.1) */
1996	nand->controller = &nfc->controller;
1997	nand->controller->ops = &sunxi_nand_controller_ops;
1998
1999	/*
2000	* Set the ECC mode to the default value in case nothing is specified
2001	* in the DT.
2002	*/
2003	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2004	nand_set_flash_node(chip: nand, np);
2005
2006	mtd = nand_to_mtd(chip: nand);
2007	mtd->dev.parent = dev;
2008
2009	ret = nand_scan(chip: nand, max_chips: nsels);
2010	if (ret)
2011	return ret;
2012
2013	ret = mtd_device_register(mtd, NULL, 0);
2014	if (ret) {
2015	dev_err(dev, "failed to register mtd device: %d\n", ret);
2016	nand_cleanup(chip: nand);
2017	return ret;
2018	}
2019
2020	list_add_tail(new: &sunxi_nand->node, head: &nfc->chips);
2021
2022	return 0;
2023	}
2024
2025	static int sunxi_nand_chips_init(struct device *dev, struct sunxi_nfc *nfc)
2026	{
2027	struct device_node *np = dev->of_node;
2028	struct device_node *nand_np;
2029	int ret;
2030
2031	for_each_child_of_node(np, nand_np) {
2032	ret = sunxi_nand_chip_init(dev, nfc, np: nand_np);
2033	if (ret) {
2034	of_node_put(node: nand_np);
2035	sunxi_nand_chips_cleanup(nfc);
2036	return ret;
2037	}
2038	}
2039
2040	return 0;
2041	}
2042
2043	static int sunxi_nfc_dma_init(struct sunxi_nfc *nfc, struct resource *r)
2044	{
2045	int ret;
2046
2047	if (nfc->caps->has_mdma)
2048	return 0;
2049
2050	nfc->dmac = dma_request_chan(dev: nfc->dev, name: "rxtx");
2051	if (IS_ERR(ptr: nfc->dmac)) {
2052	ret = PTR_ERR(ptr: nfc->dmac);
2053	if (ret == -EPROBE_DEFER)
2054	return ret;
2055
2056	/* Ignore errors to fall back to PIO mode */
2057	dev_warn(nfc->dev, "failed to request rxtx DMA channel: %d\n", ret);
2058	nfc->dmac = NULL;
2059	} else {
2060	struct dma_slave_config dmac_cfg = { };
2061
2062	dmac_cfg.src_addr = r->start + nfc->caps->reg_io_data;
2063	dmac_cfg.dst_addr = dmac_cfg.src_addr;
2064	dmac_cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2065	dmac_cfg.dst_addr_width = dmac_cfg.src_addr_width;
2066	dmac_cfg.src_maxburst = nfc->caps->dma_maxburst;
2067	dmac_cfg.dst_maxburst = nfc->caps->dma_maxburst;
2068	dmaengine_slave_config(chan: nfc->dmac, config: &dmac_cfg);
2069	}
2070	return 0;
2071	}
2072
2073	static int sunxi_nfc_probe(struct platform_device *pdev)
2074	{
2075	struct device *dev = &pdev->dev;
2076	struct resource *r;
2077	struct sunxi_nfc *nfc;
2078	int irq;
2079	int ret;
2080
2081	nfc = devm_kzalloc(dev, size: sizeof(*nfc), GFP_KERNEL);
2082	if (!nfc)
2083	return -ENOMEM;
2084
2085	nfc->dev = dev;
2086	nand_controller_init(nfc: &nfc->controller);
2087	INIT_LIST_HEAD(list: &nfc->chips);
2088
2089	nfc->regs = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &r);
2090	if (IS_ERR(ptr: nfc->regs))
2091	return PTR_ERR(ptr: nfc->regs);
2092
2093	irq = platform_get_irq(pdev, 0);
2094	if (irq < 0)
2095	return irq;
2096
2097	nfc->ahb_clk = devm_clk_get_enabled(dev, id: "ahb");
2098	if (IS_ERR(ptr: nfc->ahb_clk)) {
2099	dev_err(dev, "failed to retrieve ahb clk\n");
2100	return PTR_ERR(ptr: nfc->ahb_clk);
2101	}
2102
2103	nfc->mod_clk = devm_clk_get_enabled(dev, id: "mod");
2104	if (IS_ERR(ptr: nfc->mod_clk)) {
2105	dev_err(dev, "failed to retrieve mod clk\n");
2106	return PTR_ERR(ptr: nfc->mod_clk);
2107	}
2108
2109	nfc->reset = devm_reset_control_get_optional_exclusive(dev, id: "ahb");
2110	if (IS_ERR(ptr: nfc->reset))
2111	return PTR_ERR(ptr: nfc->reset);
2112
2113	ret = reset_control_deassert(rstc: nfc->reset);
2114	if (ret) {
2115	dev_err(dev, "reset err %d\n", ret);
2116	return ret;
2117	}
2118
2119	nfc->caps = of_device_get_match_data(dev: &pdev->dev);
2120	if (!nfc->caps) {
2121	ret = -EINVAL;
2122	goto out_ahb_reset_reassert;
2123	}
2124
2125	ret = sunxi_nfc_rst(nfc);
2126	if (ret)
2127	goto out_ahb_reset_reassert;
2128
2129	writel(val: 0, addr: nfc->regs + NFC_REG_INT);
2130	ret = devm_request_irq(dev, irq, handler: sunxi_nfc_interrupt,
2131	irqflags: 0, devname: "sunxi-nand", dev_id: nfc);
2132	if (ret)
2133	goto out_ahb_reset_reassert;
2134
2135	ret = sunxi_nfc_dma_init(nfc, r);
2136
2137	if (ret)
2138	goto out_ahb_reset_reassert;
2139
2140	platform_set_drvdata(pdev, data: nfc);
2141
2142	ret = sunxi_nand_chips_init(dev, nfc);
2143	if (ret) {
2144	dev_err(dev, "failed to init nand chips\n");
2145	goto out_release_dmac;
2146	}
2147
2148	return 0;
2149
2150	out_release_dmac:
2151	if (nfc->dmac)
2152	dma_release_channel(chan: nfc->dmac);
2153	out_ahb_reset_reassert:
2154	reset_control_assert(rstc: nfc->reset);
2155
2156	return ret;
2157	}
2158
2159	static void sunxi_nfc_remove(struct platform_device *pdev)
2160	{
2161	struct sunxi_nfc *nfc = platform_get_drvdata(pdev);
2162
2163	sunxi_nand_chips_cleanup(nfc);
2164
2165	reset_control_assert(rstc: nfc->reset);
2166
2167	if (nfc->dmac)
2168	dma_release_channel(chan: nfc->dmac);
2169	}
2170
2171	static const struct sunxi_nfc_caps sunxi_nfc_a10_caps = {
2172	.reg_io_data = NFC_REG_A10_IO_DATA,
2173	.dma_maxburst = 4,
2174	};
2175
2176	static const struct sunxi_nfc_caps sunxi_nfc_a23_caps = {
2177	.has_mdma = true,
2178	.reg_io_data = NFC_REG_A23_IO_DATA,
2179	.dma_maxburst = 8,
2180	};
2181
2182	static const struct of_device_id sunxi_nfc_ids[] = {
2183	{
2184	.compatible = "allwinner,sun4i-a10-nand",
2185	.data = &sunxi_nfc_a10_caps,
2186	},
2187	{
2188	.compatible = "allwinner,sun8i-a23-nand-controller",
2189	.data = &sunxi_nfc_a23_caps,
2190	},
2191	{ /* sentinel */ }
2192	};
2193	MODULE_DEVICE_TABLE(of, sunxi_nfc_ids);
2194
2195	static struct platform_driver sunxi_nfc_driver = {
2196	.driver = {
2197	.name = "sunxi_nand",
2198	.of_match_table = sunxi_nfc_ids,
2199	},
2200	.probe = sunxi_nfc_probe,
2201	.remove_new = sunxi_nfc_remove,
2202	};
2203	module_platform_driver(sunxi_nfc_driver);
2204
2205	MODULE_LICENSE("GPL");
2206	MODULE_AUTHOR("Boris BREZILLON");
2207	MODULE_DESCRIPTION("Allwinner NAND Flash Controller driver");
2208	MODULE_ALIAS("platform:sunxi_nand");
2209