1	// SPDX-License-Identifier: GPL-2.0 OR MIT
2	/*
3	* Rockchip NAND Flash controller driver.
4	* Copyright (C) 2020 Rockchip Inc.
5	* Author: Yifeng Zhao <yifeng.zhao@rock-chips.com>
6	*/
7
8	#include <linux/clk.h>
9	#include <linux/delay.h>
10	#include <linux/dma-mapping.h>
11	#include <linux/dmaengine.h>
12	#include <linux/interrupt.h>
13	#include <linux/iopoll.h>
14	#include <linux/module.h>
15	#include <linux/mtd/mtd.h>
16	#include <linux/mtd/rawnand.h>
17	#include <linux/of.h>
18	#include <linux/platform_device.h>
19	#include <linux/slab.h>
20
21	/*
22	* NFC Page Data Layout:
23	* 1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
24	* 1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
25	* ......
26	* NAND Page Data Layout:
27	* 1024 * n data + m Bytes oob
28	* Original Bad Block Mask Location:
29	* First byte of oob(spare).
30	* nand_chip->oob_poi data layout:
31	* 4Bytes sys data + .... + 4Bytes sys data + ECC data.
32	*/
33
34	/* NAND controller register definition */
35	#define NFC_READ (0)
36	#define NFC_WRITE (1)
37
38	#define NFC_FMCTL (0x00)
39	#define FMCTL_CE_SEL_M 0xFF
40	#define FMCTL_CE_SEL(x) (1 << (x))
41	#define FMCTL_WP BIT(8)
42	#define FMCTL_RDY BIT(9)
43
44	#define NFC_FMWAIT (0x04)
45	#define FLCTL_RST BIT(0)
46	#define FLCTL_WR (1) /* 0: read, 1: write */
47	#define FLCTL_XFER_ST BIT(2)
48	#define FLCTL_XFER_EN BIT(3)
49	#define FLCTL_ACORRECT BIT(10) /* Auto correct error bits. */
50	#define FLCTL_XFER_READY BIT(20)
51	#define FLCTL_XFER_SECTOR (22)
52	#define FLCTL_TOG_FIX BIT(29)
53
54	#define BCHCTL_BANK_M (7 << 5)
55	#define BCHCTL_BANK (5)
56
57	#define DMA_ST BIT(0)
58	#define DMA_WR (1) /* 0: write, 1: read */
59	#define DMA_EN BIT(2)
60	#define DMA_AHB_SIZE (3) /* 0: 1, 1: 2, 2: 4 */
61	#define DMA_BURST_SIZE (6) /* 0: 1, 3: 4, 5: 8, 7: 16 */
62	#define DMA_INC_NUM (9) /* 1 - 16 */
63
64	#define ECC_ERR_CNT(x, e) ((((x) >> (e).low) & (e).low_mask) |\
65	(((x) >> (e).high) & (e).high_mask) << (e).low_bn)
66	#define INT_DMA BIT(0)
67	#define NFC_BANK (0x800)
68	#define NFC_BANK_STEP (0x100)
69	#define BANK_DATA (0x00)
70	#define BANK_ADDR (0x04)
71	#define BANK_CMD (0x08)
72	#define NFC_SRAM0 (0x1000)
73	#define NFC_SRAM1 (0x1400)
74	#define NFC_SRAM_SIZE (0x400)
75	#define NFC_TIMEOUT (500000)
76	#define NFC_MAX_OOB_PER_STEP 128
77	#define NFC_MIN_OOB_PER_STEP 64
78	#define MAX_DATA_SIZE 0xFFFC
79	#define MAX_ADDRESS_CYC 6
80	#define NFC_ECC_MAX_MODES 4
81	#define NFC_MAX_NSELS (8) /* Some Socs only have 1 or 2 CSs. */
82	#define NFC_SYS_DATA_SIZE (4) /* 4 bytes sys data in oob pre 1024 data.*/
83	#define RK_DEFAULT_CLOCK_RATE (150 * 1000 * 1000) /* 150 Mhz */
84	#define ACCTIMING(csrw, rwpw, rwcs) ((csrw) << 12 | (rwpw) << 5 | (rwcs))
85
86	enum nfc_type {
87	NFC_V6,
88	NFC_V8,
89	NFC_V9,
90	};
91
92	/**
93	* struct rk_ecc_cnt_status: represent a ecc status data.
94	* @err_flag_bit: error flag bit index at register.
95	* @low: ECC count low bit index at register.
96	* @low_mask: mask bit.
97	* @low_bn: ECC count low bit number.
98	* @high: ECC count high bit index at register.
99	* @high_mask: mask bit
100	*/
101	struct rk_ecc_cnt_status {
102	u8 err_flag_bit;
103	u8 low;
104	u8 low_mask;
105	u8 low_bn;
106	u8 high;
107	u8 high_mask;
108	};
109
110	/**
111	* struct nfc_cfg: Rockchip NAND controller configuration
112	* @type: NFC version
113	* @ecc_strengths: ECC strengths
114	* @ecc_cfgs: ECC config values
115	* @flctl_off: FLCTL register offset
116	* @bchctl_off: BCHCTL register offset
117	* @dma_data_buf_off: DMA_DATA_BUF register offset
118	* @dma_oob_buf_off: DMA_OOB_BUF register offset
119	* @dma_cfg_off: DMA_CFG register offset
120	* @dma_st_off: DMA_ST register offset
121	* @bch_st_off: BCG_ST register offset
122	* @randmz_off: RANDMZ register offset
123	* @int_en_off: interrupt enable register offset
124	* @int_clr_off: interrupt clean register offset
125	* @int_st_off: interrupt status register offset
126	* @oob0_off: oob0 register offset
127	* @oob1_off: oob1 register offset
128	* @ecc0: represent ECC0 status data
129	* @ecc1: represent ECC1 status data
130	*/
131	struct nfc_cfg {
132	enum nfc_type type;
133	u8 ecc_strengths[NFC_ECC_MAX_MODES];
134	u32 ecc_cfgs[NFC_ECC_MAX_MODES];
135	u32 flctl_off;
136	u32 bchctl_off;
137	u32 dma_cfg_off;
138	u32 dma_data_buf_off;
139	u32 dma_oob_buf_off;
140	u32 dma_st_off;
141	u32 bch_st_off;
142	u32 randmz_off;
143	u32 int_en_off;
144	u32 int_clr_off;
145	u32 int_st_off;
146	u32 oob0_off;
147	u32 oob1_off;
148	struct rk_ecc_cnt_status ecc0;
149	struct rk_ecc_cnt_status ecc1;
150	};
151
152	struct rk_nfc_nand_chip {
153	struct list_head node;
154	struct nand_chip chip;
155
156	u16 boot_blks;
157	u16 metadata_size;
158	u32 boot_ecc;
159	u32 timing;
160
161	u8 nsels;
162	u8 sels[] __counted_by(nsels);
163	};
164
165	struct rk_nfc {
166	struct nand_controller controller;
167	const struct nfc_cfg *cfg;
168	struct device *dev;
169
170	struct clk *nfc_clk;
171	struct clk *ahb_clk;
172	void __iomem *regs;
173
174	u32 selected_bank;
175	u32 band_offset;
176	u32 cur_ecc;
177	u32 cur_timing;
178
179	struct completion done;
180	struct list_head chips;
181
182	u8 *page_buf;
183	u32 *oob_buf;
184	u32 page_buf_size;
185	u32 oob_buf_size;
186
187	unsigned long assigned_cs;
188	};
189
190	static inline struct rk_nfc_nand_chip *rk_nfc_to_rknand(struct nand_chip *chip)
191	{
192	return container_of(chip, struct rk_nfc_nand_chip, chip);
193	}
194
195	static inline u8 *rk_nfc_buf_to_data_ptr(struct nand_chip *chip, const u8 *p, int i)
196	{
197	return (u8 *)p + i * chip->ecc.size;
198	}
199
200	static inline u8 *rk_nfc_buf_to_oob_ptr(struct nand_chip *chip, int i)
201	{
202	u8 *poi;
203
204	poi = chip->oob_poi + i * NFC_SYS_DATA_SIZE;
205
206	return poi;
207	}
208
209	static inline u8 *rk_nfc_buf_to_oob_ecc_ptr(struct nand_chip *chip, int i)
210	{
211	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
212	u8 *poi;
213
214	poi = chip->oob_poi + rknand->metadata_size + chip->ecc.bytes * i;
215
216	return poi;
217	}
218
219	static inline int rk_nfc_data_len(struct nand_chip *chip)
220	{
221	return chip->ecc.size + chip->ecc.bytes + NFC_SYS_DATA_SIZE;
222	}
223
224	static inline u8 *rk_nfc_data_ptr(struct nand_chip *chip, int i)
225	{
226	struct rk_nfc *nfc = nand_get_controller_data(chip);
227
228	return nfc->page_buf + i * rk_nfc_data_len(chip);
229	}
230
231	static inline u8 *rk_nfc_oob_ptr(struct nand_chip *chip, int i)
232	{
233	struct rk_nfc *nfc = nand_get_controller_data(chip);
234
235	return nfc->page_buf + i * rk_nfc_data_len(chip) + chip->ecc.size;
236	}
237
238	static int rk_nfc_hw_ecc_setup(struct nand_chip *chip, u32 strength)
239	{
240	struct rk_nfc *nfc = nand_get_controller_data(chip);
241	u32 reg, i;
242
243	for (i = 0; i < NFC_ECC_MAX_MODES; i++) {
244	if (strength == nfc->cfg->ecc_strengths[i]) {
245	reg = nfc->cfg->ecc_cfgs[i];
246	break;
247	}
248	}
249
250	if (i >= NFC_ECC_MAX_MODES)
251	return -EINVAL;
252
253	writel(val: reg, addr: nfc->regs + nfc->cfg->bchctl_off);
254
255	/* Save chip ECC setting */
256	nfc->cur_ecc = strength;
257
258	return 0;
259	}
260
261	static void rk_nfc_select_chip(struct nand_chip *chip, int cs)
262	{
263	struct rk_nfc *nfc = nand_get_controller_data(chip);
264	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
265	struct nand_ecc_ctrl *ecc = &chip->ecc;
266	u32 val;
267
268	if (cs < 0) {
269	nfc->selected_bank = -1;
270	/* Deselect the currently selected target. */
271	val = readl_relaxed(nfc->regs + NFC_FMCTL);
272	val &= ~FMCTL_CE_SEL_M;
273	writel(val, addr: nfc->regs + NFC_FMCTL);
274	return;
275	}
276
277	nfc->selected_bank = rknand->sels[cs];
278	nfc->band_offset = NFC_BANK + nfc->selected_bank * NFC_BANK_STEP;
279
280	val = readl_relaxed(nfc->regs + NFC_FMCTL);
281	val &= ~FMCTL_CE_SEL_M;
282	val |= FMCTL_CE_SEL(nfc->selected_bank);
283
284	writel(val, addr: nfc->regs + NFC_FMCTL);
285
286	/*
287	* Compare current chip timing with selected chip timing and
288	* change if needed.
289	*/
290	if (nfc->cur_timing != rknand->timing) {
291	writel(val: rknand->timing, addr: nfc->regs + NFC_FMWAIT);
292	nfc->cur_timing = rknand->timing;
293	}
294
295	/*
296	* Compare current chip ECC setting with selected chip ECC setting and
297	* change if needed.
298	*/
299	if (nfc->cur_ecc != ecc->strength)
300	rk_nfc_hw_ecc_setup(chip, strength: ecc->strength);
301	}
302
303	static inline int rk_nfc_wait_ioready(struct rk_nfc *nfc)
304	{
305	int rc;
306	u32 val;
307
308	rc = readl_relaxed_poll_timeout(nfc->regs + NFC_FMCTL, val,
309	val & FMCTL_RDY, 10, NFC_TIMEOUT);
310
311	return rc;
312	}
313
314	static void rk_nfc_read_buf(struct rk_nfc *nfc, u8 *buf, int len)
315	{
316	int i;
317
318	for (i = 0; i < len; i++)
319	buf[i] = readb_relaxed(nfc->regs + nfc->band_offset +
320	BANK_DATA);
321	}
322
323	static void rk_nfc_write_buf(struct rk_nfc *nfc, const u8 *buf, int len)
324	{
325	int i;
326
327	for (i = 0; i < len; i++)
328	writeb(val: buf[i], addr: nfc->regs + nfc->band_offset + BANK_DATA);
329	}
330
331	static int rk_nfc_cmd(struct nand_chip *chip,
332	const struct nand_subop *subop)
333	{
334	struct rk_nfc *nfc = nand_get_controller_data(chip);
335	unsigned int i, j, remaining, start;
336	int reg_offset = nfc->band_offset;
337	u8 *inbuf = NULL;
338	const u8 *outbuf;
339	u32 cnt = 0;
340	int ret = 0;
341
342	for (i = 0; i < subop->ninstrs; i++) {
343	const struct nand_op_instr *instr = &subop->instrs[i];
344
345	switch (instr->type) {
346	case NAND_OP_CMD_INSTR:
347	writeb(val: instr->ctx.cmd.opcode,
348	addr: nfc->regs + reg_offset + BANK_CMD);
349	break;
350
351	case NAND_OP_ADDR_INSTR:
352	remaining = nand_subop_get_num_addr_cyc(subop, op_id: i);
353	start = nand_subop_get_addr_start_off(subop, op_id: i);
354
355	for (j = 0; j < 8 && j + start < remaining; j++)
356	writeb(val: instr->ctx.addr.addrs[j + start],
357	addr: nfc->regs + reg_offset + BANK_ADDR);
358	break;
359
360	case NAND_OP_DATA_IN_INSTR:
361	case NAND_OP_DATA_OUT_INSTR:
362	start = nand_subop_get_data_start_off(subop, op_id: i);
363	cnt = nand_subop_get_data_len(subop, op_id: i);
364
365	if (instr->type == NAND_OP_DATA_OUT_INSTR) {
366	outbuf = instr->ctx.data.buf.out + start;
367	rk_nfc_write_buf(nfc, buf: outbuf, len: cnt);
368	} else {
369	inbuf = instr->ctx.data.buf.in + start;
370	rk_nfc_read_buf(nfc, buf: inbuf, len: cnt);
371	}
372	break;
373
374	case NAND_OP_WAITRDY_INSTR:
375	if (rk_nfc_wait_ioready(nfc) < 0) {
376	ret = -ETIMEDOUT;
377	dev_err(nfc->dev, "IO not ready\n");
378	}
379	break;
380	}
381	}
382
383	return ret;
384	}
385
386	static const struct nand_op_parser rk_nfc_op_parser = NAND_OP_PARSER(
387	NAND_OP_PARSER_PATTERN(
388	rk_nfc_cmd,
389	NAND_OP_PARSER_PAT_CMD_ELEM(true),
390	NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
391	NAND_OP_PARSER_PAT_CMD_ELEM(true),
392	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
393	NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, MAX_DATA_SIZE)),
394	NAND_OP_PARSER_PATTERN(
395	rk_nfc_cmd,
396	NAND_OP_PARSER_PAT_CMD_ELEM(true),
397	NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
398	NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, MAX_DATA_SIZE),
399	NAND_OP_PARSER_PAT_CMD_ELEM(true),
400	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
401	);
402
403	static int rk_nfc_exec_op(struct nand_chip *chip,
404	const struct nand_operation *op,
405	bool check_only)
406	{
407	if (!check_only)
408	rk_nfc_select_chip(chip, cs: op->cs);
409
410	return nand_op_parser_exec_op(chip, parser: &rk_nfc_op_parser, op,
411	check_only);
412	}
413
414	static int rk_nfc_setup_interface(struct nand_chip *chip, int target,
415	const struct nand_interface_config *conf)
416	{
417	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
418	struct rk_nfc *nfc = nand_get_controller_data(chip);
419	const struct nand_sdr_timings *timings;
420	u32 rate, tc2rw, trwpw, trw2c;
421	u32 temp;
422
423	if (target < 0)
424	return 0;
425
426	timings = nand_get_sdr_timings(conf);
427	if (IS_ERR(ptr: timings))
428	return -EOPNOTSUPP;
429
430	if (IS_ERR(ptr: nfc->nfc_clk))
431	rate = clk_get_rate(clk: nfc->ahb_clk);
432	else
433	rate = clk_get_rate(clk: nfc->nfc_clk);
434
435	/* Turn clock rate into kHz. */
436	rate /= 1000;
437
438	tc2rw = 1;
439	trw2c = 1;
440
441	trwpw = max(timings->tWC_min, timings->tRC_min) / 1000;
442	trwpw = DIV_ROUND_UP(trwpw * rate, 1000000);
443
444	temp = timings->tREA_max / 1000;
445	temp = DIV_ROUND_UP(temp * rate, 1000000);
446
447	if (trwpw < temp)
448	trwpw = temp;
449
450	/*
451	* ACCON: access timing control register
452	* -------------------------------------
453	* 31:18: reserved
454	* 17:12: csrw, clock cycles from the falling edge of CSn to the
455	* falling edge of RDn or WRn
456	* 11:11: reserved
457	* 10:05: rwpw, the width of RDn or WRn in processor clock cycles
458	* 04:00: rwcs, clock cycles from the rising edge of RDn or WRn to the
459	* rising edge of CSn
460	*/
461
462	/* Save chip timing */
463	rknand->timing = ACCTIMING(tc2rw, trwpw, trw2c);
464
465	return 0;
466	}
467
468	static void rk_nfc_xfer_start(struct rk_nfc *nfc, u8 rw, u8 n_KB,
469	dma_addr_t dma_data, dma_addr_t dma_oob)
470	{
471	u32 dma_reg, fl_reg, bch_reg;
472
473	dma_reg = DMA_ST | ((!rw) << DMA_WR) | DMA_EN | (2 << DMA_AHB_SIZE) |
474	(7 << DMA_BURST_SIZE) | (16 << DMA_INC_NUM);
475
476	fl_reg = (rw << FLCTL_WR) | FLCTL_XFER_EN | FLCTL_ACORRECT |
477	(n_KB << FLCTL_XFER_SECTOR) | FLCTL_TOG_FIX;
478
479	if (nfc->cfg->type == NFC_V6 || nfc->cfg->type == NFC_V8) {
480	bch_reg = readl_relaxed(nfc->regs + nfc->cfg->bchctl_off);
481	bch_reg = (bch_reg & (~BCHCTL_BANK_M)) |
482	(nfc->selected_bank << BCHCTL_BANK);
483	writel(val: bch_reg, addr: nfc->regs + nfc->cfg->bchctl_off);
484	}
485
486	writel(val: dma_reg, addr: nfc->regs + nfc->cfg->dma_cfg_off);
487	writel(val: (u32)dma_data, addr: nfc->regs + nfc->cfg->dma_data_buf_off);
488	writel(val: (u32)dma_oob, addr: nfc->regs + nfc->cfg->dma_oob_buf_off);
489	writel(val: fl_reg, addr: nfc->regs + nfc->cfg->flctl_off);
490	fl_reg |= FLCTL_XFER_ST;
491	writel(val: fl_reg, addr: nfc->regs + nfc->cfg->flctl_off);
492	}
493
494	static int rk_nfc_wait_for_xfer_done(struct rk_nfc *nfc)
495	{
496	void __iomem *ptr;
497	u32 reg;
498
499	ptr = nfc->regs + nfc->cfg->flctl_off;
500
501	return readl_relaxed_poll_timeout(ptr, reg,
502	reg & FLCTL_XFER_READY,
503	10, NFC_TIMEOUT);
504	}
505
506	static int rk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
507	int oob_on, int page)
508	{
509	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
510	struct rk_nfc *nfc = nand_get_controller_data(chip);
511	struct mtd_info *mtd = nand_to_mtd(chip);
512	struct nand_ecc_ctrl *ecc = &chip->ecc;
513	int i, pages_per_blk;
514
515	pages_per_blk = mtd->erasesize / mtd->writesize;
516	if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
517	(page < (pages_per_blk * rknand->boot_blks)) &&
518	rknand->boot_ecc != ecc->strength) {
519	/*
520	* There's currently no method to notify the MTD framework that
521	* a different ECC strength is in use for the boot blocks.
522	*/
523	return -EIO;
524	}
525
526	if (!buf)
527	memset(nfc->page_buf, 0xff, mtd->writesize + mtd->oobsize);
528
529	for (i = 0; i < ecc->steps; i++) {
530	/* Copy data to the NFC buffer. */
531	if (buf)
532	memcpy(rk_nfc_data_ptr(chip, i),
533	rk_nfc_buf_to_data_ptr(chip, buf, i),
534	ecc->size);
535	/*
536	* The first four bytes of OOB are reserved for the
537	* boot ROM. In some debugging cases, such as with a
538	* read, erase and write back test these 4 bytes stored
539	* in OOB also need to be written back.
540	*
541	* The function nand_block_bad detects bad blocks like:
542	*
543	* bad = chip->oob_poi[chip->badblockpos];
544	*
545	* chip->badblockpos == 0 for a large page NAND Flash,
546	* so chip->oob_poi[0] is the bad block mask (BBM).
547	*
548	* The OOB data layout on the NFC is:
549	*
550	* PA0 PA1 PA2 PA3 | BBM OOB1 OOB2 OOB3 | ...
551	*
552	* or
553	*
554	* 0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
555	*
556	* The code here just swaps the first 4 bytes with the last
557	* 4 bytes without losing any data.
558	*
559	* The chip->oob_poi data layout:
560	*
561	* BBM OOB1 OOB2 OOB3 |......| PA0 PA1 PA2 PA3
562	*
563	* The rk_nfc_ooblayout_free() function already has reserved
564	* these 4 bytes together with 2 bytes for BBM
565	* by reducing it's length:
566	*
567	* oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
568	*/
569	if (!i)
570	memcpy(rk_nfc_oob_ptr(chip, i),
571	rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
572	NFC_SYS_DATA_SIZE);
573	else
574	memcpy(rk_nfc_oob_ptr(chip, i),
575	rk_nfc_buf_to_oob_ptr(chip, i - 1),
576	NFC_SYS_DATA_SIZE);
577	/* Copy ECC data to the NFC buffer. */
578	memcpy(rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
579	rk_nfc_buf_to_oob_ecc_ptr(chip, i),
580	ecc->bytes);
581	}
582
583	nand_prog_page_begin_op(chip, page, offset_in_page: 0, NULL, len: 0);
584	rk_nfc_write_buf(nfc, buf, len: mtd->writesize + mtd->oobsize);
585	return nand_prog_page_end_op(chip);
586	}
587
588	static int rk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
589	int oob_on, int page)
590	{
591	struct mtd_info *mtd = nand_to_mtd(chip);
592	struct rk_nfc *nfc = nand_get_controller_data(chip);
593	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
594	struct nand_ecc_ctrl *ecc = &chip->ecc;
595	int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
596	NFC_MIN_OOB_PER_STEP;
597	int pages_per_blk = mtd->erasesize / mtd->writesize;
598	int ret = 0, i, boot_rom_mode = 0;
599	dma_addr_t dma_data, dma_oob;
600	u32 tmp;
601	u8 *oob;
602
603	nand_prog_page_begin_op(chip, page, offset_in_page: 0, NULL, len: 0);
604
605	if (buf)
606	memcpy(nfc->page_buf, buf, mtd->writesize);
607	else
608	memset(nfc->page_buf, 0xFF, mtd->writesize);
609
610	/*
611	* The first blocks (4, 8 or 16 depending on the device) are used
612	* by the boot ROM and the first 32 bits of OOB need to link to
613	* the next page address in the same block. We can't directly copy
614	* OOB data from the MTD framework, because this page address
615	* conflicts for example with the bad block marker (BBM),
616	* so we shift all OOB data including the BBM with 4 byte positions.
617	* As a consequence the OOB size available to the MTD framework is
618	* also reduced with 4 bytes.
619	*
620	* PA0 PA1 PA2 PA3 | BBM OOB1 OOB2 OOB3 | ...
621	*
622	* If a NAND is not a boot medium or the page is not a boot block,
623	* the first 4 bytes are left untouched by writing 0xFF to them.
624	*
625	* 0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
626	*
627	* The code here just swaps the first 4 bytes with the last
628	* 4 bytes without losing any data.
629	*
630	* The chip->oob_poi data layout:
631	*
632	* BBM OOB1 OOB2 OOB3 |......| PA0 PA1 PA2 PA3
633	*
634	* Configure the ECC algorithm supported by the boot ROM.
635	*/
636	if ((page < (pages_per_blk * rknand->boot_blks)) &&
637	(chip->options & NAND_IS_BOOT_MEDIUM)) {
638	boot_rom_mode = 1;
639	if (rknand->boot_ecc != ecc->strength)
640	rk_nfc_hw_ecc_setup(chip, strength: rknand->boot_ecc);
641	}
642
643	for (i = 0; i < ecc->steps; i++) {
644	if (!i)
645	oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
646	else
647	oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
648
649	tmp = oob[0] | oob[1] << 8 | oob[2] << 16 | oob[3] << 24;
650
651	if (nfc->cfg->type == NFC_V9)
652	nfc->oob_buf[i] = tmp;
653	else
654	nfc->oob_buf[i * (oob_step / 4)] = tmp;
655	}
656
657	dma_data = dma_map_single(nfc->dev, (void *)nfc->page_buf,
658	mtd->writesize, DMA_TO_DEVICE);
659	dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
660	ecc->steps * oob_step,
661	DMA_TO_DEVICE);
662
663	reinit_completion(x: &nfc->done);
664	writel(INT_DMA, addr: nfc->regs + nfc->cfg->int_en_off);
665
666	rk_nfc_xfer_start(nfc, NFC_WRITE, n_KB: ecc->steps, dma_data,
667	dma_oob);
668	ret = wait_for_completion_timeout(x: &nfc->done,
669	timeout: msecs_to_jiffies(m: 100));
670	if (!ret)
671	dev_warn(nfc->dev, "write: wait dma done timeout.\n");
672	/*
673	* Whether the DMA transfer is completed or not. The driver
674	* needs to check the NFC`s status register to see if the data
675	* transfer was completed.
676	*/
677	ret = rk_nfc_wait_for_xfer_done(nfc);
678
679	dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
680	DMA_TO_DEVICE);
681	dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
682	DMA_TO_DEVICE);
683
684	if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
685	rk_nfc_hw_ecc_setup(chip, strength: ecc->strength);
686
687	if (ret) {
688	dev_err(nfc->dev, "write: wait transfer done timeout.\n");
689	return -ETIMEDOUT;
690	}
691
692	return nand_prog_page_end_op(chip);
693	}
694
695	static int rk_nfc_write_oob(struct nand_chip *chip, int page)
696	{
697	return rk_nfc_write_page_hwecc(chip, NULL, oob_on: 1, page);
698	}
699
700	static int rk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
701	int page)
702	{
703	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
704	struct rk_nfc *nfc = nand_get_controller_data(chip);
705	struct mtd_info *mtd = nand_to_mtd(chip);
706	struct nand_ecc_ctrl *ecc = &chip->ecc;
707	int i, pages_per_blk;
708
709	pages_per_blk = mtd->erasesize / mtd->writesize;
710	if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
711	(page < (pages_per_blk * rknand->boot_blks)) &&
712	rknand->boot_ecc != ecc->strength) {
713	/*
714	* There's currently no method to notify the MTD framework that
715	* a different ECC strength is in use for the boot blocks.
716	*/
717	return -EIO;
718	}
719
720	nand_read_page_op(chip, page, offset_in_page: 0, NULL, len: 0);
721	rk_nfc_read_buf(nfc, buf: nfc->page_buf, len: mtd->writesize + mtd->oobsize);
722	for (i = 0; i < ecc->steps; i++) {
723	/*
724	* The first four bytes of OOB are reserved for the
725	* boot ROM. In some debugging cases, such as with a read,
726	* erase and write back test, these 4 bytes also must be
727	* saved somewhere, otherwise this information will be
728	* lost during a write back.
729	*/
730	if (!i)
731	memcpy(rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
732	rk_nfc_oob_ptr(chip, i),
733	NFC_SYS_DATA_SIZE);
734	else
735	memcpy(rk_nfc_buf_to_oob_ptr(chip, i - 1),
736	rk_nfc_oob_ptr(chip, i),
737	NFC_SYS_DATA_SIZE);
738
739	/* Copy ECC data from the NFC buffer. */
740	memcpy(rk_nfc_buf_to_oob_ecc_ptr(chip, i),
741	rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
742	ecc->bytes);
743
744	/* Copy data from the NFC buffer. */
745	if (buf)
746	memcpy(rk_nfc_buf_to_data_ptr(chip, buf, i),
747	rk_nfc_data_ptr(chip, i),
748	ecc->size);
749	}
750
751	return 0;
752	}
753
754	static int rk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_on,
755	int page)
756	{
757	struct mtd_info *mtd = nand_to_mtd(chip);
758	struct rk_nfc *nfc = nand_get_controller_data(chip);
759	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
760	struct nand_ecc_ctrl *ecc = &chip->ecc;
761	int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
762	NFC_MIN_OOB_PER_STEP;
763	int pages_per_blk = mtd->erasesize / mtd->writesize;
764	dma_addr_t dma_data, dma_oob;
765	int ret = 0, i, cnt, boot_rom_mode = 0;
766	int max_bitflips = 0, bch_st, ecc_fail = 0;
767	u8 *oob;
768	u32 tmp;
769
770	nand_read_page_op(chip, page, offset_in_page: 0, NULL, len: 0);
771
772	dma_data = dma_map_single(nfc->dev, nfc->page_buf,
773	mtd->writesize,
774	DMA_FROM_DEVICE);
775	dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
776	ecc->steps * oob_step,
777	DMA_FROM_DEVICE);
778
779	/*
780	* The first blocks (4, 8 or 16 depending on the device)
781	* are used by the boot ROM.
782	* Configure the ECC algorithm supported by the boot ROM.
783	*/
784	if ((page < (pages_per_blk * rknand->boot_blks)) &&
785	(chip->options & NAND_IS_BOOT_MEDIUM)) {
786	boot_rom_mode = 1;
787	if (rknand->boot_ecc != ecc->strength)
788	rk_nfc_hw_ecc_setup(chip, strength: rknand->boot_ecc);
789	}
790
791	reinit_completion(x: &nfc->done);
792	writel(INT_DMA, addr: nfc->regs + nfc->cfg->int_en_off);
793	rk_nfc_xfer_start(nfc, NFC_READ, n_KB: ecc->steps, dma_data,
794	dma_oob);
795	ret = wait_for_completion_timeout(x: &nfc->done,
796	timeout: msecs_to_jiffies(m: 100));
797	if (!ret)
798	dev_warn(nfc->dev, "read: wait dma done timeout.\n");
799	/*
800	* Whether the DMA transfer is completed or not. The driver
801	* needs to check the NFC`s status register to see if the data
802	* transfer was completed.
803	*/
804	ret = rk_nfc_wait_for_xfer_done(nfc);
805
806	dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
807	DMA_FROM_DEVICE);
808	dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
809	DMA_FROM_DEVICE);
810
811	if (ret) {
812	ret = -ETIMEDOUT;
813	dev_err(nfc->dev, "read: wait transfer done timeout.\n");
814	goto timeout_err;
815	}
816
817	for (i = 0; i < ecc->steps; i++) {
818	if (!i)
819	oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
820	else
821	oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
822
823	if (nfc->cfg->type == NFC_V9)
824	tmp = nfc->oob_buf[i];
825	else
826	tmp = nfc->oob_buf[i * (oob_step / 4)];
827
828	*oob++ = (u8)tmp;
829	*oob++ = (u8)(tmp >> 8);
830	*oob++ = (u8)(tmp >> 16);
831	*oob++ = (u8)(tmp >> 24);
832	}
833
834	for (i = 0; i < (ecc->steps / 2); i++) {
835	bch_st = readl_relaxed(nfc->regs +
836	nfc->cfg->bch_st_off + i * 4);
837	if (bch_st & BIT(nfc->cfg->ecc0.err_flag_bit) ||
838	bch_st & BIT(nfc->cfg->ecc1.err_flag_bit)) {
839	mtd->ecc_stats.failed++;
840	ecc_fail = 1;
841	} else {
842	cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc0);
843	mtd->ecc_stats.corrected += cnt;
844	max_bitflips = max_t(u32, max_bitflips, cnt);
845
846	cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc1);
847	mtd->ecc_stats.corrected += cnt;
848	max_bitflips = max_t(u32, max_bitflips, cnt);
849	}
850	}
851
852	if (buf)
853	memcpy(buf, nfc->page_buf, mtd->writesize);
854
855	timeout_err:
856	if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
857	rk_nfc_hw_ecc_setup(chip, strength: ecc->strength);
858
859	if (ret)
860	return ret;
861
862	if (ecc_fail) {
863	dev_err(nfc->dev, "read page: %x ecc error!\n", page);
864	return 0;
865	}
866
867	return max_bitflips;
868	}
869
870	static int rk_nfc_read_oob(struct nand_chip *chip, int page)
871	{
872	return rk_nfc_read_page_hwecc(chip, NULL, oob_on: 1, page);
873	}
874
875	static inline void rk_nfc_hw_init(struct rk_nfc *nfc)
876	{
877	/* Disable flash wp. */
878	writel(FMCTL_WP, addr: nfc->regs + NFC_FMCTL);
879	/* Config default timing 40ns at 150 Mhz NFC clock. */
880	writel(val: 0x1081, addr: nfc->regs + NFC_FMWAIT);
881	nfc->cur_timing = 0x1081;
882	/* Disable randomizer and DMA. */
883	writel(val: 0, addr: nfc->regs + nfc->cfg->randmz_off);
884	writel(val: 0, addr: nfc->regs + nfc->cfg->dma_cfg_off);
885	writel(FLCTL_RST, addr: nfc->regs + nfc->cfg->flctl_off);
886	}
887
888	static irqreturn_t rk_nfc_irq(int irq, void *id)
889	{
890	struct rk_nfc *nfc = id;
891	u32 sta, ien;
892
893	sta = readl_relaxed(nfc->regs + nfc->cfg->int_st_off);
894	ien = readl_relaxed(nfc->regs + nfc->cfg->int_en_off);
895
896	if (!(sta & ien))
897	return IRQ_NONE;
898
899	writel(val: sta, addr: nfc->regs + nfc->cfg->int_clr_off);
900	writel(val: ~sta & ien, addr: nfc->regs + nfc->cfg->int_en_off);
901
902	complete(&nfc->done);
903
904	return IRQ_HANDLED;
905	}
906
907	static int rk_nfc_enable_clks(struct device *dev, struct rk_nfc *nfc)
908	{
909	int ret;
910
911	if (!IS_ERR(ptr: nfc->nfc_clk)) {
912	ret = clk_prepare_enable(clk: nfc->nfc_clk);
913	if (ret) {
914	dev_err(dev, "failed to enable NFC clk\n");
915	return ret;
916	}
917	}
918
919	ret = clk_prepare_enable(clk: nfc->ahb_clk);
920	if (ret) {
921	dev_err(dev, "failed to enable ahb clk\n");
922	clk_disable_unprepare(clk: nfc->nfc_clk);
923	return ret;
924	}
925
926	return 0;
927	}
928
929	static void rk_nfc_disable_clks(struct rk_nfc *nfc)
930	{
931	clk_disable_unprepare(clk: nfc->nfc_clk);
932	clk_disable_unprepare(clk: nfc->ahb_clk);
933	}
934
935	static int rk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
936	struct mtd_oob_region *oob_region)
937	{
938	struct nand_chip *chip = mtd_to_nand(mtd);
939	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
940
941	if (section)
942	return -ERANGE;
943
944	oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
945	oob_region->offset = 2;
946
947	return 0;
948	}
949
950	static int rk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
951	struct mtd_oob_region *oob_region)
952	{
953	struct nand_chip *chip = mtd_to_nand(mtd);
954	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
955
956	if (section)
957	return -ERANGE;
958
959	oob_region->length = mtd->oobsize - rknand->metadata_size;
960	oob_region->offset = rknand->metadata_size;
961
962	return 0;
963	}
964
965	static const struct mtd_ooblayout_ops rk_nfc_ooblayout_ops = {
966	.free = rk_nfc_ooblayout_free,
967	.ecc = rk_nfc_ooblayout_ecc,
968	};
969
970	static int rk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
971	{
972	struct nand_chip *chip = mtd_to_nand(mtd);
973	struct rk_nfc *nfc = nand_get_controller_data(chip);
974	struct nand_ecc_ctrl *ecc = &chip->ecc;
975	const u8 *strengths = nfc->cfg->ecc_strengths;
976	u8 max_strength, nfc_max_strength;
977	int i;
978
979	nfc_max_strength = nfc->cfg->ecc_strengths[0];
980	/* If optional dt settings not present. */
981	if (!ecc->size || !ecc->strength ||
982	ecc->strength > nfc_max_strength) {
983	chip->ecc.size = 1024;
984	ecc->steps = mtd->writesize / ecc->size;
985
986	/*
987	* HW ECC always requests the number of ECC bytes per 1024 byte
988	* blocks. The first 4 OOB bytes are reserved for sys data.
989	*/
990	max_strength = ((mtd->oobsize / ecc->steps) - 4) * 8 /
991	fls(x: 8 * 1024);
992	if (max_strength > nfc_max_strength)
993	max_strength = nfc_max_strength;
994
995	for (i = 0; i < 4; i++) {
996	if (max_strength >= strengths[i])
997	break;
998	}
999
1000	if (i >= 4) {
1001	dev_err(nfc->dev, "unsupported ECC strength\n");
1002	return -EOPNOTSUPP;
1003	}
1004
1005	ecc->strength = strengths[i];
1006	}
1007	ecc->steps = mtd->writesize / ecc->size;
1008	ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * chip->ecc.size), 8);
1009
1010	return 0;
1011	}
1012
1013	static int rk_nfc_attach_chip(struct nand_chip *chip)
1014	{
1015	struct mtd_info *mtd = nand_to_mtd(chip);
1016	struct device *dev = mtd->dev.parent;
1017	struct rk_nfc *nfc = nand_get_controller_data(chip);
1018	struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
1019	struct nand_ecc_ctrl *ecc = &chip->ecc;
1020	int new_page_len, new_oob_len;
1021	void *buf;
1022	int ret;
1023
1024	if (chip->options & NAND_BUSWIDTH_16) {
1025	dev_err(dev, "16 bits bus width not supported");
1026	return -EINVAL;
1027	}
1028
1029	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST)
1030	return 0;
1031
1032	ret = rk_nfc_ecc_init(dev, mtd);
1033	if (ret)
1034	return ret;
1035
1036	rknand->metadata_size = NFC_SYS_DATA_SIZE * ecc->steps;
1037
1038	if (rknand->metadata_size < NFC_SYS_DATA_SIZE + 2) {
1039	dev_err(dev,
1040	"driver needs at least %d bytes of meta data\n",
1041	NFC_SYS_DATA_SIZE + 2);
1042	return -EIO;
1043	}
1044
1045	/* Check buffer first, avoid duplicate alloc buffer. */
1046	new_page_len = mtd->writesize + mtd->oobsize;
1047	if (nfc->page_buf && new_page_len > nfc->page_buf_size) {
1048	buf = krealloc(objp: nfc->page_buf, new_size: new_page_len,
1049	GFP_KERNEL | GFP_DMA);
1050	if (!buf)
1051	return -ENOMEM;
1052	nfc->page_buf = buf;
1053	nfc->page_buf_size = new_page_len;
1054	}
1055
1056	new_oob_len = ecc->steps * NFC_MAX_OOB_PER_STEP;
1057	if (nfc->oob_buf && new_oob_len > nfc->oob_buf_size) {
1058	buf = krealloc(objp: nfc->oob_buf, new_size: new_oob_len,
1059	GFP_KERNEL | GFP_DMA);
1060	if (!buf) {
1061	kfree(objp: nfc->page_buf);
1062	nfc->page_buf = NULL;
1063	return -ENOMEM;
1064	}
1065	nfc->oob_buf = buf;
1066	nfc->oob_buf_size = new_oob_len;
1067	}
1068
1069	if (!nfc->page_buf) {
1070	nfc->page_buf = kzalloc(size: new_page_len, GFP_KERNEL | GFP_DMA);
1071	if (!nfc->page_buf)
1072	return -ENOMEM;
1073	nfc->page_buf_size = new_page_len;
1074	}
1075
1076	if (!nfc->oob_buf) {
1077	nfc->oob_buf = kzalloc(size: new_oob_len, GFP_KERNEL | GFP_DMA);
1078	if (!nfc->oob_buf) {
1079	kfree(objp: nfc->page_buf);
1080	nfc->page_buf = NULL;
1081	return -ENOMEM;
1082	}
1083	nfc->oob_buf_size = new_oob_len;
1084	}
1085
1086	chip->ecc.write_page_raw = rk_nfc_write_page_raw;
1087	chip->ecc.write_page = rk_nfc_write_page_hwecc;
1088	chip->ecc.write_oob = rk_nfc_write_oob;
1089
1090	chip->ecc.read_page_raw = rk_nfc_read_page_raw;
1091	chip->ecc.read_page = rk_nfc_read_page_hwecc;
1092	chip->ecc.read_oob = rk_nfc_read_oob;
1093
1094	return 0;
1095	}
1096
1097	static const struct nand_controller_ops rk_nfc_controller_ops = {
1098	.attach_chip = rk_nfc_attach_chip,
1099	.exec_op = rk_nfc_exec_op,
1100	.setup_interface = rk_nfc_setup_interface,
1101	};
1102
1103	static int rk_nfc_nand_chip_init(struct device *dev, struct rk_nfc *nfc,
1104	struct device_node *np)
1105	{
1106	struct rk_nfc_nand_chip *rknand;
1107	struct nand_chip *chip;
1108	struct mtd_info *mtd;
1109	int nsels;
1110	u32 tmp;
1111	int ret;
1112	int i;
1113
1114	if (!of_get_property(node: np, name: "reg", lenp: &nsels))
1115	return -ENODEV;
1116	nsels /= sizeof(u32);
1117	if (!nsels || nsels > NFC_MAX_NSELS) {
1118	dev_err(dev, "invalid reg property size %d\n", nsels);
1119	return -EINVAL;
1120	}
1121
1122	rknand = devm_kzalloc(dev, struct_size(rknand, sels, nsels),
1123	GFP_KERNEL);
1124	if (!rknand)
1125	return -ENOMEM;
1126
1127	rknand->nsels = nsels;
1128	for (i = 0; i < nsels; i++) {
1129	ret = of_property_read_u32_index(np, propname: "reg", index: i, out_value: &tmp);
1130	if (ret) {
1131	dev_err(dev, "reg property failure : %d\n", ret);
1132	return ret;
1133	}
1134
1135	if (tmp >= NFC_MAX_NSELS) {
1136	dev_err(dev, "invalid CS: %u\n", tmp);
1137	return -EINVAL;
1138	}
1139
1140	if (test_and_set_bit(nr: tmp, addr: &nfc->assigned_cs)) {
1141	dev_err(dev, "CS %u already assigned\n", tmp);
1142	return -EINVAL;
1143	}
1144
1145	rknand->sels[i] = tmp;
1146	}
1147
1148	chip = &rknand->chip;
1149	chip->controller = &nfc->controller;
1150
1151	nand_set_flash_node(chip, np);
1152
1153	nand_set_controller_data(chip, priv: nfc);
1154
1155	chip->options |= NAND_USES_DMA | NAND_NO_SUBPAGE_WRITE;
1156	chip->bbt_options = NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1157
1158	/* Set default mode in case dt entry is missing. */
1159	chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1160
1161	mtd = nand_to_mtd(chip);
1162	mtd->owner = THIS_MODULE;
1163	mtd->dev.parent = dev;
1164
1165	if (!mtd->name) {
1166	dev_err(nfc->dev, "NAND label property is mandatory\n");
1167	return -EINVAL;
1168	}
1169
1170	mtd_set_ooblayout(mtd, ooblayout: &rk_nfc_ooblayout_ops);
1171	rk_nfc_hw_init(nfc);
1172	ret = nand_scan(chip, max_chips: nsels);
1173	if (ret)
1174	return ret;
1175
1176	if (chip->options & NAND_IS_BOOT_MEDIUM) {
1177	ret = of_property_read_u32(np, propname: "rockchip,boot-blks", out_value: &tmp);
1178	rknand->boot_blks = ret ? 0 : tmp;
1179
1180	ret = of_property_read_u32(np, propname: "rockchip,boot-ecc-strength",
1181	out_value: &tmp);
1182	rknand->boot_ecc = ret ? chip->ecc.strength : tmp;
1183	}
1184
1185	ret = mtd_device_register(mtd, NULL, 0);
1186	if (ret) {
1187	dev_err(dev, "MTD parse partition error\n");
1188	nand_cleanup(chip);
1189	return ret;
1190	}
1191
1192	list_add_tail(new: &rknand->node, head: &nfc->chips);
1193
1194	return 0;
1195	}
1196
1197	static void rk_nfc_chips_cleanup(struct rk_nfc *nfc)
1198	{
1199	struct rk_nfc_nand_chip *rknand, *tmp;
1200	struct nand_chip *chip;
1201	int ret;
1202
1203	list_for_each_entry_safe(rknand, tmp, &nfc->chips, node) {
1204	chip = &rknand->chip;
1205	ret = mtd_device_unregister(master: nand_to_mtd(chip));
1206	WARN_ON(ret);
1207	nand_cleanup(chip);
1208	list_del(entry: &rknand->node);
1209	}
1210	}
1211
1212	static int rk_nfc_nand_chips_init(struct device *dev, struct rk_nfc *nfc)
1213	{
1214	struct device_node *np = dev->of_node, *nand_np;
1215	int nchips = of_get_child_count(np);
1216	int ret;
1217
1218	if (!nchips || nchips > NFC_MAX_NSELS) {
1219	dev_err(nfc->dev, "incorrect number of NAND chips (%d)\n",
1220	nchips);
1221	return -EINVAL;
1222	}
1223
1224	for_each_child_of_node(np, nand_np) {
1225	ret = rk_nfc_nand_chip_init(dev, nfc, np: nand_np);
1226	if (ret) {
1227	of_node_put(node: nand_np);
1228	rk_nfc_chips_cleanup(nfc);
1229	return ret;
1230	}
1231	}
1232
1233	return 0;
1234	}
1235
1236	static struct nfc_cfg nfc_v6_cfg = {
1237	.type = NFC_V6,
1238	.ecc_strengths = {60, 40, 24, 16},
1239	.ecc_cfgs = {
1240	0x00040011, 0x00040001, 0x00000011, 0x00000001,
1241	},
1242	.flctl_off = 0x08,
1243	.bchctl_off = 0x0C,
1244	.dma_cfg_off = 0x10,
1245	.dma_data_buf_off = 0x14,
1246	.dma_oob_buf_off = 0x18,
1247	.dma_st_off = 0x1C,
1248	.bch_st_off = 0x20,
1249	.randmz_off = 0x150,
1250	.int_en_off = 0x16C,
1251	.int_clr_off = 0x170,
1252	.int_st_off = 0x174,
1253	.oob0_off = 0x200,
1254	.oob1_off = 0x230,
1255	.ecc0 = {
1256	.err_flag_bit = 2,
1257	.low = 3,
1258	.low_mask = 0x1F,
1259	.low_bn = 5,
1260	.high = 27,
1261	.high_mask = 0x1,
1262	},
1263	.ecc1 = {
1264	.err_flag_bit = 15,
1265	.low = 16,
1266	.low_mask = 0x1F,
1267	.low_bn = 5,
1268	.high = 29,
1269	.high_mask = 0x1,
1270	},
1271	};
1272
1273	static struct nfc_cfg nfc_v8_cfg = {
1274	.type = NFC_V8,
1275	.ecc_strengths = {16, 16, 16, 16},
1276	.ecc_cfgs = {
1277	0x00000001, 0x00000001, 0x00000001, 0x00000001,
1278	},
1279	.flctl_off = 0x08,
1280	.bchctl_off = 0x0C,
1281	.dma_cfg_off = 0x10,
1282	.dma_data_buf_off = 0x14,
1283	.dma_oob_buf_off = 0x18,
1284	.dma_st_off = 0x1C,
1285	.bch_st_off = 0x20,
1286	.randmz_off = 0x150,
1287	.int_en_off = 0x16C,
1288	.int_clr_off = 0x170,
1289	.int_st_off = 0x174,
1290	.oob0_off = 0x200,
1291	.oob1_off = 0x230,
1292	.ecc0 = {
1293	.err_flag_bit = 2,
1294	.low = 3,
1295	.low_mask = 0x1F,
1296	.low_bn = 5,
1297	.high = 27,
1298	.high_mask = 0x1,
1299	},
1300	.ecc1 = {
1301	.err_flag_bit = 15,
1302	.low = 16,
1303	.low_mask = 0x1F,
1304	.low_bn = 5,
1305	.high = 29,
1306	.high_mask = 0x1,
1307	},
1308	};
1309
1310	static struct nfc_cfg nfc_v9_cfg = {
1311	.type = NFC_V9,
1312	.ecc_strengths = {70, 60, 40, 16},
1313	.ecc_cfgs = {
1314	0x00000001, 0x06000001, 0x04000001, 0x02000001,
1315	},
1316	.flctl_off = 0x10,
1317	.bchctl_off = 0x20,
1318	.dma_cfg_off = 0x30,
1319	.dma_data_buf_off = 0x34,
1320	.dma_oob_buf_off = 0x38,
1321	.dma_st_off = 0x3C,
1322	.bch_st_off = 0x150,
1323	.randmz_off = 0x208,
1324	.int_en_off = 0x120,
1325	.int_clr_off = 0x124,
1326	.int_st_off = 0x128,
1327	.oob0_off = 0x200,
1328	.oob1_off = 0x204,
1329	.ecc0 = {
1330	.err_flag_bit = 2,
1331	.low = 3,
1332	.low_mask = 0x7F,
1333	.low_bn = 7,
1334	.high = 0,
1335	.high_mask = 0x0,
1336	},
1337	.ecc1 = {
1338	.err_flag_bit = 18,
1339	.low = 19,
1340	.low_mask = 0x7F,
1341	.low_bn = 7,
1342	.high = 0,
1343	.high_mask = 0x0,
1344	},
1345	};
1346
1347	static const struct of_device_id rk_nfc_id_table[] = {
1348	{
1349	.compatible = "rockchip,px30-nfc",
1350	.data = &nfc_v9_cfg
1351	},
1352	{
1353	.compatible = "rockchip,rk2928-nfc",
1354	.data = &nfc_v6_cfg
1355	},
1356	{
1357	.compatible = "rockchip,rv1108-nfc",
1358	.data = &nfc_v8_cfg
1359	},
1360	{ /* sentinel */ }
1361	};
1362	MODULE_DEVICE_TABLE(of, rk_nfc_id_table);
1363
1364	static int rk_nfc_probe(struct platform_device *pdev)
1365	{
1366	struct device *dev = &pdev->dev;
1367	struct rk_nfc *nfc;
1368	int ret, irq;
1369
1370	nfc = devm_kzalloc(dev, size: sizeof(*nfc), GFP_KERNEL);
1371	if (!nfc)
1372	return -ENOMEM;
1373
1374	nand_controller_init(nfc: &nfc->controller);
1375	INIT_LIST_HEAD(list: &nfc->chips);
1376	nfc->controller.ops = &rk_nfc_controller_ops;
1377
1378	nfc->cfg = of_device_get_match_data(dev);
1379	nfc->dev = dev;
1380
1381	init_completion(x: &nfc->done);
1382
1383	nfc->regs = devm_platform_ioremap_resource(pdev, index: 0);
1384	if (IS_ERR(ptr: nfc->regs)) {
1385	ret = PTR_ERR(ptr: nfc->regs);
1386	goto release_nfc;
1387	}
1388
1389	nfc->nfc_clk = devm_clk_get(dev, id: "nfc");
1390	if (IS_ERR(ptr: nfc->nfc_clk)) {
1391	dev_dbg(dev, "no NFC clk\n");
1392	/* Some earlier models, such as rk3066, have no NFC clk. */
1393	}
1394
1395	nfc->ahb_clk = devm_clk_get(dev, id: "ahb");
1396	if (IS_ERR(ptr: nfc->ahb_clk)) {
1397	dev_err(dev, "no ahb clk\n");
1398	ret = PTR_ERR(ptr: nfc->ahb_clk);
1399	goto release_nfc;
1400	}
1401
1402	ret = rk_nfc_enable_clks(dev, nfc);
1403	if (ret)
1404	goto release_nfc;
1405
1406	irq = platform_get_irq(pdev, 0);
1407	if (irq < 0) {
1408	ret = -EINVAL;
1409	goto clk_disable;
1410	}
1411
1412	writel(val: 0, addr: nfc->regs + nfc->cfg->int_en_off);
1413	ret = devm_request_irq(dev, irq, handler: rk_nfc_irq, irqflags: 0x0, devname: "rk-nand", dev_id: nfc);
1414	if (ret) {
1415	dev_err(dev, "failed to request NFC irq\n");
1416	goto clk_disable;
1417	}
1418
1419	platform_set_drvdata(pdev, data: nfc);
1420
1421	ret = rk_nfc_nand_chips_init(dev, nfc);
1422	if (ret) {
1423	dev_err(dev, "failed to init NAND chips\n");
1424	goto clk_disable;
1425	}
1426	return 0;
1427
1428	clk_disable:
1429	rk_nfc_disable_clks(nfc);
1430	release_nfc:
1431	return ret;
1432	}
1433
1434	static void rk_nfc_remove(struct platform_device *pdev)
1435	{
1436	struct rk_nfc *nfc = platform_get_drvdata(pdev);
1437
1438	kfree(objp: nfc->page_buf);
1439	kfree(objp: nfc->oob_buf);
1440	rk_nfc_chips_cleanup(nfc);
1441	rk_nfc_disable_clks(nfc);
1442	}
1443
1444	static int __maybe_unused rk_nfc_suspend(struct device *dev)
1445	{
1446	struct rk_nfc *nfc = dev_get_drvdata(dev);
1447
1448	rk_nfc_disable_clks(nfc);
1449
1450	return 0;
1451	}
1452
1453	static int __maybe_unused rk_nfc_resume(struct device *dev)
1454	{
1455	struct rk_nfc *nfc = dev_get_drvdata(dev);
1456	struct rk_nfc_nand_chip *rknand;
1457	struct nand_chip *chip;
1458	int ret;
1459	u32 i;
1460
1461	ret = rk_nfc_enable_clks(dev, nfc);
1462	if (ret)
1463	return ret;
1464
1465	/* Reset NAND chip if VCC was powered off. */
1466	list_for_each_entry(rknand, &nfc->chips, node) {
1467	chip = &rknand->chip;
1468	for (i = 0; i < rknand->nsels; i++)
1469	nand_reset(chip, chipnr: i);
1470	}
1471
1472	return 0;
1473	}
1474
1475	static const struct dev_pm_ops rk_nfc_pm_ops = {
1476	SET_SYSTEM_SLEEP_PM_OPS(rk_nfc_suspend, rk_nfc_resume)
1477	};
1478
1479	static struct platform_driver rk_nfc_driver = {
1480	.probe = rk_nfc_probe,
1481	.remove_new = rk_nfc_remove,
1482	.driver = {
1483	.name = "rockchip-nfc",
1484	.of_match_table = rk_nfc_id_table,
1485	.pm = &rk_nfc_pm_ops,
1486	},
1487	};
1488
1489	module_platform_driver(rk_nfc_driver);
1490
1491	MODULE_LICENSE("Dual MIT/GPL");
1492	MODULE_AUTHOR("Yifeng Zhao <yifeng.zhao@rock-chips.com>");
1493	MODULE_DESCRIPTION("Rockchip Nand Flash Controller Driver");
1494	MODULE_ALIAS("platform:rockchip-nand-controller");
1495