1	// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
2	/*
3	* Amlogic Meson Nand Flash Controller Driver
4	*
5	* Copyright (c) 2018 Amlogic, inc.
6	* Author: Liang Yang <liang.yang@amlogic.com>
7	*/
8
9	#include <linux/platform_device.h>
10	#include <linux/dma-mapping.h>
11	#include <linux/interrupt.h>
12	#include <linux/clk.h>
13	#include <linux/clk-provider.h>
14	#include <linux/mtd/rawnand.h>
15	#include <linux/mtd/mtd.h>
16	#include <linux/mfd/syscon.h>
17	#include <linux/regmap.h>
18	#include <linux/slab.h>
19	#include <linux/module.h>
20	#include <linux/iopoll.h>
21	#include <linux/of.h>
22	#include <linux/sched/task_stack.h>
23
24	#define NFC_REG_CMD 0x00
25	#define NFC_CMD_IDLE (0xc << 14)
26	#define NFC_CMD_CLE (0x5 << 14)
27	#define NFC_CMD_ALE (0x6 << 14)
28	#define NFC_CMD_ADL ((0 << 16) | (3 << 20))
29	#define NFC_CMD_ADH ((1 << 16) | (3 << 20))
30	#define NFC_CMD_AIL ((2 << 16) | (3 << 20))
31	#define NFC_CMD_AIH ((3 << 16) | (3 << 20))
32	#define NFC_CMD_SEED ((8 << 16) | (3 << 20))
33	#define NFC_CMD_M2N ((0 << 17) | (2 << 20))
34	#define NFC_CMD_N2M ((1 << 17) | (2 << 20))
35	#define NFC_CMD_RB BIT(20)
36	#define NFC_CMD_SCRAMBLER_ENABLE BIT(19)
37	#define NFC_CMD_SCRAMBLER_DISABLE 0
38	#define NFC_CMD_SHORTMODE_DISABLE 0
39	#define NFC_CMD_RB_INT BIT(14)
40	#define NFC_CMD_RB_INT_NO_PIN ((0xb << 10) | BIT(18) | BIT(16))
41
42	#define NFC_CMD_GET_SIZE(x) (((x) >> 22) & GENMASK(4, 0))
43
44	#define NFC_REG_CFG 0x04
45	#define NFC_REG_DADR 0x08
46	#define NFC_REG_IADR 0x0c
47	#define NFC_REG_BUF 0x10
48	#define NFC_REG_INFO 0x14
49	#define NFC_REG_DC 0x18
50	#define NFC_REG_ADR 0x1c
51	#define NFC_REG_DL 0x20
52	#define NFC_REG_DH 0x24
53	#define NFC_REG_CADR 0x28
54	#define NFC_REG_SADR 0x2c
55	#define NFC_REG_PINS 0x30
56	#define NFC_REG_VER 0x38
57
58	#define NFC_RB_IRQ_EN BIT(21)
59
60	#define CLK_DIV_SHIFT 0
61	#define CLK_DIV_WIDTH 6
62
63	#define CMDRWGEN(cmd_dir, ran, bch, short_mode, page_size, pages) \
64	( \
65	(cmd_dir) | \
66	(ran) | \
67	((bch) << 14) | \
68	((short_mode) << 13) | \
69	(((page_size) & 0x7f) << 6) | \
70	((pages) & 0x3f) \
71	)
72
73	#define GENCMDDADDRL(adl, addr) ((adl) | ((addr) & 0xffff))
74	#define GENCMDDADDRH(adh, addr) ((adh) | (((addr) >> 16) & 0xffff))
75	#define GENCMDIADDRL(ail, addr) ((ail) | ((addr) & 0xffff))
76	#define GENCMDIADDRH(aih, addr) ((aih) | (((addr) >> 16) & 0xffff))
77
78	#define DMA_DIR(dir) ((dir) ? NFC_CMD_N2M : NFC_CMD_M2N)
79	#define DMA_ADDR_ALIGN 8
80
81	#define ECC_CHECK_RETURN_FF (-1)
82
83	#define NAND_CE0 (0xe << 10)
84	#define NAND_CE1 (0xd << 10)
85
86	#define DMA_BUSY_TIMEOUT 0x100000
87	#define CMD_FIFO_EMPTY_TIMEOUT 1000
88
89	#define MAX_CE_NUM 2
90
91	/* eMMC clock register, misc control */
92	#define CLK_SELECT_NAND BIT(31)
93	#define CLK_ALWAYS_ON_NAND BIT(24)
94	#define CLK_SELECT_FIX_PLL2 BIT(6)
95
96	#define NFC_CLK_CYCLE 6
97
98	/* nand flash controller delay 3 ns */
99	#define NFC_DEFAULT_DELAY 3000
100
101	#define ROW_ADDER(page, index) (((page) >> (8 * (index))) & 0xff)
102	#define MAX_CYCLE_ADDRS 5
103	#define DIRREAD 1
104	#define DIRWRITE 0
105
106	#define ECC_PARITY_BCH8_512B 14
107	#define ECC_COMPLETE BIT(31)
108	#define ECC_ERR_CNT(x) (((x) >> 24) & GENMASK(5, 0))
109	#define ECC_ZERO_CNT(x) (((x) >> 16) & GENMASK(5, 0))
110	#define ECC_UNCORRECTABLE 0x3f
111
112	#define PER_INFO_BYTE 8
113
114	#define NFC_CMD_RAW_LEN GENMASK(13, 0)
115
116	#define NFC_COLUMN_ADDR_0 0
117	#define NFC_COLUMN_ADDR_1 0
118
119	struct meson_nfc_nand_chip {
120	struct list_head node;
121	struct nand_chip nand;
122	unsigned long clk_rate;
123	unsigned long level1_divider;
124	u32 bus_timing;
125	u32 twb;
126	u32 tadl;
127	u32 tbers_max;
128
129	u32 bch_mode;
130	u8 *data_buf;
131	__le64 *info_buf;
132	u32 nsels;
133	u8 sels[] __counted_by(nsels);
134	};
135
136	struct meson_nand_ecc {
137	u32 bch;
138	u32 strength;
139	u32 size;
140	};
141
142	struct meson_nfc_data {
143	const struct nand_ecc_caps *ecc_caps;
144	};
145
146	struct meson_nfc_param {
147	u32 chip_select;
148	u32 rb_select;
149	};
150
151	struct nand_rw_cmd {
152	u32 cmd0;
153	u32 addrs[MAX_CYCLE_ADDRS];
154	u32 cmd1;
155	};
156
157	struct nand_timing {
158	u32 twb;
159	u32 tadl;
160	u32 tbers_max;
161	};
162
163	struct meson_nfc {
164	struct nand_controller controller;
165	struct clk *core_clk;
166	struct clk *device_clk;
167	struct clk *nand_clk;
168	struct clk_divider nand_divider;
169
170	unsigned long clk_rate;
171	u32 bus_timing;
172
173	struct device *dev;
174	void __iomem *reg_base;
175	void __iomem *reg_clk;
176	struct completion completion;
177	struct list_head chips;
178	const struct meson_nfc_data *data;
179	struct meson_nfc_param param;
180	struct nand_timing timing;
181	union {
182	int cmd[32];
183	struct nand_rw_cmd rw;
184	} cmdfifo;
185
186	dma_addr_t daddr;
187	dma_addr_t iaddr;
188	u32 info_bytes;
189
190	unsigned long assigned_cs;
191	bool no_rb_pin;
192	};
193
194	enum {
195	NFC_ECC_BCH8_512 = 1,
196	NFC_ECC_BCH8_1K,
197	NFC_ECC_BCH24_1K,
198	NFC_ECC_BCH30_1K,
199	NFC_ECC_BCH40_1K,
200	NFC_ECC_BCH50_1K,
201	NFC_ECC_BCH60_1K,
202	};
203
204	#define MESON_ECC_DATA(b, s, sz) { .bch = (b), .strength = (s), .size = (sz) }
205
206	static struct meson_nand_ecc meson_ecc[] = {
207	MESON_ECC_DATA(NFC_ECC_BCH8_512, 8, 512),
208	MESON_ECC_DATA(NFC_ECC_BCH8_1K, 8, 1024),
209	MESON_ECC_DATA(NFC_ECC_BCH24_1K, 24, 1024),
210	MESON_ECC_DATA(NFC_ECC_BCH30_1K, 30, 1024),
211	MESON_ECC_DATA(NFC_ECC_BCH40_1K, 40, 1024),
212	MESON_ECC_DATA(NFC_ECC_BCH50_1K, 50, 1024),
213	MESON_ECC_DATA(NFC_ECC_BCH60_1K, 60, 1024),
214	};
215
216	static int meson_nand_calc_ecc_bytes(int step_size, int strength)
217	{
218	int ecc_bytes;
219
220	if (step_size == 512 && strength == 8)
221	return ECC_PARITY_BCH8_512B;
222
223	ecc_bytes = DIV_ROUND_UP(strength * fls(step_size * 8), 8);
224	ecc_bytes = ALIGN(ecc_bytes, 2);
225
226	return ecc_bytes;
227	}
228
229	NAND_ECC_CAPS_SINGLE(meson_gxl_ecc_caps,
230	meson_nand_calc_ecc_bytes, 1024, 8, 24, 30, 40, 50, 60);
231
232	static const int axg_stepinfo_strengths[] = { 8 };
233
234	static const struct nand_ecc_step_info axg_stepinfo[] = {
235	{
236	.stepsize = 1024,
237	.strengths = axg_stepinfo_strengths,
238	.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
239	},
240	{
241	.stepsize = 512,
242	.strengths = axg_stepinfo_strengths,
243	.nstrengths = ARRAY_SIZE(axg_stepinfo_strengths)
244	},
245	};
246
247	static const struct nand_ecc_caps meson_axg_ecc_caps = {
248	.stepinfos = axg_stepinfo,
249	.nstepinfos = ARRAY_SIZE(axg_stepinfo),
250	.calc_ecc_bytes = meson_nand_calc_ecc_bytes,
251	};
252
253	static struct meson_nfc_nand_chip *to_meson_nand(struct nand_chip *nand)
254	{
255	return container_of(nand, struct meson_nfc_nand_chip, nand);
256	}
257
258	static void meson_nfc_select_chip(struct nand_chip *nand, int chip)
259	{
260	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
261	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
262	int ret, value;
263
264	if (chip < 0 || WARN_ON_ONCE(chip >= meson_chip->nsels))
265	return;
266
267	nfc->param.chip_select = meson_chip->sels[chip] ? NAND_CE1 : NAND_CE0;
268	nfc->param.rb_select = nfc->param.chip_select;
269	nfc->timing.twb = meson_chip->twb;
270	nfc->timing.tadl = meson_chip->tadl;
271	nfc->timing.tbers_max = meson_chip->tbers_max;
272
273	if (nfc->clk_rate != meson_chip->clk_rate) {
274	ret = clk_set_rate(clk: nfc->nand_clk, rate: meson_chip->clk_rate);
275	if (ret) {
276	dev_err(nfc->dev, "failed to set clock rate\n");
277	return;
278	}
279	nfc->clk_rate = meson_chip->clk_rate;
280	}
281	if (nfc->bus_timing != meson_chip->bus_timing) {
282	value = (NFC_CLK_CYCLE - 1) | (meson_chip->bus_timing << 5);
283	writel(val: value, addr: nfc->reg_base + NFC_REG_CFG);
284	writel(val: (1 << 31), addr: nfc->reg_base + NFC_REG_CMD);
285	nfc->bus_timing = meson_chip->bus_timing;
286	}
287	}
288
289	static void meson_nfc_cmd_idle(struct meson_nfc *nfc, u32 time)
290	{
291	writel(val: nfc->param.chip_select | NFC_CMD_IDLE | (time & 0x3ff),
292	addr: nfc->reg_base + NFC_REG_CMD);
293	}
294
295	static void meson_nfc_cmd_seed(struct meson_nfc *nfc, u32 seed)
296	{
297	writel(NFC_CMD_SEED | (0xc2 + (seed & 0x7fff)),
298	addr: nfc->reg_base + NFC_REG_CMD);
299	}
300
301	static void meson_nfc_cmd_access(struct nand_chip *nand, int raw, bool dir,
302	int scrambler)
303	{
304	struct mtd_info *mtd = nand_to_mtd(chip: nand);
305	struct meson_nfc *nfc = nand_get_controller_data(chip: mtd_to_nand(mtd));
306	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
307	u32 bch = meson_chip->bch_mode, cmd;
308	int len = mtd->writesize, pagesize, pages;
309
310	pagesize = nand->ecc.size;
311
312	if (raw) {
313	len = mtd->writesize + mtd->oobsize;
314	cmd = len | scrambler | DMA_DIR(dir);
315	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
316	return;
317	}
318
319	pages = len / nand->ecc.size;
320
321	cmd = CMDRWGEN(DMA_DIR(dir), scrambler, bch,
322	NFC_CMD_SHORTMODE_DISABLE, pagesize, pages);
323
324	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
325	}
326
327	static void meson_nfc_drain_cmd(struct meson_nfc *nfc)
328	{
329	/*
330	* Insert two commands to make sure all valid commands are finished.
331	*
332	* The Nand flash controller is designed as two stages pipleline -
333	* a) fetch and b) excute.
334	* There might be cases when the driver see command queue is empty,
335	* but the Nand flash controller still has two commands buffered,
336	* one is fetched into NFC request queue (ready to run), and another
337	* is actively executing. So pushing 2 "IDLE" commands guarantees that
338	* the pipeline is emptied.
339	*/
340	meson_nfc_cmd_idle(nfc, time: 0);
341	meson_nfc_cmd_idle(nfc, time: 0);
342	}
343
344	static int meson_nfc_wait_cmd_finish(struct meson_nfc *nfc,
345	unsigned int timeout_ms)
346	{
347	u32 cmd_size = 0;
348	int ret;
349
350	/* wait cmd fifo is empty */
351	ret = readl_relaxed_poll_timeout(nfc->reg_base + NFC_REG_CMD, cmd_size,
352	!NFC_CMD_GET_SIZE(cmd_size),
353	10, timeout_ms * 1000);
354	if (ret)
355	dev_err(nfc->dev, "wait for empty CMD FIFO time out\n");
356
357	return ret;
358	}
359
360	static int meson_nfc_wait_dma_finish(struct meson_nfc *nfc)
361	{
362	meson_nfc_drain_cmd(nfc);
363
364	return meson_nfc_wait_cmd_finish(nfc, DMA_BUSY_TIMEOUT);
365	}
366
367	static u8 *meson_nfc_oob_ptr(struct nand_chip *nand, int i)
368	{
369	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
370	int len;
371
372	len = nand->ecc.size * (i + 1) + (nand->ecc.bytes + 2) * i;
373
374	return meson_chip->data_buf + len;
375	}
376
377	static u8 *meson_nfc_data_ptr(struct nand_chip *nand, int i)
378	{
379	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
380	int len, temp;
381
382	temp = nand->ecc.size + nand->ecc.bytes;
383	len = (temp + 2) * i;
384
385	return meson_chip->data_buf + len;
386	}
387
388	static void meson_nfc_get_data_oob(struct nand_chip *nand,
389	u8 *buf, u8 *oobbuf)
390	{
391	int i, oob_len = 0;
392	u8 *dsrc, *osrc;
393
394	oob_len = nand->ecc.bytes + 2;
395	for (i = 0; i < nand->ecc.steps; i++) {
396	if (buf) {
397	dsrc = meson_nfc_data_ptr(nand, i);
398	memcpy(buf, dsrc, nand->ecc.size);
399	buf += nand->ecc.size;
400	}
401	osrc = meson_nfc_oob_ptr(nand, i);
402	memcpy(oobbuf, osrc, oob_len);
403	oobbuf += oob_len;
404	}
405	}
406
407	static void meson_nfc_set_data_oob(struct nand_chip *nand,
408	const u8 *buf, u8 *oobbuf)
409	{
410	int i, oob_len = 0;
411	u8 *dsrc, *osrc;
412
413	oob_len = nand->ecc.bytes + 2;
414	for (i = 0; i < nand->ecc.steps; i++) {
415	if (buf) {
416	dsrc = meson_nfc_data_ptr(nand, i);
417	memcpy(dsrc, buf, nand->ecc.size);
418	buf += nand->ecc.size;
419	}
420	osrc = meson_nfc_oob_ptr(nand, i);
421	memcpy(osrc, oobbuf, oob_len);
422	oobbuf += oob_len;
423	}
424	}
425
426	static int meson_nfc_wait_no_rb_pin(struct nand_chip *nand, int timeout_ms,
427	bool need_cmd_read0)
428	{
429	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
430	u32 cmd, cfg;
431
432	meson_nfc_cmd_idle(nfc, time: nfc->timing.twb);
433	meson_nfc_drain_cmd(nfc);
434	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
435
436	cfg = readl(addr: nfc->reg_base + NFC_REG_CFG);
437	cfg |= NFC_RB_IRQ_EN;
438	writel(val: cfg, addr: nfc->reg_base + NFC_REG_CFG);
439
440	reinit_completion(x: &nfc->completion);
441	nand_status_op(chip: nand, NULL);
442
443	/* use the max erase time as the maximum clock for waiting R/B */
444	cmd = NFC_CMD_RB | NFC_CMD_RB_INT_NO_PIN | nfc->timing.tbers_max;
445	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
446
447	if (!wait_for_completion_timeout(x: &nfc->completion,
448	timeout: msecs_to_jiffies(m: timeout_ms)))
449	return -ETIMEDOUT;
450
451	if (need_cmd_read0)
452	nand_exit_status_op(chip: nand);
453
454	return 0;
455	}
456
457	static int meson_nfc_wait_rb_pin(struct meson_nfc *nfc, int timeout_ms)
458	{
459	u32 cmd, cfg;
460	int ret = 0;
461
462	meson_nfc_cmd_idle(nfc, time: nfc->timing.twb);
463	meson_nfc_drain_cmd(nfc);
464	meson_nfc_wait_cmd_finish(nfc, CMD_FIFO_EMPTY_TIMEOUT);
465
466	cfg = readl(addr: nfc->reg_base + NFC_REG_CFG);
467	cfg |= NFC_RB_IRQ_EN;
468	writel(val: cfg, addr: nfc->reg_base + NFC_REG_CFG);
469
470	reinit_completion(x: &nfc->completion);
471
472	/* use the max erase time as the maximum clock for waiting R/B */
473	cmd = NFC_CMD_RB | NFC_CMD_RB_INT
474	| nfc->param.chip_select | nfc->timing.tbers_max;
475	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
476
477	ret = wait_for_completion_timeout(x: &nfc->completion,
478	timeout: msecs_to_jiffies(m: timeout_ms));
479	if (ret == 0)
480	ret = -1;
481
482	return ret;
483	}
484
485	static int meson_nfc_queue_rb(struct nand_chip *nand, int timeout_ms,
486	bool need_cmd_read0)
487	{
488	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
489
490	if (nfc->no_rb_pin) {
491	/* This mode is used when there is no wired R/B pin.
492	* It works like 'nand_soft_waitrdy()', but instead of
493	* polling NAND_CMD_STATUS bit in the software loop,
494	* it will wait for interrupt - controllers checks IO
495	* bus and when it detects NAND_CMD_STATUS on it, it
496	* raises interrupt. After interrupt, NAND_CMD_READ0 is
497	* sent as terminator of the ready waiting procedure if
498	* needed (for all cases except page programming - this
499	* is reason of 'need_cmd_read0' flag).
500	*/
501	return meson_nfc_wait_no_rb_pin(nand, timeout_ms,
502	need_cmd_read0);
503	} else {
504	return meson_nfc_wait_rb_pin(nfc, timeout_ms);
505	}
506	}
507
508	static void meson_nfc_set_user_byte(struct nand_chip *nand, u8 *oob_buf)
509	{
510	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
511	__le64 *info;
512	int i, count;
513
514	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
515	info = &meson_chip->info_buf[i];
516	*info |= oob_buf[count];
517	*info |= oob_buf[count + 1] << 8;
518	}
519	}
520
521	static void meson_nfc_get_user_byte(struct nand_chip *nand, u8 *oob_buf)
522	{
523	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
524	__le64 *info;
525	int i, count;
526
527	for (i = 0, count = 0; i < nand->ecc.steps; i++, count += (2 + nand->ecc.bytes)) {
528	info = &meson_chip->info_buf[i];
529	oob_buf[count] = *info;
530	oob_buf[count + 1] = *info >> 8;
531	}
532	}
533
534	static int meson_nfc_ecc_correct(struct nand_chip *nand, u32 *bitflips,
535	u64 *correct_bitmap)
536	{
537	struct mtd_info *mtd = nand_to_mtd(chip: nand);
538	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
539	__le64 *info;
540	int ret = 0, i;
541
542	for (i = 0; i < nand->ecc.steps; i++) {
543	info = &meson_chip->info_buf[i];
544	if (ECC_ERR_CNT(*info) != ECC_UNCORRECTABLE) {
545	mtd->ecc_stats.corrected += ECC_ERR_CNT(*info);
546	*bitflips = max_t(u32, *bitflips, ECC_ERR_CNT(*info));
547	*correct_bitmap |= BIT_ULL(i);
548	continue;
549	}
550	if ((nand->options & NAND_NEED_SCRAMBLING) &&
551	ECC_ZERO_CNT(*info) < nand->ecc.strength) {
552	mtd->ecc_stats.corrected += ECC_ZERO_CNT(*info);
553	*bitflips = max_t(u32, *bitflips,
554	ECC_ZERO_CNT(*info));
555	ret = ECC_CHECK_RETURN_FF;
556	} else {
557	ret = -EBADMSG;
558	}
559	}
560	return ret;
561	}
562
563	static int meson_nfc_dma_buffer_setup(struct nand_chip *nand, void *databuf,
564	int datalen, void *infobuf, int infolen,
565	enum dma_data_direction dir)
566	{
567	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
568	u32 cmd;
569	int ret = 0;
570
571	nfc->daddr = dma_map_single(nfc->dev, databuf, datalen, dir);
572	ret = dma_mapping_error(dev: nfc->dev, dma_addr: nfc->daddr);
573	if (ret) {
574	dev_err(nfc->dev, "DMA mapping error\n");
575	return ret;
576	}
577	cmd = GENCMDDADDRL(NFC_CMD_ADL, nfc->daddr);
578	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
579
580	cmd = GENCMDDADDRH(NFC_CMD_ADH, nfc->daddr);
581	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
582
583	if (infobuf) {
584	nfc->iaddr = dma_map_single(nfc->dev, infobuf, infolen, dir);
585	ret = dma_mapping_error(dev: nfc->dev, dma_addr: nfc->iaddr);
586	if (ret) {
587	dev_err(nfc->dev, "DMA mapping error\n");
588	dma_unmap_single(nfc->dev,
589	nfc->daddr, datalen, dir);
590	return ret;
591	}
592	nfc->info_bytes = infolen;
593	cmd = GENCMDIADDRL(NFC_CMD_AIL, nfc->iaddr);
594	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
595
596	cmd = GENCMDIADDRH(NFC_CMD_AIH, nfc->iaddr);
597	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
598	}
599
600	return ret;
601	}
602
603	static void meson_nfc_dma_buffer_release(struct nand_chip *nand,
604	int datalen, int infolen,
605	enum dma_data_direction dir)
606	{
607	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
608
609	dma_unmap_single(nfc->dev, nfc->daddr, datalen, dir);
610	if (infolen) {
611	dma_unmap_single(nfc->dev, nfc->iaddr, infolen, dir);
612	nfc->info_bytes = 0;
613	}
614	}
615
616	static int meson_nfc_read_buf(struct nand_chip *nand, u8 *buf, int len)
617	{
618	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
619	int ret = 0;
620	u32 cmd;
621	u8 *info;
622
623	info = kzalloc(PER_INFO_BYTE, GFP_KERNEL);
624	if (!info)
625	return -ENOMEM;
626
627	ret = meson_nfc_dma_buffer_setup(nand, databuf: buf, datalen: len, infobuf: info,
628	PER_INFO_BYTE, dir: DMA_FROM_DEVICE);
629	if (ret)
630	goto out;
631
632	cmd = NFC_CMD_N2M | len;
633	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
634
635	meson_nfc_drain_cmd(nfc);
636	meson_nfc_wait_cmd_finish(nfc, timeout_ms: 1000);
637	meson_nfc_dma_buffer_release(nand, datalen: len, PER_INFO_BYTE, dir: DMA_FROM_DEVICE);
638
639	out:
640	kfree(objp: info);
641
642	return ret;
643	}
644
645	static int meson_nfc_write_buf(struct nand_chip *nand, u8 *buf, int len)
646	{
647	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
648	int ret = 0;
649	u32 cmd;
650
651	ret = meson_nfc_dma_buffer_setup(nand, databuf: buf, datalen: len, NULL,
652	infolen: 0, dir: DMA_TO_DEVICE);
653	if (ret)
654	return ret;
655
656	cmd = NFC_CMD_M2N | len;
657	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
658
659	meson_nfc_drain_cmd(nfc);
660	meson_nfc_wait_cmd_finish(nfc, timeout_ms: 1000);
661	meson_nfc_dma_buffer_release(nand, datalen: len, infolen: 0, dir: DMA_TO_DEVICE);
662
663	return ret;
664	}
665
666	static int meson_nfc_rw_cmd_prepare_and_execute(struct nand_chip *nand,
667	int page, bool in)
668	{
669	const struct nand_sdr_timings *sdr =
670	nand_get_sdr_timings(conf: nand_get_interface_config(chip: nand));
671	struct mtd_info *mtd = nand_to_mtd(chip: nand);
672	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
673	u32 *addrs = nfc->cmdfifo.rw.addrs;
674	u32 cs = nfc->param.chip_select;
675	u32 cmd0, cmd_num, row_start;
676	int i;
677
678	cmd_num = sizeof(struct nand_rw_cmd) / sizeof(int);
679
680	cmd0 = in ? NAND_CMD_READ0 : NAND_CMD_SEQIN;
681	nfc->cmdfifo.rw.cmd0 = cs | NFC_CMD_CLE | cmd0;
682
683	addrs[0] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_0;
684	if (mtd->writesize <= 512) {
685	cmd_num--;
686	row_start = 1;
687	} else {
688	addrs[1] = cs | NFC_CMD_ALE | NFC_COLUMN_ADDR_1;
689	row_start = 2;
690	}
691
692	addrs[row_start] = cs | NFC_CMD_ALE | ROW_ADDER(page, 0);
693	addrs[row_start + 1] = cs | NFC_CMD_ALE | ROW_ADDER(page, 1);
694
695	if (nand->options & NAND_ROW_ADDR_3)
696	addrs[row_start + 2] =
697	cs | NFC_CMD_ALE | ROW_ADDER(page, 2);
698	else
699	cmd_num--;
700
701	/* subtract cmd1 */
702	cmd_num--;
703
704	for (i = 0; i < cmd_num; i++)
705	writel_relaxed(nfc->cmdfifo.cmd[i],
706	nfc->reg_base + NFC_REG_CMD);
707
708	if (in) {
709	nfc->cmdfifo.rw.cmd1 = cs | NFC_CMD_CLE | NAND_CMD_READSTART;
710	writel(val: nfc->cmdfifo.rw.cmd1, addr: nfc->reg_base + NFC_REG_CMD);
711	meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tR_max), need_cmd_read0: true);
712	} else {
713	meson_nfc_cmd_idle(nfc, time: nfc->timing.tadl);
714	}
715
716	return 0;
717	}
718
719	static int meson_nfc_write_page_sub(struct nand_chip *nand,
720	int page, int raw)
721	{
722	const struct nand_sdr_timings *sdr =
723	nand_get_sdr_timings(conf: nand_get_interface_config(chip: nand));
724	struct mtd_info *mtd = nand_to_mtd(chip: nand);
725	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
726	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
727	int data_len, info_len;
728	u32 cmd;
729	int ret;
730
731	meson_nfc_select_chip(nand, chip: nand->cur_cs);
732
733	data_len = mtd->writesize + mtd->oobsize;
734	info_len = nand->ecc.steps * PER_INFO_BYTE;
735
736	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRWRITE);
737	if (ret)
738	return ret;
739
740	ret = meson_nfc_dma_buffer_setup(nand, databuf: meson_chip->data_buf,
741	datalen: data_len, infobuf: meson_chip->info_buf,
742	infolen: info_len, dir: DMA_TO_DEVICE);
743	if (ret)
744	return ret;
745
746	if (nand->options & NAND_NEED_SCRAMBLING) {
747	meson_nfc_cmd_seed(nfc, seed: page);
748	meson_nfc_cmd_access(nand, raw, DIRWRITE,
749	NFC_CMD_SCRAMBLER_ENABLE);
750	} else {
751	meson_nfc_cmd_access(nand, raw, DIRWRITE,
752	NFC_CMD_SCRAMBLER_DISABLE);
753	}
754
755	cmd = nfc->param.chip_select | NFC_CMD_CLE | NAND_CMD_PAGEPROG;
756	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
757	meson_nfc_queue_rb(nand, PSEC_TO_MSEC(sdr->tPROG_max), need_cmd_read0: false);
758
759	meson_nfc_dma_buffer_release(nand, datalen: data_len, infolen: info_len, dir: DMA_TO_DEVICE);
760
761	return ret;
762	}
763
764	static int meson_nfc_write_page_raw(struct nand_chip *nand, const u8 *buf,
765	int oob_required, int page)
766	{
767	u8 *oob_buf = nand->oob_poi;
768
769	meson_nfc_set_data_oob(nand, buf, oobbuf: oob_buf);
770
771	return meson_nfc_write_page_sub(nand, page, raw: 1);
772	}
773
774	static int meson_nfc_write_page_hwecc(struct nand_chip *nand,
775	const u8 *buf, int oob_required, int page)
776	{
777	struct mtd_info *mtd = nand_to_mtd(chip: nand);
778	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
779	u8 *oob_buf = nand->oob_poi;
780
781	memcpy(meson_chip->data_buf, buf, mtd->writesize);
782	memset(meson_chip->info_buf, 0, nand->ecc.steps * PER_INFO_BYTE);
783	meson_nfc_set_user_byte(nand, oob_buf);
784
785	return meson_nfc_write_page_sub(nand, page, raw: 0);
786	}
787
788	static void meson_nfc_check_ecc_pages_valid(struct meson_nfc *nfc,
789	struct nand_chip *nand, int raw)
790	{
791	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
792	__le64 *info;
793	u32 neccpages;
794	int ret;
795
796	neccpages = raw ? 1 : nand->ecc.steps;
797	info = &meson_chip->info_buf[neccpages - 1];
798	do {
799	usleep_range(min: 10, max: 15);
800	/* info is updated by nfc dma engine*/
801	smp_rmb();
802	dma_sync_single_for_cpu(dev: nfc->dev, addr: nfc->iaddr, size: nfc->info_bytes,
803	dir: DMA_FROM_DEVICE);
804	ret = *info & ECC_COMPLETE;
805	} while (!ret);
806	}
807
808	static int meson_nfc_read_page_sub(struct nand_chip *nand,
809	int page, int raw)
810	{
811	struct mtd_info *mtd = nand_to_mtd(chip: nand);
812	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
813	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
814	int data_len, info_len;
815	int ret;
816
817	meson_nfc_select_chip(nand, chip: nand->cur_cs);
818
819	data_len = mtd->writesize + mtd->oobsize;
820	info_len = nand->ecc.steps * PER_INFO_BYTE;
821
822	ret = meson_nfc_rw_cmd_prepare_and_execute(nand, page, DIRREAD);
823	if (ret)
824	return ret;
825
826	ret = meson_nfc_dma_buffer_setup(nand, databuf: meson_chip->data_buf,
827	datalen: data_len, infobuf: meson_chip->info_buf,
828	infolen: info_len, dir: DMA_FROM_DEVICE);
829	if (ret)
830	return ret;
831
832	if (nand->options & NAND_NEED_SCRAMBLING) {
833	meson_nfc_cmd_seed(nfc, seed: page);
834	meson_nfc_cmd_access(nand, raw, DIRREAD,
835	NFC_CMD_SCRAMBLER_ENABLE);
836	} else {
837	meson_nfc_cmd_access(nand, raw, DIRREAD,
838	NFC_CMD_SCRAMBLER_DISABLE);
839	}
840
841	ret = meson_nfc_wait_dma_finish(nfc);
842	meson_nfc_check_ecc_pages_valid(nfc, nand, raw);
843
844	meson_nfc_dma_buffer_release(nand, datalen: data_len, infolen: info_len, dir: DMA_FROM_DEVICE);
845
846	return ret;
847	}
848
849	static int meson_nfc_read_page_raw(struct nand_chip *nand, u8 *buf,
850	int oob_required, int page)
851	{
852	u8 *oob_buf = nand->oob_poi;
853	int ret;
854
855	ret = meson_nfc_read_page_sub(nand, page, raw: 1);
856	if (ret)
857	return ret;
858
859	meson_nfc_get_data_oob(nand, buf, oobbuf: oob_buf);
860
861	return 0;
862	}
863
864	static int meson_nfc_read_page_hwecc(struct nand_chip *nand, u8 *buf,
865	int oob_required, int page)
866	{
867	struct mtd_info *mtd = nand_to_mtd(chip: nand);
868	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
869	struct nand_ecc_ctrl *ecc = &nand->ecc;
870	u64 correct_bitmap = 0;
871	u32 bitflips = 0;
872	u8 *oob_buf = nand->oob_poi;
873	int ret, i;
874
875	ret = meson_nfc_read_page_sub(nand, page, raw: 0);
876	if (ret)
877	return ret;
878
879	meson_nfc_get_user_byte(nand, oob_buf);
880	ret = meson_nfc_ecc_correct(nand, bitflips: &bitflips, correct_bitmap: &correct_bitmap);
881	if (ret == ECC_CHECK_RETURN_FF) {
882	if (buf)
883	memset(buf, 0xff, mtd->writesize);
884	memset(oob_buf, 0xff, mtd->oobsize);
885	} else if (ret < 0) {
886	if ((nand->options & NAND_NEED_SCRAMBLING) || !buf) {
887	mtd->ecc_stats.failed++;
888	return bitflips;
889	}
890	ret = meson_nfc_read_page_raw(nand, buf, oob_required: 0, page);
891	if (ret)
892	return ret;
893
894	for (i = 0; i < nand->ecc.steps ; i++) {
895	u8 *data = buf + i * ecc->size;
896	u8 *oob = nand->oob_poi + i * (ecc->bytes + 2);
897
898	if (correct_bitmap & BIT_ULL(i))
899	continue;
900	ret = nand_check_erased_ecc_chunk(data, datalen: ecc->size,
901	ecc: oob, ecclen: ecc->bytes + 2,
902	NULL, extraooblen: 0,
903	threshold: ecc->strength);
904	if (ret < 0) {
905	mtd->ecc_stats.failed++;
906	} else {
907	mtd->ecc_stats.corrected += ret;
908	bitflips = max_t(u32, bitflips, ret);
909	}
910	}
911	} else if (buf && buf != meson_chip->data_buf) {
912	memcpy(buf, meson_chip->data_buf, mtd->writesize);
913	}
914
915	return bitflips;
916	}
917
918	static int meson_nfc_read_oob_raw(struct nand_chip *nand, int page)
919	{
920	return meson_nfc_read_page_raw(nand, NULL, oob_required: 1, page);
921	}
922
923	static int meson_nfc_read_oob(struct nand_chip *nand, int page)
924	{
925	return meson_nfc_read_page_hwecc(nand, NULL, oob_required: 1, page);
926	}
927
928	static bool meson_nfc_is_buffer_dma_safe(const void *buffer)
929	{
930	if ((uintptr_t)buffer % DMA_ADDR_ALIGN)
931	return false;
932
933	if (virt_addr_valid(buffer) && (!object_is_on_stack(obj: buffer)))
934	return true;
935	return false;
936	}
937
938	static void *
939	meson_nand_op_get_dma_safe_input_buf(const struct nand_op_instr *instr)
940	{
941	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR))
942	return NULL;
943
944	if (meson_nfc_is_buffer_dma_safe(buffer: instr->ctx.data.buf.in))
945	return instr->ctx.data.buf.in;
946
947	return kzalloc(size: instr->ctx.data.len, GFP_KERNEL);
948	}
949
950	static void
951	meson_nand_op_put_dma_safe_input_buf(const struct nand_op_instr *instr,
952	void *buf)
953	{
954	if (WARN_ON(instr->type != NAND_OP_DATA_IN_INSTR) ||
955	WARN_ON(!buf))
956	return;
957
958	if (buf == instr->ctx.data.buf.in)
959	return;
960
961	memcpy(instr->ctx.data.buf.in, buf, instr->ctx.data.len);
962	kfree(objp: buf);
963	}
964
965	static void *
966	meson_nand_op_get_dma_safe_output_buf(const struct nand_op_instr *instr)
967	{
968	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR))
969	return NULL;
970
971	if (meson_nfc_is_buffer_dma_safe(buffer: instr->ctx.data.buf.out))
972	return (void *)instr->ctx.data.buf.out;
973
974	return kmemdup(p: instr->ctx.data.buf.out,
975	size: instr->ctx.data.len, GFP_KERNEL);
976	}
977
978	static void
979	meson_nand_op_put_dma_safe_output_buf(const struct nand_op_instr *instr,
980	const void *buf)
981	{
982	if (WARN_ON(instr->type != NAND_OP_DATA_OUT_INSTR) ||
983	WARN_ON(!buf))
984	return;
985
986	if (buf != instr->ctx.data.buf.out)
987	kfree(objp: buf);
988	}
989
990	static int meson_nfc_check_op(struct nand_chip *chip,
991	const struct nand_operation *op)
992	{
993	int op_id;
994
995	for (op_id = 0; op_id < op->ninstrs; op_id++) {
996	const struct nand_op_instr *instr;
997
998	instr = &op->instrs[op_id];
999
1000	switch (instr->type) {
1001	case NAND_OP_DATA_IN_INSTR:
1002	case NAND_OP_DATA_OUT_INSTR:
1003	if (instr->ctx.data.len > NFC_CMD_RAW_LEN)
1004	return -ENOTSUPP;
1005
1006	break;
1007	default:
1008	break;
1009	}
1010	}
1011
1012	return 0;
1013	}
1014
1015	static int meson_nfc_exec_op(struct nand_chip *nand,
1016	const struct nand_operation *op, bool check_only)
1017	{
1018	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1019	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
1020	const struct nand_op_instr *instr = NULL;
1021	void *buf;
1022	u32 op_id, delay_idle, cmd;
1023	int err;
1024	int i;
1025
1026	err = meson_nfc_check_op(chip: nand, op);
1027	if (err)
1028	return err;
1029
1030	if (check_only)
1031	return 0;
1032
1033	meson_nfc_select_chip(nand, chip: op->cs);
1034	for (op_id = 0; op_id < op->ninstrs; op_id++) {
1035	instr = &op->instrs[op_id];
1036	delay_idle = DIV_ROUND_UP(PSEC_TO_NSEC(instr->delay_ns),
1037	meson_chip->level1_divider *
1038	NFC_CLK_CYCLE);
1039	switch (instr->type) {
1040	case NAND_OP_CMD_INSTR:
1041	cmd = nfc->param.chip_select | NFC_CMD_CLE;
1042	cmd |= instr->ctx.cmd.opcode & 0xff;
1043	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
1044	meson_nfc_cmd_idle(nfc, time: delay_idle);
1045	break;
1046
1047	case NAND_OP_ADDR_INSTR:
1048	for (i = 0; i < instr->ctx.addr.naddrs; i++) {
1049	cmd = nfc->param.chip_select | NFC_CMD_ALE;
1050	cmd |= instr->ctx.addr.addrs[i] & 0xff;
1051	writel(val: cmd, addr: nfc->reg_base + NFC_REG_CMD);
1052	}
1053	meson_nfc_cmd_idle(nfc, time: delay_idle);
1054	break;
1055
1056	case NAND_OP_DATA_IN_INSTR:
1057	buf = meson_nand_op_get_dma_safe_input_buf(instr);
1058	if (!buf)
1059	return -ENOMEM;
1060	meson_nfc_read_buf(nand, buf, len: instr->ctx.data.len);
1061	meson_nand_op_put_dma_safe_input_buf(instr, buf);
1062	break;
1063
1064	case NAND_OP_DATA_OUT_INSTR:
1065	buf = meson_nand_op_get_dma_safe_output_buf(instr);
1066	if (!buf)
1067	return -ENOMEM;
1068	meson_nfc_write_buf(nand, buf, len: instr->ctx.data.len);
1069	meson_nand_op_put_dma_safe_output_buf(instr, buf);
1070	break;
1071
1072	case NAND_OP_WAITRDY_INSTR:
1073	meson_nfc_queue_rb(nand, timeout_ms: instr->ctx.waitrdy.timeout_ms,
1074	need_cmd_read0: true);
1075	if (instr->delay_ns)
1076	meson_nfc_cmd_idle(nfc, time: delay_idle);
1077	break;
1078	}
1079	}
1080	meson_nfc_wait_cmd_finish(nfc, timeout_ms: 1000);
1081	return 0;
1082	}
1083
1084	static int meson_ooblayout_ecc(struct mtd_info *mtd, int section,
1085	struct mtd_oob_region *oobregion)
1086	{
1087	struct nand_chip *nand = mtd_to_nand(mtd);
1088
1089	if (section >= nand->ecc.steps)
1090	return -ERANGE;
1091
1092	oobregion->offset = 2 + (section * (2 + nand->ecc.bytes));
1093	oobregion->length = nand->ecc.bytes;
1094
1095	return 0;
1096	}
1097
1098	static int meson_ooblayout_free(struct mtd_info *mtd, int section,
1099	struct mtd_oob_region *oobregion)
1100	{
1101	struct nand_chip *nand = mtd_to_nand(mtd);
1102
1103	if (section >= nand->ecc.steps)
1104	return -ERANGE;
1105
1106	oobregion->offset = section * (2 + nand->ecc.bytes);
1107	oobregion->length = 2;
1108
1109	return 0;
1110	}
1111
1112	static const struct mtd_ooblayout_ops meson_ooblayout_ops = {
1113	.ecc = meson_ooblayout_ecc,
1114	.free = meson_ooblayout_free,
1115	};
1116
1117	static int meson_nfc_clk_init(struct meson_nfc *nfc)
1118	{
1119	struct clk_parent_data nfc_divider_parent_data[1] = {0};
1120	struct clk_init_data init = {0};
1121	int ret;
1122
1123	/* request core clock */
1124	nfc->core_clk = devm_clk_get(dev: nfc->dev, id: "core");
1125	if (IS_ERR(ptr: nfc->core_clk)) {
1126	dev_err(nfc->dev, "failed to get core clock\n");
1127	return PTR_ERR(ptr: nfc->core_clk);
1128	}
1129
1130	nfc->device_clk = devm_clk_get(dev: nfc->dev, id: "device");
1131	if (IS_ERR(ptr: nfc->device_clk)) {
1132	dev_err(nfc->dev, "failed to get device clock\n");
1133	return PTR_ERR(ptr: nfc->device_clk);
1134	}
1135
1136	init.name = devm_kasprintf(dev: nfc->dev,
1137	GFP_KERNEL, fmt: "%s#div",
1138	dev_name(dev: nfc->dev));
1139	if (!init.name)
1140	return -ENOMEM;
1141
1142	init.ops = &clk_divider_ops;
1143	nfc_divider_parent_data[0].fw_name = "device";
1144	init.parent_data = nfc_divider_parent_data;
1145	init.num_parents = 1;
1146	nfc->nand_divider.reg = nfc->reg_clk;
1147	nfc->nand_divider.shift = CLK_DIV_SHIFT;
1148	nfc->nand_divider.width = CLK_DIV_WIDTH;
1149	nfc->nand_divider.hw.init = &init;
1150	nfc->nand_divider.flags = CLK_DIVIDER_ONE_BASED |
1151	CLK_DIVIDER_ROUND_CLOSEST |
1152	CLK_DIVIDER_ALLOW_ZERO;
1153
1154	nfc->nand_clk = devm_clk_register(dev: nfc->dev, hw: &nfc->nand_divider.hw);
1155	if (IS_ERR(ptr: nfc->nand_clk))
1156	return PTR_ERR(ptr: nfc->nand_clk);
1157
1158	/* init SD_EMMC_CLOCK to sane defaults w/min clock rate */
1159	writel(CLK_ALWAYS_ON_NAND | CLK_SELECT_NAND | CLK_SELECT_FIX_PLL2,
1160	addr: nfc->reg_clk);
1161
1162	ret = clk_prepare_enable(clk: nfc->core_clk);
1163	if (ret) {
1164	dev_err(nfc->dev, "failed to enable core clock\n");
1165	return ret;
1166	}
1167
1168	ret = clk_prepare_enable(clk: nfc->device_clk);
1169	if (ret) {
1170	dev_err(nfc->dev, "failed to enable device clock\n");
1171	goto err_device_clk;
1172	}
1173
1174	ret = clk_prepare_enable(clk: nfc->nand_clk);
1175	if (ret) {
1176	dev_err(nfc->dev, "pre enable NFC divider fail\n");
1177	goto err_nand_clk;
1178	}
1179
1180	ret = clk_set_rate(clk: nfc->nand_clk, rate: 24000000);
1181	if (ret)
1182	goto err_disable_clk;
1183
1184	return 0;
1185
1186	err_disable_clk:
1187	clk_disable_unprepare(clk: nfc->nand_clk);
1188	err_nand_clk:
1189	clk_disable_unprepare(clk: nfc->device_clk);
1190	err_device_clk:
1191	clk_disable_unprepare(clk: nfc->core_clk);
1192	return ret;
1193	}
1194
1195	static void meson_nfc_disable_clk(struct meson_nfc *nfc)
1196	{
1197	clk_disable_unprepare(clk: nfc->nand_clk);
1198	clk_disable_unprepare(clk: nfc->device_clk);
1199	clk_disable_unprepare(clk: nfc->core_clk);
1200	}
1201
1202	static void meson_nfc_free_buffer(struct nand_chip *nand)
1203	{
1204	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1205
1206	kfree(objp: meson_chip->info_buf);
1207	kfree(objp: meson_chip->data_buf);
1208	}
1209
1210	static int meson_chip_buffer_init(struct nand_chip *nand)
1211	{
1212	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1213	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1214	u32 page_bytes, info_bytes, nsectors;
1215
1216	nsectors = mtd->writesize / nand->ecc.size;
1217
1218	page_bytes = mtd->writesize + mtd->oobsize;
1219	info_bytes = nsectors * PER_INFO_BYTE;
1220
1221	meson_chip->data_buf = kmalloc(size: page_bytes, GFP_KERNEL);
1222	if (!meson_chip->data_buf)
1223	return -ENOMEM;
1224
1225	meson_chip->info_buf = kmalloc(size: info_bytes, GFP_KERNEL);
1226	if (!meson_chip->info_buf) {
1227	kfree(objp: meson_chip->data_buf);
1228	return -ENOMEM;
1229	}
1230
1231	return 0;
1232	}
1233
1234	static
1235	int meson_nfc_setup_interface(struct nand_chip *nand, int csline,
1236	const struct nand_interface_config *conf)
1237	{
1238	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1239	const struct nand_sdr_timings *timings;
1240	u32 div, bt_min, bt_max, tbers_clocks;
1241
1242	timings = nand_get_sdr_timings(conf);
1243	if (IS_ERR(ptr: timings))
1244	return -ENOTSUPP;
1245
1246	if (csline == NAND_DATA_IFACE_CHECK_ONLY)
1247	return 0;
1248
1249	div = DIV_ROUND_UP((timings->tRC_min / 1000), NFC_CLK_CYCLE);
1250	bt_min = (timings->tREA_max + NFC_DEFAULT_DELAY) / div;
1251	bt_max = (NFC_DEFAULT_DELAY + timings->tRHOH_min +
1252	timings->tRC_min / 2) / div;
1253
1254	meson_chip->twb = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tWB_max),
1255	div * NFC_CLK_CYCLE);
1256	meson_chip->tadl = DIV_ROUND_UP(PSEC_TO_NSEC(timings->tADL_min),
1257	div * NFC_CLK_CYCLE);
1258	tbers_clocks = DIV_ROUND_UP_ULL(PSEC_TO_NSEC(timings->tBERS_max),
1259	div * NFC_CLK_CYCLE);
1260	meson_chip->tbers_max = ilog2(tbers_clocks);
1261	if (!is_power_of_2(n: tbers_clocks))
1262	meson_chip->tbers_max++;
1263
1264	bt_min = DIV_ROUND_UP(bt_min, 1000);
1265	bt_max = DIV_ROUND_UP(bt_max, 1000);
1266
1267	if (bt_max < bt_min)
1268	return -EINVAL;
1269
1270	meson_chip->level1_divider = div;
1271	meson_chip->clk_rate = 1000000000 / meson_chip->level1_divider;
1272	meson_chip->bus_timing = (bt_min + bt_max) / 2 + 1;
1273
1274	return 0;
1275	}
1276
1277	static int meson_nand_bch_mode(struct nand_chip *nand)
1278	{
1279	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1280	int i;
1281
1282	if (nand->ecc.strength > 60 || nand->ecc.strength < 8)
1283	return -EINVAL;
1284
1285	for (i = 0; i < ARRAY_SIZE(meson_ecc); i++) {
1286	if (meson_ecc[i].strength == nand->ecc.strength &&
1287	meson_ecc[i].size == nand->ecc.size) {
1288	meson_chip->bch_mode = meson_ecc[i].bch;
1289	return 0;
1290	}
1291	}
1292
1293	return -EINVAL;
1294	}
1295
1296	static void meson_nand_detach_chip(struct nand_chip *nand)
1297	{
1298	meson_nfc_free_buffer(nand);
1299	}
1300
1301	static int meson_nand_attach_chip(struct nand_chip *nand)
1302	{
1303	struct meson_nfc *nfc = nand_get_controller_data(chip: nand);
1304	struct meson_nfc_nand_chip *meson_chip = to_meson_nand(nand);
1305	struct mtd_info *mtd = nand_to_mtd(chip: nand);
1306	int raw_writesize;
1307	int ret;
1308
1309	if (!mtd->name) {
1310	mtd->name = devm_kasprintf(dev: nfc->dev, GFP_KERNEL,
1311	fmt: "%s:nand%d",
1312	dev_name(dev: nfc->dev),
1313	meson_chip->sels[0]);
1314	if (!mtd->name)
1315	return -ENOMEM;
1316	}
1317
1318	raw_writesize = mtd->writesize + mtd->oobsize;
1319	if (raw_writesize > NFC_CMD_RAW_LEN) {
1320	dev_err(nfc->dev, "too big write size in raw mode: %d > %ld\n",
1321	raw_writesize, NFC_CMD_RAW_LEN);
1322	return -EINVAL;
1323	}
1324
1325	if (nand->bbt_options & NAND_BBT_USE_FLASH)
1326	nand->bbt_options |= NAND_BBT_NO_OOB;
1327
1328	nand->options |= NAND_NO_SUBPAGE_WRITE;
1329
1330	ret = nand_ecc_choose_conf(chip: nand, caps: nfc->data->ecc_caps,
1331	oobavail: mtd->oobsize - 2);
1332	if (ret) {
1333	dev_err(nfc->dev, "failed to ECC init\n");
1334	return -EINVAL;
1335	}
1336
1337	mtd_set_ooblayout(mtd, ooblayout: &meson_ooblayout_ops);
1338
1339	ret = meson_nand_bch_mode(nand);
1340	if (ret)
1341	return -EINVAL;
1342
1343	nand->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1344	nand->ecc.write_page_raw = meson_nfc_write_page_raw;
1345	nand->ecc.write_page = meson_nfc_write_page_hwecc;
1346	nand->ecc.write_oob_raw = nand_write_oob_std;
1347	nand->ecc.write_oob = nand_write_oob_std;
1348
1349	nand->ecc.read_page_raw = meson_nfc_read_page_raw;
1350	nand->ecc.read_page = meson_nfc_read_page_hwecc;
1351	nand->ecc.read_oob_raw = meson_nfc_read_oob_raw;
1352	nand->ecc.read_oob = meson_nfc_read_oob;
1353
1354	if (nand->options & NAND_BUSWIDTH_16) {
1355	dev_err(nfc->dev, "16bits bus width not supported");
1356	return -EINVAL;
1357	}
1358	ret = meson_chip_buffer_init(nand);
1359	if (ret)
1360	return -ENOMEM;
1361
1362	return ret;
1363	}
1364
1365	static const struct nand_controller_ops meson_nand_controller_ops = {
1366	.attach_chip = meson_nand_attach_chip,
1367	.detach_chip = meson_nand_detach_chip,
1368	.setup_interface = meson_nfc_setup_interface,
1369	.exec_op = meson_nfc_exec_op,
1370	};
1371
1372	static int
1373	meson_nfc_nand_chip_init(struct device *dev,
1374	struct meson_nfc *nfc, struct device_node *np)
1375	{
1376	struct meson_nfc_nand_chip *meson_chip;
1377	struct nand_chip *nand;
1378	struct mtd_info *mtd;
1379	int ret, i;
1380	u32 tmp, nsels;
1381	u32 nand_rb_val = 0;
1382
1383	nsels = of_property_count_elems_of_size(np, propname: "reg", elem_size: sizeof(u32));
1384	if (!nsels || nsels > MAX_CE_NUM) {
1385	dev_err(dev, "invalid register property size\n");
1386	return -EINVAL;
1387	}
1388
1389	meson_chip = devm_kzalloc(dev, struct_size(meson_chip, sels, nsels),
1390	GFP_KERNEL);
1391	if (!meson_chip)
1392	return -ENOMEM;
1393
1394	meson_chip->nsels = nsels;
1395
1396	for (i = 0; i < nsels; i++) {
1397	ret = of_property_read_u32_index(np, propname: "reg", index: i, out_value: &tmp);
1398	if (ret) {
1399	dev_err(dev, "could not retrieve register property: %d\n",
1400	ret);
1401	return ret;
1402	}
1403
1404	if (test_and_set_bit(nr: tmp, addr: &nfc->assigned_cs)) {
1405	dev_err(dev, "CS %d already assigned\n", tmp);
1406	return -EINVAL;
1407	}
1408	}
1409
1410	nand = &meson_chip->nand;
1411	nand->controller = &nfc->controller;
1412	nand->controller->ops = &meson_nand_controller_ops;
1413	nand_set_flash_node(chip: nand, np);
1414	nand_set_controller_data(chip: nand, priv: nfc);
1415
1416	nand->options |= NAND_USES_DMA;
1417	mtd = nand_to_mtd(chip: nand);
1418	mtd->owner = THIS_MODULE;
1419	mtd->dev.parent = dev;
1420
1421	ret = of_property_read_u32(np, propname: "nand-rb", out_value: &nand_rb_val);
1422	if (ret == -EINVAL)
1423	nfc->no_rb_pin = true;
1424	else if (ret)
1425	return ret;
1426
1427	if (nand_rb_val)
1428	return -EINVAL;
1429
1430	ret = nand_scan(chip: nand, max_chips: nsels);
1431	if (ret)
1432	return ret;
1433
1434	ret = mtd_device_register(mtd, NULL, 0);
1435	if (ret) {
1436	dev_err(dev, "failed to register MTD device: %d\n", ret);
1437	nand_cleanup(chip: nand);
1438	return ret;
1439	}
1440
1441	list_add_tail(new: &meson_chip->node, head: &nfc->chips);
1442
1443	return 0;
1444	}
1445
1446	static void meson_nfc_nand_chip_cleanup(struct meson_nfc *nfc)
1447	{
1448	struct meson_nfc_nand_chip *meson_chip;
1449	struct mtd_info *mtd;
1450
1451	while (!list_empty(head: &nfc->chips)) {
1452	meson_chip = list_first_entry(&nfc->chips,
1453	struct meson_nfc_nand_chip, node);
1454	mtd = nand_to_mtd(chip: &meson_chip->nand);
1455	WARN_ON(mtd_device_unregister(mtd));
1456
1457	nand_cleanup(chip: &meson_chip->nand);
1458	list_del(entry: &meson_chip->node);
1459	}
1460	}
1461
1462	static int meson_nfc_nand_chips_init(struct device *dev,
1463	struct meson_nfc *nfc)
1464	{
1465	struct device_node *np = dev->of_node;
1466	struct device_node *nand_np;
1467	int ret;
1468
1469	for_each_child_of_node(np, nand_np) {
1470	ret = meson_nfc_nand_chip_init(dev, nfc, np: nand_np);
1471	if (ret) {
1472	meson_nfc_nand_chip_cleanup(nfc);
1473	of_node_put(node: nand_np);
1474	return ret;
1475	}
1476	}
1477
1478	return 0;
1479	}
1480
1481	static irqreturn_t meson_nfc_irq(int irq, void *id)
1482	{
1483	struct meson_nfc *nfc = id;
1484	u32 cfg;
1485
1486	cfg = readl(addr: nfc->reg_base + NFC_REG_CFG);
1487	if (!(cfg & NFC_RB_IRQ_EN))
1488	return IRQ_NONE;
1489
1490	cfg &= ~(NFC_RB_IRQ_EN);
1491	writel(val: cfg, addr: nfc->reg_base + NFC_REG_CFG);
1492
1493	complete(&nfc->completion);
1494	return IRQ_HANDLED;
1495	}
1496
1497	static const struct meson_nfc_data meson_gxl_data = {
1498	.ecc_caps = &meson_gxl_ecc_caps,
1499	};
1500
1501	static const struct meson_nfc_data meson_axg_data = {
1502	.ecc_caps = &meson_axg_ecc_caps,
1503	};
1504
1505	static const struct of_device_id meson_nfc_id_table[] = {
1506	{
1507	.compatible = "amlogic,meson-gxl-nfc",
1508	.data = &meson_gxl_data,
1509	}, {
1510	.compatible = "amlogic,meson-axg-nfc",
1511	.data = &meson_axg_data,
1512	},
1513	{}
1514	};
1515	MODULE_DEVICE_TABLE(of, meson_nfc_id_table);
1516
1517	static int meson_nfc_probe(struct platform_device *pdev)
1518	{
1519	struct device *dev = &pdev->dev;
1520	struct meson_nfc *nfc;
1521	int ret, irq;
1522
1523	nfc = devm_kzalloc(dev, size: sizeof(*nfc), GFP_KERNEL);
1524	if (!nfc)
1525	return -ENOMEM;
1526
1527	nfc->data = of_device_get_match_data(dev: &pdev->dev);
1528	if (!nfc->data)
1529	return -ENODEV;
1530
1531	nand_controller_init(nfc: &nfc->controller);
1532	INIT_LIST_HEAD(list: &nfc->chips);
1533	init_completion(x: &nfc->completion);
1534
1535	nfc->dev = dev;
1536
1537	nfc->reg_base = devm_platform_ioremap_resource_byname(pdev, name: "nfc");
1538	if (IS_ERR(ptr: nfc->reg_base))
1539	return PTR_ERR(ptr: nfc->reg_base);
1540
1541	nfc->reg_clk = devm_platform_ioremap_resource_byname(pdev, name: "emmc");
1542	if (IS_ERR(ptr: nfc->reg_clk))
1543	return PTR_ERR(ptr: nfc->reg_clk);
1544
1545	irq = platform_get_irq(pdev, 0);
1546	if (irq < 0)
1547	return -EINVAL;
1548
1549	ret = meson_nfc_clk_init(nfc);
1550	if (ret) {
1551	dev_err(dev, "failed to initialize NAND clock\n");
1552	return ret;
1553	}
1554
1555	writel(val: 0, addr: nfc->reg_base + NFC_REG_CFG);
1556	ret = devm_request_irq(dev, irq, handler: meson_nfc_irq, irqflags: 0, devname: dev_name(dev), dev_id: nfc);
1557	if (ret) {
1558	dev_err(dev, "failed to request NFC IRQ\n");
1559	ret = -EINVAL;
1560	goto err_clk;
1561	}
1562
1563	ret = dma_set_mask(dev, DMA_BIT_MASK(32));
1564	if (ret) {
1565	dev_err(dev, "failed to set DMA mask\n");
1566	goto err_clk;
1567	}
1568
1569	platform_set_drvdata(pdev, data: nfc);
1570
1571	ret = meson_nfc_nand_chips_init(dev, nfc);
1572	if (ret) {
1573	dev_err(dev, "failed to init NAND chips\n");
1574	goto err_clk;
1575	}
1576
1577	return 0;
1578	err_clk:
1579	meson_nfc_disable_clk(nfc);
1580	return ret;
1581	}
1582
1583	static void meson_nfc_remove(struct platform_device *pdev)
1584	{
1585	struct meson_nfc *nfc = platform_get_drvdata(pdev);
1586
1587	meson_nfc_nand_chip_cleanup(nfc);
1588
1589	meson_nfc_disable_clk(nfc);
1590	}
1591
1592	static struct platform_driver meson_nfc_driver = {
1593	.probe = meson_nfc_probe,
1594	.remove_new = meson_nfc_remove,
1595	.driver = {
1596	.name = "meson-nand",
1597	.of_match_table = meson_nfc_id_table,
1598	},
1599	};
1600	module_platform_driver(meson_nfc_driver);
1601
1602	MODULE_LICENSE("Dual MIT/GPL");
1603	MODULE_AUTHOR("Liang Yang <liang.yang@amlogic.com>");
1604	MODULE_DESCRIPTION("Amlogic's Meson NAND Flash Controller driver");
1605