1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Marvell NAND flash controller driver
4	*
5	* Copyright (C) 2017 Marvell
6	* Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7	*
8	*
9	* This NAND controller driver handles two versions of the hardware,
10	* one is called NFCv1 and is available on PXA SoCs and the other is
11	* called NFCv2 and is available on Armada SoCs.
12	*
13	* The main visible difference is that NFCv1 only has Hamming ECC
14	* capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15	* is not used with NFCv2.
16	*
17	* The ECC layouts are depicted in details in Marvell AN-379, but here
18	* is a brief description.
19	*
20	* When using Hamming, the data is split in 512B chunks (either 1, 2
21	* or 4) and each chunk will have its own ECC "digest" of 6B at the
22	* beginning of the OOB area and eventually the remaining free OOB
23	* bytes (also called "spare" bytes in the driver). This engine
24	* corrects up to 1 bit per chunk and detects reliably an error if
25	* there are at most 2 bitflips. Here is the page layout used by the
26	* controller when Hamming is chosen:
27	*
28	* +-------------------------------------------------------------+
29	* | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30	* +-------------------------------------------------------------+
31	*
32	* When using the BCH engine, there are N identical (data + free OOB +
33	* ECC) sections and potentially an extra one to deal with
34	* configurations where the chosen (data + free OOB + ECC) sizes do
35	* not align with the page (data + OOB) size. ECC bytes are always
36	* 30B per ECC chunk. Here is the page layout used by the controller
37	* when BCH is chosen:
38	*
39	* +-----------------------------------------
40	* | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41	* +-----------------------------------------
42	*
43	* -------------------------------------------
44	* ... | Data N | Free OOB bytes N | ECC N |
45	* -------------------------------------------
46	*
47	* --------------------------------------------+
48	* Last Data | Last Free OOB bytes | Last ECC |
49	* --------------------------------------------+
50	*
51	* In both cases, the layout seen by the user is always: all data
52	* first, then all free OOB bytes and finally all ECC bytes. With BCH,
53	* ECC bytes are 30B long and are padded with 0xFF to align on 32
54	* bytes.
55	*
56	* The controller has certain limitations that are handled by the
57	* driver:
58	* - It can only read 2k at a time. To overcome this limitation, the
59	* driver issues data cycles on the bus, without issuing new
60	* CMD + ADDR cycles. The Marvell term is "naked" operations.
61	* - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62	* bits. What can be tuned is the ECC block size as long as it
63	* stays between 512B and 2kiB. It's usually chosen based on the
64	* chip ECC requirements. For instance, using 2kiB ECC chunks
65	* provides 4b/512B correctability.
66	* - The controller will always treat data bytes, free OOB bytes
67	* and ECC bytes in that order, no matter what the real layout is
68	* (which is usually all data then all OOB bytes). The
69	* marvell_nfc_layouts array below contains the currently
70	* supported layouts.
71	* - Because of these weird layouts, the Bad Block Markers can be
72	* located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73	* option must be set to prevent scanning/writing bad block
74	* markers.
75	*/
76
77	#include <linux/module.h>
78	#include <linux/clk.h>
79	#include <linux/mtd/rawnand.h>
80	#include <linux/of.h>
81	#include <linux/iopoll.h>
82	#include <linux/interrupt.h>
83	#include <linux/platform_device.h>
84	#include <linux/slab.h>
85	#include <linux/mfd/syscon.h>
86	#include <linux/regmap.h>
87	#include <asm/unaligned.h>
88
89	#include <linux/dmaengine.h>
90	#include <linux/dma-mapping.h>
91	#include <linux/dma/pxa-dma.h>
92	#include <linux/platform_data/mtd-nand-pxa3xx.h>
93
94	/* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
95	#define FIFO_DEPTH 8
96	#define FIFO_REP(x) (x / sizeof(u32))
97	#define BCH_SEQ_READS (32 / FIFO_DEPTH)
98	/* NFC does not support transfers of larger chunks at a time */
99	#define MAX_CHUNK_SIZE 2112
100	/* NFCv1 cannot read more that 7 bytes of ID */
101	#define NFCV1_READID_LEN 7
102	/* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
103	#define POLL_PERIOD 0
104	#define POLL_TIMEOUT 100000
105	/* Interrupt maximum wait period in ms */
106	#define IRQ_TIMEOUT 1000
107	/* Latency in clock cycles between SoC pins and NFC logic */
108	#define MIN_RD_DEL_CNT 3
109	/* Maximum number of contiguous address cycles */
110	#define MAX_ADDRESS_CYC_NFCV1 5
111	#define MAX_ADDRESS_CYC_NFCV2 7
112	/* System control registers/bits to enable the NAND controller on some SoCs */
113	#define GENCONF_SOC_DEVICE_MUX 0x208
114	#define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
115	#define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
116	#define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
117	#define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
118	#define GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN BIT(27)
119	#define GENCONF_CLK_GATING_CTRL 0x220
120	#define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
121	#define GENCONF_ND_CLK_CTRL 0x700
122	#define GENCONF_ND_CLK_CTRL_EN BIT(0)
123
124	/* NAND controller data flash control register */
125	#define NDCR 0x00
126	#define NDCR_ALL_INT GENMASK(11, 0)
127	#define NDCR_CS1_CMDDM BIT(7)
128	#define NDCR_CS0_CMDDM BIT(8)
129	#define NDCR_RDYM BIT(11)
130	#define NDCR_ND_ARB_EN BIT(12)
131	#define NDCR_RA_START BIT(15)
132	#define NDCR_RD_ID_CNT(x) (min_t(unsigned int, x, 0x7) << 16)
133	#define NDCR_PAGE_SZ(x) (x >= 2048 ? BIT(24) : 0)
134	#define NDCR_DWIDTH_M BIT(26)
135	#define NDCR_DWIDTH_C BIT(27)
136	#define NDCR_ND_RUN BIT(28)
137	#define NDCR_DMA_EN BIT(29)
138	#define NDCR_ECC_EN BIT(30)
139	#define NDCR_SPARE_EN BIT(31)
140	#define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
141	NDCR_DWIDTH_M | NDCR_DWIDTH_C))
142
143	/* NAND interface timing parameter 0 register */
144	#define NDTR0 0x04
145	#define NDTR0_TRP(x) ((min_t(unsigned int, x, 0xF) & 0x7) << 0)
146	#define NDTR0_TRH(x) (min_t(unsigned int, x, 0x7) << 3)
147	#define NDTR0_ETRP(x) ((min_t(unsigned int, x, 0xF) & 0x8) << 3)
148	#define NDTR0_SEL_NRE_EDGE BIT(7)
149	#define NDTR0_TWP(x) (min_t(unsigned int, x, 0x7) << 8)
150	#define NDTR0_TWH(x) (min_t(unsigned int, x, 0x7) << 11)
151	#define NDTR0_TCS(x) (min_t(unsigned int, x, 0x7) << 16)
152	#define NDTR0_TCH(x) (min_t(unsigned int, x, 0x7) << 19)
153	#define NDTR0_RD_CNT_DEL(x) (min_t(unsigned int, x, 0xF) << 22)
154	#define NDTR0_SELCNTR BIT(26)
155	#define NDTR0_TADL(x) (min_t(unsigned int, x, 0x1F) << 27)
156
157	/* NAND interface timing parameter 1 register */
158	#define NDTR1 0x0C
159	#define NDTR1_TAR(x) (min_t(unsigned int, x, 0xF) << 0)
160	#define NDTR1_TWHR(x) (min_t(unsigned int, x, 0xF) << 4)
161	#define NDTR1_TRHW(x) (min_t(unsigned int, x / 16, 0x3) << 8)
162	#define NDTR1_PRESCALE BIT(14)
163	#define NDTR1_WAIT_MODE BIT(15)
164	#define NDTR1_TR(x) (min_t(unsigned int, x, 0xFFFF) << 16)
165
166	/* NAND controller status register */
167	#define NDSR 0x14
168	#define NDSR_WRCMDREQ BIT(0)
169	#define NDSR_RDDREQ BIT(1)
170	#define NDSR_WRDREQ BIT(2)
171	#define NDSR_CORERR BIT(3)
172	#define NDSR_UNCERR BIT(4)
173	#define NDSR_CMDD(cs) BIT(8 - cs)
174	#define NDSR_RDY(rb) BIT(11 + rb)
175	#define NDSR_ERRCNT(x) ((x >> 16) & 0x1F)
176
177	/* NAND ECC control register */
178	#define NDECCCTRL 0x28
179	#define NDECCCTRL_BCH_EN BIT(0)
180
181	/* NAND controller data buffer register */
182	#define NDDB 0x40
183
184	/* NAND controller command buffer 0 register */
185	#define NDCB0 0x48
186	#define NDCB0_CMD1(x) ((x & 0xFF) << 0)
187	#define NDCB0_CMD2(x) ((x & 0xFF) << 8)
188	#define NDCB0_ADDR_CYC(x) ((x & 0x7) << 16)
189	#define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
190	#define NDCB0_DBC BIT(19)
191	#define NDCB0_CMD_TYPE(x) ((x & 0x7) << 21)
192	#define NDCB0_CSEL BIT(24)
193	#define NDCB0_RDY_BYP BIT(27)
194	#define NDCB0_LEN_OVRD BIT(28)
195	#define NDCB0_CMD_XTYPE(x) ((x & 0x7) << 29)
196
197	/* NAND controller command buffer 1 register */
198	#define NDCB1 0x4C
199	#define NDCB1_COLS(x) ((x & 0xFFFF) << 0)
200	#define NDCB1_ADDRS_PAGE(x) (x << 16)
201
202	/* NAND controller command buffer 2 register */
203	#define NDCB2 0x50
204	#define NDCB2_ADDR5_PAGE(x) (((x >> 16) & 0xFF) << 0)
205	#define NDCB2_ADDR5_CYC(x) ((x & 0xFF) << 0)
206
207	/* NAND controller command buffer 3 register */
208	#define NDCB3 0x54
209	#define NDCB3_ADDR6_CYC(x) ((x & 0xFF) << 16)
210	#define NDCB3_ADDR7_CYC(x) ((x & 0xFF) << 24)
211
212	/* NAND controller command buffer 0 register 'type' and 'xtype' fields */
213	#define TYPE_READ 0
214	#define TYPE_WRITE 1
215	#define TYPE_ERASE 2
216	#define TYPE_READ_ID 3
217	#define TYPE_STATUS 4
218	#define TYPE_RESET 5
219	#define TYPE_NAKED_CMD 6
220	#define TYPE_NAKED_ADDR 7
221	#define TYPE_MASK 7
222	#define XTYPE_MONOLITHIC_RW 0
223	#define XTYPE_LAST_NAKED_RW 1
224	#define XTYPE_FINAL_COMMAND 3
225	#define XTYPE_READ 4
226	#define XTYPE_WRITE_DISPATCH 4
227	#define XTYPE_NAKED_RW 5
228	#define XTYPE_COMMAND_DISPATCH 6
229	#define XTYPE_MASK 7
230
231	/**
232	* struct marvell_hw_ecc_layout - layout of Marvell ECC
233	*
234	* Marvell ECC engine works differently than the others, in order to limit the
235	* size of the IP, hardware engineers chose to set a fixed strength at 16 bits
236	* per subpage, and depending on a the desired strength needed by the NAND chip,
237	* a particular layout mixing data/spare/ecc is defined, with a possible last
238	* chunk smaller that the others.
239	*
240	* @writesize: Full page size on which the layout applies
241	* @chunk: Desired ECC chunk size on which the layout applies
242	* @strength: Desired ECC strength (per chunk size bytes) on which the
243	* layout applies
244	* @nchunks: Total number of chunks
245	* @full_chunk_cnt: Number of full-sized chunks, which is the number of
246	* repetitions of the pattern:
247	* (data_bytes + spare_bytes + ecc_bytes).
248	* @data_bytes: Number of data bytes per chunk
249	* @spare_bytes: Number of spare bytes per chunk
250	* @ecc_bytes: Number of ecc bytes per chunk
251	* @last_data_bytes: Number of data bytes in the last chunk
252	* @last_spare_bytes: Number of spare bytes in the last chunk
253	* @last_ecc_bytes: Number of ecc bytes in the last chunk
254	*/
255	struct marvell_hw_ecc_layout {
256	/* Constraints */
257	int writesize;
258	int chunk;
259	int strength;
260	/* Corresponding layout */
261	int nchunks;
262	int full_chunk_cnt;
263	int data_bytes;
264	int spare_bytes;
265	int ecc_bytes;
266	int last_data_bytes;
267	int last_spare_bytes;
268	int last_ecc_bytes;
269	};
270
271	#define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb) \
272	{ \
273	.writesize = ws, \
274	.chunk = dc, \
275	.strength = ds, \
276	.nchunks = nc, \
277	.full_chunk_cnt = fcc, \
278	.data_bytes = db, \
279	.spare_bytes = sb, \
280	.ecc_bytes = eb, \
281	.last_data_bytes = ldb, \
282	.last_spare_bytes = lsb, \
283	.last_ecc_bytes = leb, \
284	}
285
286	/* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
287	static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
288	MARVELL_LAYOUT( 512, 512, 1, 1, 1, 512, 8, 8, 0, 0, 0),
289	MARVELL_LAYOUT( 2048, 512, 1, 1, 1, 2048, 40, 24, 0, 0, 0),
290	MARVELL_LAYOUT( 2048, 512, 4, 1, 1, 2048, 32, 30, 0, 0, 0),
291	MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,32, 30),
292	MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,64, 30),
293	MARVELL_LAYOUT( 2048, 512, 16, 4, 4, 512, 0, 30, 0, 32, 30),
294	MARVELL_LAYOUT( 4096, 512, 4, 2, 2, 2048, 32, 30, 0, 0, 0),
295	MARVELL_LAYOUT( 4096, 512, 8, 4, 4, 1024, 0, 30, 0, 64, 30),
296	MARVELL_LAYOUT( 4096, 512, 16, 8, 8, 512, 0, 30, 0, 32, 30),
297	MARVELL_LAYOUT( 8192, 512, 4, 4, 4, 2048, 0, 30, 0, 0, 0),
298	MARVELL_LAYOUT( 8192, 512, 8, 8, 8, 1024, 0, 30, 0, 160, 30),
299	MARVELL_LAYOUT( 8192, 512, 16, 16, 16, 512, 0, 30, 0, 32, 30),
300	};
301
302	/**
303	* struct marvell_nand_chip_sel - CS line description
304	*
305	* The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
306	* is made by a field in NDCB0 register, and in another field in NDCB2 register.
307	* The datasheet describes the logic with an error: ADDR5 field is once
308	* declared at the beginning of NDCB2, and another time at its end. Because the
309	* ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
310	* to use the last bit of this field instead of the first ones.
311	*
312	* @cs: Wanted CE lane.
313	* @ndcb0_csel: Value of the NDCB0 register with or without the flag
314	* selecting the wanted CE lane. This is set once when
315	* the Device Tree is probed.
316	* @rb: Ready/Busy pin for the flash chip
317	*/
318	struct marvell_nand_chip_sel {
319	unsigned int cs;
320	u32 ndcb0_csel;
321	unsigned int rb;
322	};
323
324	/**
325	* struct marvell_nand_chip - stores NAND chip device related information
326	*
327	* @chip: Base NAND chip structure
328	* @node: Used to store NAND chips into a list
329	* @layout: NAND layout when using hardware ECC
330	* @ndcr: Controller register value for this NAND chip
331	* @ndtr0: Timing registers 0 value for this NAND chip
332	* @ndtr1: Timing registers 1 value for this NAND chip
333	* @addr_cyc: Amount of cycles needed to pass column address
334	* @selected_die: Current active CS
335	* @nsels: Number of CS lines required by the NAND chip
336	* @sels: Array of CS lines descriptions
337	*/
338	struct marvell_nand_chip {
339	struct nand_chip chip;
340	struct list_head node;
341	const struct marvell_hw_ecc_layout *layout;
342	u32 ndcr;
343	u32 ndtr0;
344	u32 ndtr1;
345	int addr_cyc;
346	int selected_die;
347	unsigned int nsels;
348	struct marvell_nand_chip_sel sels[] __counted_by(nsels);
349	};
350
351	static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
352	{
353	return container_of(chip, struct marvell_nand_chip, chip);
354	}
355
356	static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
357	*nand)
358	{
359	return &nand->sels[nand->selected_die];
360	}
361
362	/**
363	* struct marvell_nfc_caps - NAND controller capabilities for distinction
364	* between compatible strings
365	*
366	* @max_cs_nb: Number of Chip Select lines available
367	* @max_rb_nb: Number of Ready/Busy lines available
368	* @need_system_controller: Indicates if the SoC needs to have access to the
369	* system controller (ie. to enable the NAND controller)
370	* @legacy_of_bindings: Indicates if DT parsing must be done using the old
371	* fashion way
372	* @is_nfcv2: NFCv2 has numerous enhancements compared to NFCv1, ie.
373	* BCH error detection and correction algorithm,
374	* NDCB3 register has been added
375	* @use_dma: Use dma for data transfers
376	* @max_mode_number: Maximum timing mode supported by the controller
377	*/
378	struct marvell_nfc_caps {
379	unsigned int max_cs_nb;
380	unsigned int max_rb_nb;
381	bool need_system_controller;
382	bool legacy_of_bindings;
383	bool is_nfcv2;
384	bool use_dma;
385	unsigned int max_mode_number;
386	};
387
388	/**
389	* struct marvell_nfc - stores Marvell NAND controller information
390	*
391	* @controller: Base controller structure
392	* @dev: Parent device (used to print error messages)
393	* @regs: NAND controller registers
394	* @core_clk: Core clock
395	* @reg_clk: Registers clock
396	* @complete: Completion object to wait for NAND controller events
397	* @assigned_cs: Bitmask describing already assigned CS lines
398	* @chips: List containing all the NAND chips attached to
399	* this NAND controller
400	* @selected_chip: Currently selected target chip
401	* @caps: NAND controller capabilities for each compatible string
402	* @use_dma: Whetner DMA is used
403	* @dma_chan: DMA channel (NFCv1 only)
404	* @dma_buf: 32-bit aligned buffer for DMA transfers (NFCv1 only)
405	*/
406	struct marvell_nfc {
407	struct nand_controller controller;
408	struct device *dev;
409	void __iomem *regs;
410	struct clk *core_clk;
411	struct clk *reg_clk;
412	struct completion complete;
413	unsigned long assigned_cs;
414	struct list_head chips;
415	struct nand_chip *selected_chip;
416	const struct marvell_nfc_caps *caps;
417
418	/* DMA (NFCv1 only) */
419	bool use_dma;
420	struct dma_chan *dma_chan;
421	u8 *dma_buf;
422	};
423
424	static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
425	{
426	return container_of(ctrl, struct marvell_nfc, controller);
427	}
428
429	/**
430	* struct marvell_nfc_timings - NAND controller timings expressed in NAND
431	* Controller clock cycles
432	*
433	* @tRP: ND_nRE pulse width
434	* @tRH: ND_nRE high duration
435	* @tWP: ND_nWE pulse time
436	* @tWH: ND_nWE high duration
437	* @tCS: Enable signal setup time
438	* @tCH: Enable signal hold time
439	* @tADL: Address to write data delay
440	* @tAR: ND_ALE low to ND_nRE low delay
441	* @tWHR: ND_nWE high to ND_nRE low for status read
442	* @tRHW: ND_nRE high duration, read to write delay
443	* @tR: ND_nWE high to ND_nRE low for read
444	*/
445	struct marvell_nfc_timings {
446	/* NDTR0 fields */
447	unsigned int tRP;
448	unsigned int tRH;
449	unsigned int tWP;
450	unsigned int tWH;
451	unsigned int tCS;
452	unsigned int tCH;
453	unsigned int tADL;
454	/* NDTR1 fields */
455	unsigned int tAR;
456	unsigned int tWHR;
457	unsigned int tRHW;
458	unsigned int tR;
459	};
460
461	/**
462	* TO_CYCLES() - Derives a duration in numbers of clock cycles.
463	*
464	* @ps: Duration in pico-seconds
465	* @period_ns: Clock period in nano-seconds
466	*
467	* Convert the duration in nano-seconds, then divide by the period and
468	* return the number of clock periods.
469	*/
470	#define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
471	#define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
472	period_ns))
473
474	/**
475	* struct marvell_nfc_op - filled during the parsing of the ->exec_op()
476	* subop subset of instructions.
477	*
478	* @ndcb: Array of values written to NDCBx registers
479	* @cle_ale_delay_ns: Optional delay after the last CMD or ADDR cycle
480	* @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
481	* @rdy_delay_ns: Optional delay after waiting for the RB pin
482	* @data_delay_ns: Optional delay after the data xfer
483	* @data_instr_idx: Index of the data instruction in the subop
484	* @data_instr: Pointer to the data instruction in the subop
485	*/
486	struct marvell_nfc_op {
487	u32 ndcb[4];
488	unsigned int cle_ale_delay_ns;
489	unsigned int rdy_timeout_ms;
490	unsigned int rdy_delay_ns;
491	unsigned int data_delay_ns;
492	unsigned int data_instr_idx;
493	const struct nand_op_instr *data_instr;
494	};
495
496	/*
497	* Internal helper to conditionnally apply a delay (from the above structure,
498	* most of the time).
499	*/
500	static void cond_delay(unsigned int ns)
501	{
502	if (!ns)
503	return;
504
505	if (ns < 10000)
506	ndelay(ns);
507	else
508	udelay(DIV_ROUND_UP(ns, 1000));
509	}
510
511	/*
512	* The controller has many flags that could generate interrupts, most of them
513	* are disabled and polling is used. For the very slow signals, using interrupts
514	* may relax the CPU charge.
515	*/
516	static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
517	{
518	u32 reg;
519
520	/* Writing 1 disables the interrupt */
521	reg = readl_relaxed(nfc->regs + NDCR);
522	writel_relaxed(reg | int_mask, nfc->regs + NDCR);
523	}
524
525	static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
526	{
527	u32 reg;
528
529	/* Writing 0 enables the interrupt */
530	reg = readl_relaxed(nfc->regs + NDCR);
531	writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
532	}
533
534	static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
535	{
536	u32 reg;
537
538	reg = readl_relaxed(nfc->regs + NDSR);
539	writel_relaxed(int_mask, nfc->regs + NDSR);
540
541	return reg & int_mask;
542	}
543
544	static void marvell_nfc_force_byte_access(struct nand_chip *chip,
545	bool force_8bit)
546	{
547	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
548	u32 ndcr;
549
550	/*
551	* Callers of this function do not verify if the NAND is using a 16-bit
552	* an 8-bit bus for normal operations, so we need to take care of that
553	* here by leaving the configuration unchanged if the NAND does not have
554	* the NAND_BUSWIDTH_16 flag set.
555	*/
556	if (!(chip->options & NAND_BUSWIDTH_16))
557	return;
558
559	ndcr = readl_relaxed(nfc->regs + NDCR);
560
561	if (force_8bit)
562	ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
563	else
564	ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
565
566	writel_relaxed(ndcr, nfc->regs + NDCR);
567	}
568
569	static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
570	{
571	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
572	u32 val;
573	int ret;
574
575	/*
576	* The command is being processed, wait for the ND_RUN bit to be
577	* cleared by the NFC. If not, we must clear it by hand.
578	*/
579	ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
580	(val & NDCR_ND_RUN) == 0,
581	POLL_PERIOD, POLL_TIMEOUT);
582	if (ret) {
583	dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
584	writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
585	nfc->regs + NDCR);
586	return ret;
587	}
588
589	return 0;
590	}
591
592	/*
593	* Any time a command has to be sent to the controller, the following sequence
594	* has to be followed:
595	* - call marvell_nfc_prepare_cmd()
596	* -> activate the ND_RUN bit that will kind of 'start a job'
597	* -> wait the signal indicating the NFC is waiting for a command
598	* - send the command (cmd and address cycles)
599	* - enventually send or receive the data
600	* - call marvell_nfc_end_cmd() with the corresponding flag
601	* -> wait the flag to be triggered or cancel the job with a timeout
602	*
603	* The following helpers are here to factorize the code a bit so that
604	* specialized functions responsible for executing the actual NAND
605	* operations do not have to replicate the same code blocks.
606	*/
607	static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
608	{
609	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
610	u32 ndcr, val;
611	int ret;
612
613	/* Poll ND_RUN and clear NDSR before issuing any command */
614	ret = marvell_nfc_wait_ndrun(chip);
615	if (ret) {
616	dev_err(nfc->dev, "Last operation did not succeed\n");
617	return ret;
618	}
619
620	ndcr = readl_relaxed(nfc->regs + NDCR);
621	writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
622
623	/* Assert ND_RUN bit and wait the NFC to be ready */
624	writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
625	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
626	val & NDSR_WRCMDREQ,
627	POLL_PERIOD, POLL_TIMEOUT);
628	if (ret) {
629	dev_err(nfc->dev, "Timeout on WRCMDRE\n");
630	return -ETIMEDOUT;
631	}
632
633	/* Command may be written, clear WRCMDREQ status bit */
634	writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
635
636	return 0;
637	}
638
639	static void marvell_nfc_send_cmd(struct nand_chip *chip,
640	struct marvell_nfc_op *nfc_op)
641	{
642	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
643	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
644
645	dev_dbg(nfc->dev, "\nNDCR: 0x%08x\n"
646	"NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
647	(u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
648	nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
649
650	writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
651	nfc->regs + NDCB0);
652	writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
653	writel(val: nfc_op->ndcb[2], addr: nfc->regs + NDCB0);
654
655	/*
656	* Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
657	* fields are used (only available on NFCv2).
658	*/
659	if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
660	NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
661	if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
662	writel(val: nfc_op->ndcb[3], addr: nfc->regs + NDCB0);
663	}
664	}
665
666	static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
667	const char *label)
668	{
669	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
670	u32 val;
671	int ret;
672
673	ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
674	val & flag,
675	POLL_PERIOD, POLL_TIMEOUT);
676
677	if (ret) {
678	dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
679	label, val);
680	if (nfc->dma_chan)
681	dmaengine_terminate_all(chan: nfc->dma_chan);
682	return ret;
683	}
684
685	/*
686	* DMA function uses this helper to poll on CMDD bits without wanting
687	* them to be cleared.
688	*/
689	if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
690	return 0;
691
692	writel_relaxed(flag, nfc->regs + NDSR);
693
694	return 0;
695	}
696
697	static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
698	{
699	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
700	int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
701
702	return marvell_nfc_end_cmd(chip, flag: cs_flag, label: "CMDD");
703	}
704
705	static int marvell_nfc_poll_status(struct marvell_nfc *nfc, u32 mask,
706	u32 expected_val, unsigned long timeout_ms)
707	{
708	unsigned long limit;
709	u32 st;
710
711	limit = jiffies + msecs_to_jiffies(m: timeout_ms);
712	do {
713	st = readl_relaxed(nfc->regs + NDSR);
714	if (st & NDSR_RDY(1))
715	st |= NDSR_RDY(0);
716
717	if ((st & mask) == expected_val)
718	return 0;
719
720	cpu_relax();
721	} while (time_after(limit, jiffies));
722
723	return -ETIMEDOUT;
724	}
725
726	static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
727	{
728	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
729	struct mtd_info *mtd = nand_to_mtd(chip);
730	u32 pending;
731	int ret;
732
733	/* Timeout is expressed in ms */
734	if (!timeout_ms)
735	timeout_ms = IRQ_TIMEOUT;
736
737	if (mtd->oops_panic_write) {
738	ret = marvell_nfc_poll_status(nfc, NDSR_RDY(0),
739	NDSR_RDY(0),
740	timeout_ms);
741	} else {
742	init_completion(x: &nfc->complete);
743
744	marvell_nfc_enable_int(nfc, NDCR_RDYM);
745	ret = wait_for_completion_timeout(x: &nfc->complete,
746	timeout: msecs_to_jiffies(m: timeout_ms));
747	marvell_nfc_disable_int(nfc, NDCR_RDYM);
748	}
749	pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
750
751	/*
752	* In case the interrupt was not served in the required time frame,
753	* check if the ISR was not served or if something went actually wrong.
754	*/
755	if (!ret && !pending) {
756	dev_err(nfc->dev, "Timeout waiting for RB signal\n");
757	return -ETIMEDOUT;
758	}
759
760	return 0;
761	}
762
763	static void marvell_nfc_select_target(struct nand_chip *chip,
764	unsigned int die_nr)
765	{
766	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
767	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
768	u32 ndcr_generic;
769
770	/*
771	* Reset the NDCR register to a clean state for this particular chip,
772	* also clear ND_RUN bit.
773	*/
774	ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
775	NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
776	writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
777
778	/* Also reset the interrupt status register */
779	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
780
781	if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
782	return;
783
784	writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
785	writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
786
787	nfc->selected_chip = chip;
788	marvell_nand->selected_die = die_nr;
789	}
790
791	static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
792	{
793	struct marvell_nfc *nfc = dev_id;
794	u32 st = readl_relaxed(nfc->regs + NDSR);
795	u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
796
797	/*
798	* RDY interrupt mask is one bit in NDCR while there are two status
799	* bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
800	*/
801	if (st & NDSR_RDY(1))
802	st |= NDSR_RDY(0);
803
804	if (!(st & ien))
805	return IRQ_NONE;
806
807	marvell_nfc_disable_int(nfc, int_mask: st & NDCR_ALL_INT);
808
809	if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
810	complete(&nfc->complete);
811
812	return IRQ_HANDLED;
813	}
814
815	/* HW ECC related functions */
816	static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
817	{
818	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
819	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
820
821	if (!(ndcr & NDCR_ECC_EN)) {
822	writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
823
824	/*
825	* When enabling BCH, set threshold to 0 to always know the
826	* number of corrected bitflips.
827	*/
828	if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
829	writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
830	}
831	}
832
833	static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
834	{
835	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
836	u32 ndcr = readl_relaxed(nfc->regs + NDCR);
837
838	if (ndcr & NDCR_ECC_EN) {
839	writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
840	if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
841	writel_relaxed(0, nfc->regs + NDECCCTRL);
842	}
843	}
844
845	/* DMA related helpers */
846	static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
847	{
848	u32 reg;
849
850	reg = readl_relaxed(nfc->regs + NDCR);
851	writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
852	}
853
854	static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
855	{
856	u32 reg;
857
858	reg = readl_relaxed(nfc->regs + NDCR);
859	writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
860	}
861
862	/* Read/write PIO/DMA accessors */
863	static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
864	enum dma_data_direction direction,
865	unsigned int len)
866	{
867	unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
868	struct dma_async_tx_descriptor *tx;
869	struct scatterlist sg;
870	dma_cookie_t cookie;
871	int ret;
872
873	marvell_nfc_enable_dma(nfc);
874	/* Prepare the DMA transfer */
875	sg_init_one(&sg, nfc->dma_buf, dma_len);
876	ret = dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
877	if (!ret) {
878	dev_err(nfc->dev, "Could not map DMA S/G list\n");
879	return -ENXIO;
880	}
881
882	tx = dmaengine_prep_slave_sg(chan: nfc->dma_chan, sgl: &sg, sg_len: 1,
883	dir: direction == DMA_FROM_DEVICE ?
884	DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
885	flags: DMA_PREP_INTERRUPT);
886	if (!tx) {
887	dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
888	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
889	return -ENXIO;
890	}
891
892	/* Do the task and wait for it to finish */
893	cookie = dmaengine_submit(desc: tx);
894	ret = dma_submit_error(cookie);
895	if (ret)
896	return -EIO;
897
898	dma_async_issue_pending(chan: nfc->dma_chan);
899	ret = marvell_nfc_wait_cmdd(chip: nfc->selected_chip);
900	dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
901	marvell_nfc_disable_dma(nfc);
902	if (ret) {
903	dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
904	dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
905	dmaengine_terminate_all(chan: nfc->dma_chan);
906	return -ETIMEDOUT;
907	}
908
909	return 0;
910	}
911
912	static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
913	unsigned int len)
914	{
915	unsigned int last_len = len % FIFO_DEPTH;
916	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
917	int i;
918
919	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
920	ioread32_rep(port: nfc->regs + NDDB, buf: in + i, FIFO_REP(FIFO_DEPTH));
921
922	if (last_len) {
923	u8 tmp_buf[FIFO_DEPTH];
924
925	ioread32_rep(port: nfc->regs + NDDB, buf: tmp_buf, FIFO_REP(FIFO_DEPTH));
926	memcpy(in + last_full_offset, tmp_buf, last_len);
927	}
928
929	return 0;
930	}
931
932	static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
933	unsigned int len)
934	{
935	unsigned int last_len = len % FIFO_DEPTH;
936	unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
937	int i;
938
939	for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
940	iowrite32_rep(port: nfc->regs + NDDB, buf: out + i, FIFO_REP(FIFO_DEPTH));
941
942	if (last_len) {
943	u8 tmp_buf[FIFO_DEPTH];
944
945	memcpy(tmp_buf, out + last_full_offset, last_len);
946	iowrite32_rep(port: nfc->regs + NDDB, buf: tmp_buf, FIFO_REP(FIFO_DEPTH));
947	}
948
949	return 0;
950	}
951
952	static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
953	u8 *data, int data_len,
954	u8 *spare, int spare_len,
955	u8 *ecc, int ecc_len,
956	unsigned int *max_bitflips)
957	{
958	struct mtd_info *mtd = nand_to_mtd(chip);
959	int bf;
960
961	/*
962	* Blank pages (all 0xFF) that have not been written may be recognized
963	* as bad if bitflips occur, so whenever an uncorrectable error occurs,
964	* check if the entire page (with ECC bytes) is actually blank or not.
965	*/
966	if (!data)
967	data_len = 0;
968	if (!spare)
969	spare_len = 0;
970	if (!ecc)
971	ecc_len = 0;
972
973	bf = nand_check_erased_ecc_chunk(data, datalen: data_len, ecc, ecclen: ecc_len,
974	extraoob: spare, extraooblen: spare_len, threshold: chip->ecc.strength);
975	if (bf < 0) {
976	mtd->ecc_stats.failed++;
977	return;
978	}
979
980	/* Update the stats and max_bitflips */
981	mtd->ecc_stats.corrected += bf;
982	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
983	}
984
985	/*
986	* Check if a chunk is correct or not according to the hardware ECC engine.
987	* mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
988	* mtd->ecc_stats.failure is not, the function will instead return a non-zero
989	* value indicating that a check on the emptyness of the subpage must be
990	* performed before actually declaring the subpage as "corrupted".
991	*/
992	static int marvell_nfc_hw_ecc_check_bitflips(struct nand_chip *chip,
993	unsigned int *max_bitflips)
994	{
995	struct mtd_info *mtd = nand_to_mtd(chip);
996	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
997	int bf = 0;
998	u32 ndsr;
999
1000	ndsr = readl_relaxed(nfc->regs + NDSR);
1001
1002	/* Check uncorrectable error flag */
1003	if (ndsr & NDSR_UNCERR) {
1004	writel_relaxed(ndsr, nfc->regs + NDSR);
1005
1006	/*
1007	* Do not increment ->ecc_stats.failed now, instead, return a
1008	* non-zero value to indicate that this chunk was apparently
1009	* bad, and it should be check to see if it empty or not. If
1010	* the chunk (with ECC bytes) is not declared empty, the calling
1011	* function must increment the failure count.
1012	*/
1013	return -EBADMSG;
1014	}
1015
1016	/* Check correctable error flag */
1017	if (ndsr & NDSR_CORERR) {
1018	writel_relaxed(ndsr, nfc->regs + NDSR);
1019
1020	if (chip->ecc.algo == NAND_ECC_ALGO_BCH)
1021	bf = NDSR_ERRCNT(ndsr);
1022	else
1023	bf = 1;
1024	}
1025
1026	/* Update the stats and max_bitflips */
1027	mtd->ecc_stats.corrected += bf;
1028	*max_bitflips = max_t(unsigned int, *max_bitflips, bf);
1029
1030	return 0;
1031	}
1032
1033	/* Hamming read helpers */
1034	static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
1035	u8 *data_buf, u8 *oob_buf,
1036	bool raw, int page)
1037	{
1038	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1039	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1040	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1041	struct marvell_nfc_op nfc_op = {
1042	.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1043	NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1044	NDCB0_DBC |
1045	NDCB0_CMD1(NAND_CMD_READ0) |
1046	NDCB0_CMD2(NAND_CMD_READSTART),
1047	.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1048	.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1049	};
1050	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1051	int ret;
1052
1053	/* NFCv2 needs more information about the operation being executed */
1054	if (nfc->caps->is_nfcv2)
1055	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1056
1057	ret = marvell_nfc_prepare_cmd(chip);
1058	if (ret)
1059	return ret;
1060
1061	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1062	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1063	label: "RDDREQ while draining FIFO (data/oob)");
1064	if (ret)
1065	return ret;
1066
1067	/*
1068	* Read the page then the OOB area. Unlike what is shown in current
1069	* documentation, spare bytes are protected by the ECC engine, and must
1070	* be at the beginning of the OOB area or running this driver on legacy
1071	* systems will prevent the discovery of the BBM/BBT.
1072	*/
1073	if (nfc->use_dma) {
1074	marvell_nfc_xfer_data_dma(nfc, direction: DMA_FROM_DEVICE,
1075	len: lt->data_bytes + oob_bytes);
1076	memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1077	memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1078	} else {
1079	marvell_nfc_xfer_data_in_pio(nfc, in: data_buf, len: lt->data_bytes);
1080	marvell_nfc_xfer_data_in_pio(nfc, in: oob_buf, len: oob_bytes);
1081	}
1082
1083	ret = marvell_nfc_wait_cmdd(chip);
1084	return ret;
1085	}
1086
1087	static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
1088	int oob_required, int page)
1089	{
1090	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1091	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, data_buf: buf, oob_buf: chip->oob_poi,
1092	raw: true, page);
1093	}
1094
1095	static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1096	int oob_required, int page)
1097	{
1098	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1099	unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1100	int max_bitflips = 0, ret;
1101	u8 *raw_buf;
1102
1103	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1104	marvell_nfc_enable_hw_ecc(chip);
1105	marvell_nfc_hw_ecc_hmg_do_read_page(chip, data_buf: buf, oob_buf: chip->oob_poi, raw: false,
1106	page);
1107	ret = marvell_nfc_hw_ecc_check_bitflips(chip, max_bitflips: &max_bitflips);
1108	marvell_nfc_disable_hw_ecc(chip);
1109
1110	if (!ret)
1111	return max_bitflips;
1112
1113	/*
1114	* When ECC failures are detected, check if the full page has been
1115	* written or not. Ignore the failure if it is actually empty.
1116	*/
1117	raw_buf = kmalloc(size: full_sz, GFP_KERNEL);
1118	if (!raw_buf)
1119	return -ENOMEM;
1120
1121	marvell_nfc_hw_ecc_hmg_do_read_page(chip, data_buf: raw_buf, oob_buf: raw_buf +
1122	lt->data_bytes, raw: true, page);
1123	marvell_nfc_check_empty_chunk(chip, data: raw_buf, data_len: full_sz, NULL, spare_len: 0, NULL, ecc_len: 0,
1124	max_bitflips: &max_bitflips);
1125	kfree(objp: raw_buf);
1126
1127	return max_bitflips;
1128	}
1129
1130	/*
1131	* Spare area in Hamming layouts is not protected by the ECC engine (even if
1132	* it appears before the ECC bytes when reading), the ->read_oob_raw() function
1133	* also stands for ->read_oob().
1134	*/
1135	static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
1136	{
1137	u8 *buf = nand_get_data_buf(chip);
1138
1139	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1140	return marvell_nfc_hw_ecc_hmg_do_read_page(chip, data_buf: buf, oob_buf: chip->oob_poi,
1141	raw: true, page);
1142	}
1143
1144	/* Hamming write helpers */
1145	static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1146	const u8 *data_buf,
1147	const u8 *oob_buf, bool raw,
1148	int page)
1149	{
1150	const struct nand_sdr_timings *sdr =
1151	nand_get_sdr_timings(conf: nand_get_interface_config(chip));
1152	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1153	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1154	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1155	struct marvell_nfc_op nfc_op = {
1156	.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1157	NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1158	NDCB0_CMD1(NAND_CMD_SEQIN) |
1159	NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1160	NDCB0_DBC,
1161	.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1162	.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1163	};
1164	unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1165	u8 status;
1166	int ret;
1167
1168	/* NFCv2 needs more information about the operation being executed */
1169	if (nfc->caps->is_nfcv2)
1170	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1171
1172	ret = marvell_nfc_prepare_cmd(chip);
1173	if (ret)
1174	return ret;
1175
1176	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1177	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1178	label: "WRDREQ while loading FIFO (data)");
1179	if (ret)
1180	return ret;
1181
1182	/* Write the page then the OOB area */
1183	if (nfc->use_dma) {
1184	memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1185	memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1186	marvell_nfc_xfer_data_dma(nfc, direction: DMA_TO_DEVICE, len: lt->data_bytes +
1187	lt->ecc_bytes + lt->spare_bytes);
1188	} else {
1189	marvell_nfc_xfer_data_out_pio(nfc, out: data_buf, len: lt->data_bytes);
1190	marvell_nfc_xfer_data_out_pio(nfc, out: oob_buf, len: oob_bytes);
1191	}
1192
1193	ret = marvell_nfc_wait_cmdd(chip);
1194	if (ret)
1195	return ret;
1196
1197	ret = marvell_nfc_wait_op(chip,
1198	PSEC_TO_MSEC(sdr->tPROG_max));
1199	if (ret)
1200	return ret;
1201
1202	/* Check write status on the chip side */
1203	ret = nand_status_op(chip, status: &status);
1204	if (ret)
1205	return ret;
1206
1207	if (status & NAND_STATUS_FAIL)
1208	return -EIO;
1209
1210	return 0;
1211	}
1212
1213	static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
1214	const u8 *buf,
1215	int oob_required, int page)
1216	{
1217	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1218	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, data_buf: buf, oob_buf: chip->oob_poi,
1219	raw: true, page);
1220	}
1221
1222	static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
1223	const u8 *buf,
1224	int oob_required, int page)
1225	{
1226	int ret;
1227
1228	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1229	marvell_nfc_enable_hw_ecc(chip);
1230	ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, data_buf: buf, oob_buf: chip->oob_poi,
1231	raw: false, page);
1232	marvell_nfc_disable_hw_ecc(chip);
1233
1234	return ret;
1235	}
1236
1237	/*
1238	* Spare area in Hamming layouts is not protected by the ECC engine (even if
1239	* it appears before the ECC bytes when reading), the ->write_oob_raw() function
1240	* also stands for ->write_oob().
1241	*/
1242	static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
1243	int page)
1244	{
1245	struct mtd_info *mtd = nand_to_mtd(chip);
1246	u8 *buf = nand_get_data_buf(chip);
1247
1248	memset(buf, 0xFF, mtd->writesize);
1249
1250	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1251	return marvell_nfc_hw_ecc_hmg_do_write_page(chip, data_buf: buf, oob_buf: chip->oob_poi,
1252	raw: true, page);
1253	}
1254
1255	/* BCH read helpers */
1256	static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
1257	int oob_required, int page)
1258	{
1259	struct mtd_info *mtd = nand_to_mtd(chip);
1260	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1261	u8 *oob = chip->oob_poi;
1262	int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1263	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1264	lt->last_spare_bytes;
1265	int data_len = lt->data_bytes;
1266	int spare_len = lt->spare_bytes;
1267	int ecc_len = lt->ecc_bytes;
1268	int chunk;
1269
1270	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1271
1272	if (oob_required)
1273	memset(chip->oob_poi, 0xFF, mtd->oobsize);
1274
1275	nand_read_page_op(chip, page, offset_in_page: 0, NULL, len: 0);
1276
1277	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1278	/* Update last chunk length */
1279	if (chunk >= lt->full_chunk_cnt) {
1280	data_len = lt->last_data_bytes;
1281	spare_len = lt->last_spare_bytes;
1282	ecc_len = lt->last_ecc_bytes;
1283	}
1284
1285	/* Read data bytes*/
1286	nand_change_read_column_op(chip, offset_in_page: chunk * chunk_size,
1287	buf: buf + (lt->data_bytes * chunk),
1288	len: data_len, force_8bit: false);
1289
1290	/* Read spare bytes */
1291	nand_read_data_op(chip, buf: oob + (lt->spare_bytes * chunk),
1292	len: spare_len, force_8bit: false, check_only: false);
1293
1294	/* Read ECC bytes */
1295	nand_read_data_op(chip, buf: oob + ecc_offset +
1296	(ALIGN(lt->ecc_bytes, 32) * chunk),
1297	len: ecc_len, force_8bit: false, check_only: false);
1298	}
1299
1300	return 0;
1301	}
1302
1303	static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1304	u8 *data, unsigned int data_len,
1305	u8 *spare, unsigned int spare_len,
1306	int page)
1307	{
1308	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1309	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1310	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1311	int i, ret;
1312	struct marvell_nfc_op nfc_op = {
1313	.ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1314	NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1315	NDCB0_LEN_OVRD,
1316	.ndcb[1] = NDCB1_ADDRS_PAGE(page),
1317	.ndcb[2] = NDCB2_ADDR5_PAGE(page),
1318	.ndcb[3] = data_len + spare_len,
1319	};
1320
1321	ret = marvell_nfc_prepare_cmd(chip);
1322	if (ret)
1323	return;
1324
1325	if (chunk == 0)
1326	nfc_op.ndcb[0] |= NDCB0_DBC |
1327	NDCB0_CMD1(NAND_CMD_READ0) |
1328	NDCB0_CMD2(NAND_CMD_READSTART);
1329
1330	/*
1331	* Trigger the monolithic read on the first chunk, then naked read on
1332	* intermediate chunks and finally a last naked read on the last chunk.
1333	*/
1334	if (chunk == 0)
1335	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1336	else if (chunk < lt->nchunks - 1)
1337	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1338	else
1339	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1340
1341	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1342
1343	/*
1344	* According to the datasheet, when reading from NDDB
1345	* with BCH enabled, after each 32 bytes reads, we
1346	* have to make sure that the NDSR.RDDREQ bit is set.
1347	*
1348	* Drain the FIFO, 8 32-bit reads at a time, and skip
1349	* the polling on the last read.
1350	*
1351	* Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1352	*/
1353	for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1354	marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1355	label: "RDDREQ while draining FIFO (data)");
1356	marvell_nfc_xfer_data_in_pio(nfc, in: data,
1357	FIFO_DEPTH * BCH_SEQ_READS);
1358	data += FIFO_DEPTH * BCH_SEQ_READS;
1359	}
1360
1361	for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1362	marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1363	label: "RDDREQ while draining FIFO (OOB)");
1364	marvell_nfc_xfer_data_in_pio(nfc, in: spare,
1365	FIFO_DEPTH * BCH_SEQ_READS);
1366	spare += FIFO_DEPTH * BCH_SEQ_READS;
1367	}
1368	}
1369
1370	static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
1371	u8 *buf, int oob_required,
1372	int page)
1373	{
1374	struct mtd_info *mtd = nand_to_mtd(chip);
1375	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1376	int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1377	u8 *data = buf, *spare = chip->oob_poi;
1378	int max_bitflips = 0;
1379	u32 failure_mask = 0;
1380	int chunk, ret;
1381
1382	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1383
1384	/*
1385	* With BCH, OOB is not fully used (and thus not read entirely), not
1386	* expected bytes could show up at the end of the OOB buffer if not
1387	* explicitly erased.
1388	*/
1389	if (oob_required)
1390	memset(chip->oob_poi, 0xFF, mtd->oobsize);
1391
1392	marvell_nfc_enable_hw_ecc(chip);
1393
1394	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1395	/* Update length for the last chunk */
1396	if (chunk >= lt->full_chunk_cnt) {
1397	data_len = lt->last_data_bytes;
1398	spare_len = lt->last_spare_bytes;
1399	}
1400
1401	/* Read the chunk and detect number of bitflips */
1402	marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1403	spare, spare_len, page);
1404	ret = marvell_nfc_hw_ecc_check_bitflips(chip, max_bitflips: &max_bitflips);
1405	if (ret)
1406	failure_mask |= BIT(chunk);
1407
1408	data += data_len;
1409	spare += spare_len;
1410	}
1411
1412	marvell_nfc_disable_hw_ecc(chip);
1413
1414	if (!failure_mask)
1415	return max_bitflips;
1416
1417	/*
1418	* Please note that dumping the ECC bytes during a normal read with OOB
1419	* area would add a significant overhead as ECC bytes are "consumed" by
1420	* the controller in normal mode and must be re-read in raw mode. To
1421	* avoid dropping the performances, we prefer not to include them. The
1422	* user should re-read the page in raw mode if ECC bytes are required.
1423	*/
1424
1425	/*
1426	* In case there is any subpage read error, we usually re-read only ECC
1427	* bytes in raw mode and check if the whole page is empty. In this case,
1428	* it is normal that the ECC check failed and we just ignore the error.
1429	*
1430	* However, it has been empirically observed that for some layouts (e.g
1431	* 2k page, 8b strength per 512B chunk), the controller tries to correct
1432	* bits and may create itself bitflips in the erased area. To overcome
1433	* this strange behavior, the whole page is re-read in raw mode, not
1434	* only the ECC bytes.
1435	*/
1436	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1437	int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1438	int data_off, spare_off, ecc_off;
1439	int data_len, spare_len, ecc_len;
1440
1441	/* No failure reported for this chunk, move to the next one */
1442	if (!(failure_mask & BIT(chunk)))
1443	continue;
1444
1445	data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1446	lt->ecc_bytes);
1447	spare_off_in_page = data_off_in_page +
1448	(chunk < lt->full_chunk_cnt ? lt->data_bytes :
1449	lt->last_data_bytes);
1450	ecc_off_in_page = spare_off_in_page +
1451	(chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1452	lt->last_spare_bytes);
1453
1454	data_off = chunk * lt->data_bytes;
1455	spare_off = chunk * lt->spare_bytes;
1456	ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1457	lt->last_spare_bytes +
1458	(chunk * (lt->ecc_bytes + 2));
1459
1460	data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1461	lt->last_data_bytes;
1462	spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1463	lt->last_spare_bytes;
1464	ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1465	lt->last_ecc_bytes;
1466
1467	/*
1468	* Only re-read the ECC bytes, unless we are using the 2k/8b
1469	* layout which is buggy in the sense that the ECC engine will
1470	* try to correct data bytes anyway, creating bitflips. In this
1471	* case, re-read the entire page.
1472	*/
1473	if (lt->writesize == 2048 && lt->strength == 8) {
1474	nand_change_read_column_op(chip, offset_in_page: data_off_in_page,
1475	buf: buf + data_off, len: data_len,
1476	force_8bit: false);
1477	nand_change_read_column_op(chip, offset_in_page: spare_off_in_page,
1478	buf: chip->oob_poi + spare_off, len: spare_len,
1479	force_8bit: false);
1480	}
1481
1482	nand_change_read_column_op(chip, offset_in_page: ecc_off_in_page,
1483	buf: chip->oob_poi + ecc_off, len: ecc_len,
1484	force_8bit: false);
1485
1486	/* Check the entire chunk (data + spare + ecc) for emptyness */
1487	marvell_nfc_check_empty_chunk(chip, data: buf + data_off, data_len,
1488	spare: chip->oob_poi + spare_off, spare_len,
1489	ecc: chip->oob_poi + ecc_off, ecc_len,
1490	max_bitflips: &max_bitflips);
1491	}
1492
1493	return max_bitflips;
1494	}
1495
1496	static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
1497	{
1498	u8 *buf = nand_get_data_buf(chip);
1499
1500	return chip->ecc.read_page_raw(chip, buf, true, page);
1501	}
1502
1503	static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
1504	{
1505	u8 *buf = nand_get_data_buf(chip);
1506
1507	return chip->ecc.read_page(chip, buf, true, page);
1508	}
1509
1510	/* BCH write helpers */
1511	static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
1512	const u8 *buf,
1513	int oob_required, int page)
1514	{
1515	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1516	int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1517	int data_len = lt->data_bytes;
1518	int spare_len = lt->spare_bytes;
1519	int ecc_len = lt->ecc_bytes;
1520	int spare_offset = 0;
1521	int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1522	lt->last_spare_bytes;
1523	int chunk;
1524
1525	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1526
1527	nand_prog_page_begin_op(chip, page, offset_in_page: 0, NULL, len: 0);
1528
1529	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1530	if (chunk >= lt->full_chunk_cnt) {
1531	data_len = lt->last_data_bytes;
1532	spare_len = lt->last_spare_bytes;
1533	ecc_len = lt->last_ecc_bytes;
1534	}
1535
1536	/* Point to the column of the next chunk */
1537	nand_change_write_column_op(chip, offset_in_page: chunk * full_chunk_size,
1538	NULL, len: 0, force_8bit: false);
1539
1540	/* Write the data */
1541	nand_write_data_op(chip, buf: buf + (chunk * lt->data_bytes),
1542	len: data_len, force_8bit: false);
1543
1544	if (!oob_required)
1545	continue;
1546
1547	/* Write the spare bytes */
1548	if (spare_len)
1549	nand_write_data_op(chip, buf: chip->oob_poi + spare_offset,
1550	len: spare_len, force_8bit: false);
1551
1552	/* Write the ECC bytes */
1553	if (ecc_len)
1554	nand_write_data_op(chip, buf: chip->oob_poi + ecc_offset,
1555	len: ecc_len, force_8bit: false);
1556
1557	spare_offset += spare_len;
1558	ecc_offset += ALIGN(ecc_len, 32);
1559	}
1560
1561	return nand_prog_page_end_op(chip);
1562	}
1563
1564	static int
1565	marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1566	const u8 *data, unsigned int data_len,
1567	const u8 *spare, unsigned int spare_len,
1568	int page)
1569	{
1570	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1571	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1572	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1573	u32 xtype;
1574	int ret;
1575	struct marvell_nfc_op nfc_op = {
1576	.ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1577	.ndcb[3] = data_len + spare_len,
1578	};
1579
1580	/*
1581	* First operation dispatches the CMD_SEQIN command, issue the address
1582	* cycles and asks for the first chunk of data.
1583	* All operations in the middle (if any) will issue a naked write and
1584	* also ask for data.
1585	* Last operation (if any) asks for the last chunk of data through a
1586	* last naked write.
1587	*/
1588	if (chunk == 0) {
1589	if (lt->nchunks == 1)
1590	xtype = XTYPE_MONOLITHIC_RW;
1591	else
1592	xtype = XTYPE_WRITE_DISPATCH;
1593
1594	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
1595	NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1596	NDCB0_CMD1(NAND_CMD_SEQIN);
1597	nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1598	nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1599	} else if (chunk < lt->nchunks - 1) {
1600	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1601	} else {
1602	nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1603	}
1604
1605	/* Always dispatch the PAGEPROG command on the last chunk */
1606	if (chunk == lt->nchunks - 1)
1607	nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1608
1609	ret = marvell_nfc_prepare_cmd(chip);
1610	if (ret)
1611	return ret;
1612
1613	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1614	ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1615	label: "WRDREQ while loading FIFO (data)");
1616	if (ret)
1617	return ret;
1618
1619	/* Transfer the contents */
1620	iowrite32_rep(port: nfc->regs + NDDB, buf: data, FIFO_REP(data_len));
1621	iowrite32_rep(port: nfc->regs + NDDB, buf: spare, FIFO_REP(spare_len));
1622
1623	return 0;
1624	}
1625
1626	static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
1627	const u8 *buf,
1628	int oob_required, int page)
1629	{
1630	const struct nand_sdr_timings *sdr =
1631	nand_get_sdr_timings(conf: nand_get_interface_config(chip));
1632	struct mtd_info *mtd = nand_to_mtd(chip);
1633	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1634	const u8 *data = buf;
1635	const u8 *spare = chip->oob_poi;
1636	int data_len = lt->data_bytes;
1637	int spare_len = lt->spare_bytes;
1638	int chunk, ret;
1639	u8 status;
1640
1641	marvell_nfc_select_target(chip, die_nr: chip->cur_cs);
1642
1643	/* Spare data will be written anyway, so clear it to avoid garbage */
1644	if (!oob_required)
1645	memset(chip->oob_poi, 0xFF, mtd->oobsize);
1646
1647	marvell_nfc_enable_hw_ecc(chip);
1648
1649	for (chunk = 0; chunk < lt->nchunks; chunk++) {
1650	if (chunk >= lt->full_chunk_cnt) {
1651	data_len = lt->last_data_bytes;
1652	spare_len = lt->last_spare_bytes;
1653	}
1654
1655	marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1656	spare, spare_len, page);
1657	data += data_len;
1658	spare += spare_len;
1659
1660	/*
1661	* Waiting only for CMDD or PAGED is not enough, ECC are
1662	* partially written. No flag is set once the operation is
1663	* really finished but the ND_RUN bit is cleared, so wait for it
1664	* before stepping into the next command.
1665	*/
1666	marvell_nfc_wait_ndrun(chip);
1667	}
1668
1669	ret = marvell_nfc_wait_op(chip, PSEC_TO_MSEC(sdr->tPROG_max));
1670
1671	marvell_nfc_disable_hw_ecc(chip);
1672
1673	if (ret)
1674	return ret;
1675
1676	/* Check write status on the chip side */
1677	ret = nand_status_op(chip, status: &status);
1678	if (ret)
1679	return ret;
1680
1681	if (status & NAND_STATUS_FAIL)
1682	return -EIO;
1683
1684	return 0;
1685	}
1686
1687	static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
1688	int page)
1689	{
1690	struct mtd_info *mtd = nand_to_mtd(chip);
1691	u8 *buf = nand_get_data_buf(chip);
1692
1693	memset(buf, 0xFF, mtd->writesize);
1694
1695	return chip->ecc.write_page_raw(chip, buf, true, page);
1696	}
1697
1698	static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
1699	{
1700	struct mtd_info *mtd = nand_to_mtd(chip);
1701	u8 *buf = nand_get_data_buf(chip);
1702
1703	memset(buf, 0xFF, mtd->writesize);
1704
1705	return chip->ecc.write_page(chip, buf, true, page);
1706	}
1707
1708	/* NAND framework ->exec_op() hooks and related helpers */
1709	static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1710	const struct nand_subop *subop,
1711	struct marvell_nfc_op *nfc_op)
1712	{
1713	const struct nand_op_instr *instr = NULL;
1714	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1715	bool first_cmd = true;
1716	unsigned int op_id;
1717	int i;
1718
1719	/* Reset the input structure as most of its fields will be OR'ed */
1720	memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1721
1722	for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1723	unsigned int offset, naddrs;
1724	const u8 *addrs;
1725	int len;
1726
1727	instr = &subop->instrs[op_id];
1728
1729	switch (instr->type) {
1730	case NAND_OP_CMD_INSTR:
1731	if (first_cmd)
1732	nfc_op->ndcb[0] |=
1733	NDCB0_CMD1(instr->ctx.cmd.opcode);
1734	else
1735	nfc_op->ndcb[0] |=
1736	NDCB0_CMD2(instr->ctx.cmd.opcode) |
1737	NDCB0_DBC;
1738
1739	nfc_op->cle_ale_delay_ns = instr->delay_ns;
1740	first_cmd = false;
1741	break;
1742
1743	case NAND_OP_ADDR_INSTR:
1744	offset = nand_subop_get_addr_start_off(subop, op_id);
1745	naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1746	addrs = &instr->ctx.addr.addrs[offset];
1747
1748	nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1749
1750	for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1751	nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1752
1753	if (naddrs >= 5)
1754	nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1755	if (naddrs >= 6)
1756	nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1757	if (naddrs == 7)
1758	nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1759
1760	nfc_op->cle_ale_delay_ns = instr->delay_ns;
1761	break;
1762
1763	case NAND_OP_DATA_IN_INSTR:
1764	nfc_op->data_instr = instr;
1765	nfc_op->data_instr_idx = op_id;
1766	nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1767	if (nfc->caps->is_nfcv2) {
1768	nfc_op->ndcb[0] |=
1769	NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1770	NDCB0_LEN_OVRD;
1771	len = nand_subop_get_data_len(subop, op_id);
1772	nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1773	}
1774	nfc_op->data_delay_ns = instr->delay_ns;
1775	break;
1776
1777	case NAND_OP_DATA_OUT_INSTR:
1778	nfc_op->data_instr = instr;
1779	nfc_op->data_instr_idx = op_id;
1780	nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1781	if (nfc->caps->is_nfcv2) {
1782	nfc_op->ndcb[0] |=
1783	NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1784	NDCB0_LEN_OVRD;
1785	len = nand_subop_get_data_len(subop, op_id);
1786	nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1787	}
1788	nfc_op->data_delay_ns = instr->delay_ns;
1789	break;
1790
1791	case NAND_OP_WAITRDY_INSTR:
1792	nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1793	nfc_op->rdy_delay_ns = instr->delay_ns;
1794	break;
1795	}
1796	}
1797	}
1798
1799	static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1800	const struct nand_subop *subop,
1801	struct marvell_nfc_op *nfc_op)
1802	{
1803	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1804	const struct nand_op_instr *instr = nfc_op->data_instr;
1805	unsigned int op_id = nfc_op->data_instr_idx;
1806	unsigned int len = nand_subop_get_data_len(subop, op_id);
1807	unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1808	bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1809	int ret;
1810
1811	if (instr->ctx.data.force_8bit)
1812	marvell_nfc_force_byte_access(chip, force_8bit: true);
1813
1814	if (reading) {
1815	u8 *in = instr->ctx.data.buf.in + offset;
1816
1817	ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1818	} else {
1819	const u8 *out = instr->ctx.data.buf.out + offset;
1820
1821	ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1822	}
1823
1824	if (instr->ctx.data.force_8bit)
1825	marvell_nfc_force_byte_access(chip, force_8bit: false);
1826
1827	return ret;
1828	}
1829
1830	static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1831	const struct nand_subop *subop)
1832	{
1833	struct marvell_nfc_op nfc_op;
1834	bool reading;
1835	int ret;
1836
1837	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
1838	reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1839
1840	ret = marvell_nfc_prepare_cmd(chip);
1841	if (ret)
1842	return ret;
1843
1844	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1845	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1846	label: "RDDREQ/WRDREQ while draining raw data");
1847	if (ret)
1848	return ret;
1849
1850	cond_delay(ns: nfc_op.cle_ale_delay_ns);
1851
1852	if (reading) {
1853	if (nfc_op.rdy_timeout_ms) {
1854	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
1855	if (ret)
1856	return ret;
1857	}
1858
1859	cond_delay(ns: nfc_op.rdy_delay_ns);
1860	}
1861
1862	marvell_nfc_xfer_data_pio(chip, subop, nfc_op: &nfc_op);
1863	ret = marvell_nfc_wait_cmdd(chip);
1864	if (ret)
1865	return ret;
1866
1867	cond_delay(ns: nfc_op.data_delay_ns);
1868
1869	if (!reading) {
1870	if (nfc_op.rdy_timeout_ms) {
1871	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
1872	if (ret)
1873	return ret;
1874	}
1875
1876	cond_delay(ns: nfc_op.rdy_delay_ns);
1877	}
1878
1879	/*
1880	* NDCR ND_RUN bit should be cleared automatically at the end of each
1881	* operation but experience shows that the behavior is buggy when it
1882	* comes to writes (with LEN_OVRD). Clear it by hand in this case.
1883	*/
1884	if (!reading) {
1885	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1886
1887	writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1888	nfc->regs + NDCR);
1889	}
1890
1891	return 0;
1892	}
1893
1894	static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1895	const struct nand_subop *subop)
1896	{
1897	struct marvell_nfc_op nfc_op;
1898	int ret;
1899
1900	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
1901
1902	/*
1903	* Naked access are different in that they need to be flagged as naked
1904	* by the controller. Reset the controller registers fields that inform
1905	* on the type and refill them according to the ongoing operation.
1906	*/
1907	nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1908	NDCB0_CMD_XTYPE(XTYPE_MASK));
1909	switch (subop->instrs[0].type) {
1910	case NAND_OP_CMD_INSTR:
1911	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1912	break;
1913	case NAND_OP_ADDR_INSTR:
1914	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1915	break;
1916	case NAND_OP_DATA_IN_INSTR:
1917	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1918	NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1919	break;
1920	case NAND_OP_DATA_OUT_INSTR:
1921	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1922	NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1923	break;
1924	default:
1925	/* This should never happen */
1926	break;
1927	}
1928
1929	ret = marvell_nfc_prepare_cmd(chip);
1930	if (ret)
1931	return ret;
1932
1933	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1934
1935	if (!nfc_op.data_instr) {
1936	ret = marvell_nfc_wait_cmdd(chip);
1937	cond_delay(ns: nfc_op.cle_ale_delay_ns);
1938	return ret;
1939	}
1940
1941	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1942	label: "RDDREQ/WRDREQ while draining raw data");
1943	if (ret)
1944	return ret;
1945
1946	marvell_nfc_xfer_data_pio(chip, subop, nfc_op: &nfc_op);
1947	ret = marvell_nfc_wait_cmdd(chip);
1948	if (ret)
1949	return ret;
1950
1951	/*
1952	* NDCR ND_RUN bit should be cleared automatically at the end of each
1953	* operation but experience shows that the behavior is buggy when it
1954	* comes to writes (with LEN_OVRD). Clear it by hand in this case.
1955	*/
1956	if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1957	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
1958
1959	writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1960	nfc->regs + NDCR);
1961	}
1962
1963	return 0;
1964	}
1965
1966	static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1967	const struct nand_subop *subop)
1968	{
1969	struct marvell_nfc_op nfc_op;
1970	int ret;
1971
1972	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
1973
1974	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
1975	cond_delay(ns: nfc_op.rdy_delay_ns);
1976
1977	return ret;
1978	}
1979
1980	static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1981	const struct nand_subop *subop)
1982	{
1983	struct marvell_nfc_op nfc_op;
1984	int ret;
1985
1986	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
1987	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1988	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1989
1990	ret = marvell_nfc_prepare_cmd(chip);
1991	if (ret)
1992	return ret;
1993
1994	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
1995	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1996	label: "RDDREQ while reading ID");
1997	if (ret)
1998	return ret;
1999
2000	cond_delay(ns: nfc_op.cle_ale_delay_ns);
2001
2002	if (nfc_op.rdy_timeout_ms) {
2003	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
2004	if (ret)
2005	return ret;
2006	}
2007
2008	cond_delay(ns: nfc_op.rdy_delay_ns);
2009
2010	marvell_nfc_xfer_data_pio(chip, subop, nfc_op: &nfc_op);
2011	ret = marvell_nfc_wait_cmdd(chip);
2012	if (ret)
2013	return ret;
2014
2015	cond_delay(ns: nfc_op.data_delay_ns);
2016
2017	return 0;
2018	}
2019
2020	static int marvell_nfc_read_status_exec(struct nand_chip *chip,
2021	const struct nand_subop *subop)
2022	{
2023	struct marvell_nfc_op nfc_op;
2024	int ret;
2025
2026	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
2027	nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
2028	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
2029
2030	ret = marvell_nfc_prepare_cmd(chip);
2031	if (ret)
2032	return ret;
2033
2034	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
2035	ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
2036	label: "RDDREQ while reading status");
2037	if (ret)
2038	return ret;
2039
2040	cond_delay(ns: nfc_op.cle_ale_delay_ns);
2041
2042	if (nfc_op.rdy_timeout_ms) {
2043	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
2044	if (ret)
2045	return ret;
2046	}
2047
2048	cond_delay(ns: nfc_op.rdy_delay_ns);
2049
2050	marvell_nfc_xfer_data_pio(chip, subop, nfc_op: &nfc_op);
2051	ret = marvell_nfc_wait_cmdd(chip);
2052	if (ret)
2053	return ret;
2054
2055	cond_delay(ns: nfc_op.data_delay_ns);
2056
2057	return 0;
2058	}
2059
2060	static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
2061	const struct nand_subop *subop)
2062	{
2063	struct marvell_nfc_op nfc_op;
2064	int ret;
2065
2066	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
2067	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
2068
2069	ret = marvell_nfc_prepare_cmd(chip);
2070	if (ret)
2071	return ret;
2072
2073	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
2074	ret = marvell_nfc_wait_cmdd(chip);
2075	if (ret)
2076	return ret;
2077
2078	cond_delay(ns: nfc_op.cle_ale_delay_ns);
2079
2080	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
2081	if (ret)
2082	return ret;
2083
2084	cond_delay(ns: nfc_op.rdy_delay_ns);
2085
2086	return 0;
2087	}
2088
2089	static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2090	const struct nand_subop *subop)
2091	{
2092	struct marvell_nfc_op nfc_op;
2093	int ret;
2094
2095	marvell_nfc_parse_instructions(chip, subop, nfc_op: &nfc_op);
2096	nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2097
2098	ret = marvell_nfc_prepare_cmd(chip);
2099	if (ret)
2100	return ret;
2101
2102	marvell_nfc_send_cmd(chip, nfc_op: &nfc_op);
2103	ret = marvell_nfc_wait_cmdd(chip);
2104	if (ret)
2105	return ret;
2106
2107	cond_delay(ns: nfc_op.cle_ale_delay_ns);
2108
2109	ret = marvell_nfc_wait_op(chip, timeout_ms: nfc_op.rdy_timeout_ms);
2110	if (ret)
2111	return ret;
2112
2113	cond_delay(ns: nfc_op.rdy_delay_ns);
2114
2115	return 0;
2116	}
2117
2118	static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2119	/* Monolithic reads/writes */
2120	NAND_OP_PARSER_PATTERN(
2121	marvell_nfc_monolithic_access_exec,
2122	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2123	NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2124	NAND_OP_PARSER_PAT_CMD_ELEM(true),
2125	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2126	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2127	NAND_OP_PARSER_PATTERN(
2128	marvell_nfc_monolithic_access_exec,
2129	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2130	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2131	NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2132	NAND_OP_PARSER_PAT_CMD_ELEM(true),
2133	NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2134	/* Naked commands */
2135	NAND_OP_PARSER_PATTERN(
2136	marvell_nfc_naked_access_exec,
2137	NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2138	NAND_OP_PARSER_PATTERN(
2139	marvell_nfc_naked_access_exec,
2140	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2141	NAND_OP_PARSER_PATTERN(
2142	marvell_nfc_naked_access_exec,
2143	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2144	NAND_OP_PARSER_PATTERN(
2145	marvell_nfc_naked_access_exec,
2146	NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2147	NAND_OP_PARSER_PATTERN(
2148	marvell_nfc_naked_waitrdy_exec,
2149	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2150	);
2151
2152	static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2153	/* Naked commands not supported, use a function for each pattern */
2154	NAND_OP_PARSER_PATTERN(
2155	marvell_nfc_read_id_type_exec,
2156	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2157	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2158	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2159	NAND_OP_PARSER_PATTERN(
2160	marvell_nfc_erase_cmd_type_exec,
2161	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2162	NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2163	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2164	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2165	NAND_OP_PARSER_PATTERN(
2166	marvell_nfc_read_status_exec,
2167	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2168	NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2169	NAND_OP_PARSER_PATTERN(
2170	marvell_nfc_reset_cmd_type_exec,
2171	NAND_OP_PARSER_PAT_CMD_ELEM(false),
2172	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2173	NAND_OP_PARSER_PATTERN(
2174	marvell_nfc_naked_waitrdy_exec,
2175	NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2176	);
2177
2178	static int marvell_nfc_exec_op(struct nand_chip *chip,
2179	const struct nand_operation *op,
2180	bool check_only)
2181	{
2182	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
2183
2184	if (!check_only)
2185	marvell_nfc_select_target(chip, die_nr: op->cs);
2186
2187	if (nfc->caps->is_nfcv2)
2188	return nand_op_parser_exec_op(chip, parser: &marvell_nfcv2_op_parser,
2189	op, check_only);
2190	else
2191	return nand_op_parser_exec_op(chip, parser: &marvell_nfcv1_op_parser,
2192	op, check_only);
2193	}
2194
2195	/*
2196	* Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2197	* usable.
2198	*/
2199	static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2200	struct mtd_oob_region *oobregion)
2201	{
2202	struct nand_chip *chip = mtd_to_nand(mtd);
2203	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2204
2205	if (section)
2206	return -ERANGE;
2207
2208	oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2209	lt->last_ecc_bytes;
2210	oobregion->offset = mtd->oobsize - oobregion->length;
2211
2212	return 0;
2213	}
2214
2215	static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2216	struct mtd_oob_region *oobregion)
2217	{
2218	struct nand_chip *chip = mtd_to_nand(mtd);
2219	const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2220
2221	if (section)
2222	return -ERANGE;
2223
2224	/*
2225	* Bootrom looks in bytes 0 & 5 for bad blocks for the
2226	* 4KB page / 4bit BCH combination.
2227	*/
2228	if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2229	oobregion->offset = 6;
2230	else
2231	oobregion->offset = 2;
2232
2233	oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2234	lt->last_spare_bytes - oobregion->offset;
2235
2236	return 0;
2237	}
2238
2239	static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2240	.ecc = marvell_nand_ooblayout_ecc,
2241	.free = marvell_nand_ooblayout_free,
2242	};
2243
2244	static int marvell_nand_hw_ecc_controller_init(struct mtd_info *mtd,
2245	struct nand_ecc_ctrl *ecc)
2246	{
2247	struct nand_chip *chip = mtd_to_nand(mtd);
2248	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
2249	const struct marvell_hw_ecc_layout *l;
2250	int i;
2251
2252	if (!nfc->caps->is_nfcv2 &&
2253	(mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2254	dev_err(nfc->dev,
2255	"NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2256	mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2257	return -ENOTSUPP;
2258	}
2259
2260	to_marvell_nand(chip)->layout = NULL;
2261	for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2262	l = &marvell_nfc_layouts[i];
2263	if (mtd->writesize == l->writesize &&
2264	ecc->size == l->chunk && ecc->strength == l->strength) {
2265	to_marvell_nand(chip)->layout = l;
2266	break;
2267	}
2268	}
2269
2270	if (!to_marvell_nand(chip)->layout ||
2271	(!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2272	dev_err(nfc->dev,
2273	"ECC strength %d at page size %d is not supported\n",
2274	ecc->strength, mtd->writesize);
2275	return -ENOTSUPP;
2276	}
2277
2278	/* Special care for the layout 2k/8-bit/512B */
2279	if (l->writesize == 2048 && l->strength == 8) {
2280	if (mtd->oobsize < 128) {
2281	dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2282	return -ENOTSUPP;
2283	} else {
2284	chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2285	}
2286	}
2287
2288	mtd_set_ooblayout(mtd, ooblayout: &marvell_nand_ooblayout_ops);
2289	ecc->steps = l->nchunks;
2290	ecc->size = l->data_bytes;
2291
2292	if (ecc->strength == 1) {
2293	chip->ecc.algo = NAND_ECC_ALGO_HAMMING;
2294	ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2295	ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2296	ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2297	ecc->read_oob = ecc->read_oob_raw;
2298	ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2299	ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2300	ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2301	ecc->write_oob = ecc->write_oob_raw;
2302	} else {
2303	chip->ecc.algo = NAND_ECC_ALGO_BCH;
2304	ecc->strength = 16;
2305	ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2306	ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2307	ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2308	ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2309	ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2310	ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2311	ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2312	ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2313	}
2314
2315	return 0;
2316	}
2317
2318	static int marvell_nand_ecc_init(struct mtd_info *mtd,
2319	struct nand_ecc_ctrl *ecc)
2320	{
2321	struct nand_chip *chip = mtd_to_nand(mtd);
2322	const struct nand_ecc_props *requirements =
2323	nanddev_get_ecc_requirements(nand: &chip->base);
2324	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
2325	int ret;
2326
2327	if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_NONE &&
2328	(!ecc->size || !ecc->strength)) {
2329	if (requirements->step_size && requirements->strength) {
2330	ecc->size = requirements->step_size;
2331	ecc->strength = requirements->strength;
2332	} else {
2333	dev_info(nfc->dev,
2334	"No minimum ECC strength, using 1b/512B\n");
2335	ecc->size = 512;
2336	ecc->strength = 1;
2337	}
2338	}
2339
2340	switch (ecc->engine_type) {
2341	case NAND_ECC_ENGINE_TYPE_ON_HOST:
2342	ret = marvell_nand_hw_ecc_controller_init(mtd, ecc);
2343	if (ret)
2344	return ret;
2345	break;
2346	case NAND_ECC_ENGINE_TYPE_NONE:
2347	case NAND_ECC_ENGINE_TYPE_SOFT:
2348	case NAND_ECC_ENGINE_TYPE_ON_DIE:
2349	if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2350	mtd->writesize != SZ_2K) {
2351	dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2352	mtd->writesize);
2353	return -EINVAL;
2354	}
2355	break;
2356	default:
2357	return -EINVAL;
2358	}
2359
2360	return 0;
2361	}
2362
2363	static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2364	static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2365
2366	static struct nand_bbt_descr bbt_main_descr = {
2367	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2368	NAND_BBT_2BIT | NAND_BBT_VERSION,
2369	.offs = 8,
2370	.len = 6,
2371	.veroffs = 14,
2372	.maxblocks = 8, /* Last 8 blocks in each chip */
2373	.pattern = bbt_pattern
2374	};
2375
2376	static struct nand_bbt_descr bbt_mirror_descr = {
2377	.options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2378	NAND_BBT_2BIT | NAND_BBT_VERSION,
2379	.offs = 8,
2380	.len = 6,
2381	.veroffs = 14,
2382	.maxblocks = 8, /* Last 8 blocks in each chip */
2383	.pattern = bbt_mirror_pattern
2384	};
2385
2386	static int marvell_nfc_setup_interface(struct nand_chip *chip, int chipnr,
2387	const struct nand_interface_config *conf)
2388	{
2389	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2390	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
2391	unsigned int period_ns = 1000000000 / clk_get_rate(clk: nfc->core_clk) * 2;
2392	const struct nand_sdr_timings *sdr;
2393	struct marvell_nfc_timings nfc_tmg;
2394	int read_delay;
2395
2396	sdr = nand_get_sdr_timings(conf);
2397	if (IS_ERR(ptr: sdr))
2398	return PTR_ERR(ptr: sdr);
2399
2400	if (nfc->caps->max_mode_number && nfc->caps->max_mode_number < conf->timings.mode)
2401	return -EOPNOTSUPP;
2402
2403	/*
2404	* SDR timings are given in pico-seconds while NFC timings must be
2405	* expressed in NAND controller clock cycles, which is half of the
2406	* frequency of the accessible ECC clock retrieved by clk_get_rate().
2407	* This is not written anywhere in the datasheet but was observed
2408	* with an oscilloscope.
2409	*
2410	* NFC datasheet gives equations from which thoses calculations
2411	* are derived, they tend to be slightly more restrictives than the
2412	* given core timings and may improve the overall speed.
2413	*/
2414	nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2415	nfc_tmg.tRH = nfc_tmg.tRP;
2416	nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2417	nfc_tmg.tWH = nfc_tmg.tWP;
2418	nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2419	nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2420	nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2421	/*
2422	* Read delay is the time of propagation from SoC pins to NFC internal
2423	* logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2424	* EDO mode, an additional delay of tRH must be taken into account so
2425	* the data is sampled on the falling edge instead of the rising edge.
2426	*/
2427	read_delay = sdr->tRC_min >= 30000 ?
2428	MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2429
2430	nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2431	/*
2432	* tWHR and tRHW are supposed to be read to write delays (and vice
2433	* versa) but in some cases, ie. when doing a change column, they must
2434	* be greater than that to be sure tCCS delay is respected.
2435	*/
2436	nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2437	period_ns) - 2;
2438	nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2439	period_ns);
2440
2441	/*
2442	* NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2443	* NFCv1: No WAIT_MODE, tR must be maximal.
2444	*/
2445	if (nfc->caps->is_nfcv2) {
2446	nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2447	} else {
2448	nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2449	period_ns);
2450	if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2451	nfc_tmg.tR = nfc_tmg.tCH - 3;
2452	else
2453	nfc_tmg.tR = 0;
2454	}
2455
2456	if (chipnr < 0)
2457	return 0;
2458
2459	marvell_nand->ndtr0 =
2460	NDTR0_TRP(nfc_tmg.tRP) |
2461	NDTR0_TRH(nfc_tmg.tRH) |
2462	NDTR0_ETRP(nfc_tmg.tRP) |
2463	NDTR0_TWP(nfc_tmg.tWP) |
2464	NDTR0_TWH(nfc_tmg.tWH) |
2465	NDTR0_TCS(nfc_tmg.tCS) |
2466	NDTR0_TCH(nfc_tmg.tCH);
2467
2468	marvell_nand->ndtr1 =
2469	NDTR1_TAR(nfc_tmg.tAR) |
2470	NDTR1_TWHR(nfc_tmg.tWHR) |
2471	NDTR1_TR(nfc_tmg.tR);
2472
2473	if (nfc->caps->is_nfcv2) {
2474	marvell_nand->ndtr0 |=
2475	NDTR0_RD_CNT_DEL(read_delay) |
2476	NDTR0_SELCNTR |
2477	NDTR0_TADL(nfc_tmg.tADL);
2478
2479	marvell_nand->ndtr1 |=
2480	NDTR1_TRHW(nfc_tmg.tRHW) |
2481	NDTR1_WAIT_MODE;
2482	}
2483
2484	/*
2485	* Reset nfc->selected_chip so the next command will cause the timing
2486	* registers to be updated in marvell_nfc_select_target().
2487	*/
2488	nfc->selected_chip = NULL;
2489
2490	return 0;
2491	}
2492
2493	static int marvell_nand_attach_chip(struct nand_chip *chip)
2494	{
2495	struct mtd_info *mtd = nand_to_mtd(chip);
2496	struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2497	struct marvell_nfc *nfc = to_marvell_nfc(ctrl: chip->controller);
2498	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev: nfc->dev);
2499	int ret;
2500
2501	if (pdata && pdata->flash_bbt)
2502	chip->bbt_options |= NAND_BBT_USE_FLASH;
2503
2504	if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2505	/*
2506	* We'll use a bad block table stored in-flash and don't
2507	* allow writing the bad block marker to the flash.
2508	*/
2509	chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2510	chip->bbt_td = &bbt_main_descr;
2511	chip->bbt_md = &bbt_mirror_descr;
2512	}
2513
2514	/* Save the chip-specific fields of NDCR */
2515	marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2516	if (chip->options & NAND_BUSWIDTH_16)
2517	marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2518
2519	/*
2520	* On small page NANDs, only one cycle is needed to pass the
2521	* column address.
2522	*/
2523	if (mtd->writesize <= 512) {
2524	marvell_nand->addr_cyc = 1;
2525	} else {
2526	marvell_nand->addr_cyc = 2;
2527	marvell_nand->ndcr |= NDCR_RA_START;
2528	}
2529
2530	/*
2531	* Now add the number of cycles needed to pass the row
2532	* address.
2533	*
2534	* Addressing a chip using CS 2 or 3 should also need the third row
2535	* cycle but due to inconsistance in the documentation and lack of
2536	* hardware to test this situation, this case is not supported.
2537	*/
2538	if (chip->options & NAND_ROW_ADDR_3)
2539	marvell_nand->addr_cyc += 3;
2540	else
2541	marvell_nand->addr_cyc += 2;
2542
2543	if (pdata) {
2544	chip->ecc.size = pdata->ecc_step_size;
2545	chip->ecc.strength = pdata->ecc_strength;
2546	}
2547
2548	ret = marvell_nand_ecc_init(mtd, ecc: &chip->ecc);
2549	if (ret) {
2550	dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2551	return ret;
2552	}
2553
2554	if (chip->ecc.engine_type == NAND_ECC_ENGINE_TYPE_ON_HOST) {
2555	/*
2556	* Subpage write not available with hardware ECC, prohibit also
2557	* subpage read as in userspace subpage access would still be
2558	* allowed and subpage write, if used, would lead to numerous
2559	* uncorrectable ECC errors.
2560	*/
2561	chip->options |= NAND_NO_SUBPAGE_WRITE;
2562	}
2563
2564	if (pdata || nfc->caps->legacy_of_bindings) {
2565	/*
2566	* We keep the MTD name unchanged to avoid breaking platforms
2567	* where the MTD cmdline parser is used and the bootloader
2568	* has not been updated to use the new naming scheme.
2569	*/
2570	mtd->name = "pxa3xx_nand-0";
2571	} else if (!mtd->name) {
2572	/*
2573	* If the new bindings are used and the bootloader has not been
2574	* updated to pass a new mtdparts parameter on the cmdline, you
2575	* should define the following property in your NAND node, ie:
2576	*
2577	* label = "main-storage";
2578	*
2579	* This way, mtd->name will be set by the core when
2580	* nand_set_flash_node() is called.
2581	*/
2582	mtd->name = devm_kasprintf(dev: nfc->dev, GFP_KERNEL,
2583	fmt: "%s:nand.%d", dev_name(dev: nfc->dev),
2584	marvell_nand->sels[0].cs);
2585	if (!mtd->name) {
2586	dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2587	return -ENOMEM;
2588	}
2589	}
2590
2591	return 0;
2592	}
2593
2594	static const struct nand_controller_ops marvell_nand_controller_ops = {
2595	.attach_chip = marvell_nand_attach_chip,
2596	.exec_op = marvell_nfc_exec_op,
2597	.setup_interface = marvell_nfc_setup_interface,
2598	};
2599
2600	static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2601	struct device_node *np)
2602	{
2603	struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2604	struct marvell_nand_chip *marvell_nand;
2605	struct mtd_info *mtd;
2606	struct nand_chip *chip;
2607	int nsels, ret, i;
2608	u32 cs, rb;
2609
2610	/*
2611	* The legacy "num-cs" property indicates the number of CS on the only
2612	* chip connected to the controller (legacy bindings does not support
2613	* more than one chip). The CS and RB pins are always the #0.
2614	*
2615	* When not using legacy bindings, a couple of "reg" and "nand-rb"
2616	* properties must be filled. For each chip, expressed as a subnode,
2617	* "reg" points to the CS lines and "nand-rb" to the RB line.
2618	*/
2619	if (pdata || nfc->caps->legacy_of_bindings) {
2620	nsels = 1;
2621	} else {
2622	nsels = of_property_count_elems_of_size(np, propname: "reg", elem_size: sizeof(u32));
2623	if (nsels <= 0) {
2624	dev_err(dev, "missing/invalid reg property\n");
2625	return -EINVAL;
2626	}
2627	}
2628
2629	/* Alloc the nand chip structure */
2630	marvell_nand = devm_kzalloc(dev,
2631	struct_size(marvell_nand, sels, nsels),
2632	GFP_KERNEL);
2633	if (!marvell_nand) {
2634	dev_err(dev, "could not allocate chip structure\n");
2635	return -ENOMEM;
2636	}
2637
2638	marvell_nand->nsels = nsels;
2639	marvell_nand->selected_die = -1;
2640
2641	for (i = 0; i < nsels; i++) {
2642	if (pdata || nfc->caps->legacy_of_bindings) {
2643	/*
2644	* Legacy bindings use the CS lines in natural
2645	* order (0, 1, ...)
2646	*/
2647	cs = i;
2648	} else {
2649	/* Retrieve CS id */
2650	ret = of_property_read_u32_index(np, propname: "reg", index: i, out_value: &cs);
2651	if (ret) {
2652	dev_err(dev, "could not retrieve reg property: %d\n",
2653	ret);
2654	return ret;
2655	}
2656	}
2657
2658	if (cs >= nfc->caps->max_cs_nb) {
2659	dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2660	cs, nfc->caps->max_cs_nb);
2661	return -EINVAL;
2662	}
2663
2664	if (test_and_set_bit(nr: cs, addr: &nfc->assigned_cs)) {
2665	dev_err(dev, "CS %d already assigned\n", cs);
2666	return -EINVAL;
2667	}
2668
2669	/*
2670	* The cs variable represents the chip select id, which must be
2671	* converted in bit fields for NDCB0 and NDCB2 to select the
2672	* right chip. Unfortunately, due to a lack of information on
2673	* the subject and incoherent documentation, the user should not
2674	* use CS1 and CS3 at all as asserting them is not supported in
2675	* a reliable way (due to multiplexing inside ADDR5 field).
2676	*/
2677	marvell_nand->sels[i].cs = cs;
2678	switch (cs) {
2679	case 0:
2680	case 2:
2681	marvell_nand->sels[i].ndcb0_csel = 0;
2682	break;
2683	case 1:
2684	case 3:
2685	marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2686	break;
2687	default:
2688	return -EINVAL;
2689	}
2690
2691	/* Retrieve RB id */
2692	if (pdata || nfc->caps->legacy_of_bindings) {
2693	/* Legacy bindings always use RB #0 */
2694	rb = 0;
2695	} else {
2696	ret = of_property_read_u32_index(np, propname: "nand-rb", index: i,
2697	out_value: &rb);
2698	if (ret) {
2699	dev_err(dev,
2700	"could not retrieve RB property: %d\n",
2701	ret);
2702	return ret;
2703	}
2704	}
2705
2706	if (rb >= nfc->caps->max_rb_nb) {
2707	dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2708	rb, nfc->caps->max_rb_nb);
2709	return -EINVAL;
2710	}
2711
2712	marvell_nand->sels[i].rb = rb;
2713	}
2714
2715	chip = &marvell_nand->chip;
2716	chip->controller = &nfc->controller;
2717	nand_set_flash_node(chip, np);
2718
2719	if (of_property_read_bool(np, propname: "marvell,nand-keep-config"))
2720	chip->options |= NAND_KEEP_TIMINGS;
2721
2722	mtd = nand_to_mtd(chip);
2723	mtd->dev.parent = dev;
2724
2725	/*
2726	* Save a reference value for timing registers before
2727	* ->setup_interface() is called.
2728	*/
2729	marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2730	marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2731
2732	chip->options |= NAND_BUSWIDTH_AUTO;
2733
2734	ret = nand_scan(chip, max_chips: marvell_nand->nsels);
2735	if (ret) {
2736	dev_err(dev, "could not scan the nand chip\n");
2737	return ret;
2738	}
2739
2740	if (pdata)
2741	/* Legacy bindings support only one chip */
2742	ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
2743	else
2744	ret = mtd_device_register(mtd, NULL, 0);
2745	if (ret) {
2746	dev_err(dev, "failed to register mtd device: %d\n", ret);
2747	nand_cleanup(chip);
2748	return ret;
2749	}
2750
2751	list_add_tail(new: &marvell_nand->node, head: &nfc->chips);
2752
2753	return 0;
2754	}
2755
2756	static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2757	{
2758	struct marvell_nand_chip *entry, *temp;
2759	struct nand_chip *chip;
2760	int ret;
2761
2762	list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
2763	chip = &entry->chip;
2764	ret = mtd_device_unregister(master: nand_to_mtd(chip));
2765	WARN_ON(ret);
2766	nand_cleanup(chip);
2767	list_del(entry: &entry->node);
2768	}
2769	}
2770
2771	static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2772	{
2773	struct device_node *np = dev->of_node;
2774	struct device_node *nand_np;
2775	int max_cs = nfc->caps->max_cs_nb;
2776	int nchips;
2777	int ret;
2778
2779	if (!np)
2780	nchips = 1;
2781	else
2782	nchips = of_get_child_count(np);
2783
2784	if (nchips > max_cs) {
2785	dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2786	max_cs);
2787	return -EINVAL;
2788	}
2789
2790	/*
2791	* Legacy bindings do not use child nodes to exhibit NAND chip
2792	* properties and layout. Instead, NAND properties are mixed with the
2793	* controller ones, and partitions are defined as direct subnodes of the
2794	* NAND controller node.
2795	*/
2796	if (nfc->caps->legacy_of_bindings) {
2797	ret = marvell_nand_chip_init(dev, nfc, np);
2798	return ret;
2799	}
2800
2801	for_each_child_of_node(np, nand_np) {
2802	ret = marvell_nand_chip_init(dev, nfc, np: nand_np);
2803	if (ret) {
2804	of_node_put(node: nand_np);
2805	goto cleanup_chips;
2806	}
2807	}
2808
2809	return 0;
2810
2811	cleanup_chips:
2812	marvell_nand_chips_cleanup(nfc);
2813
2814	return ret;
2815	}
2816
2817	static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2818	{
2819	struct platform_device *pdev = container_of(nfc->dev,
2820	struct platform_device,
2821	dev);
2822	struct dma_slave_config config = {};
2823	struct resource *r;
2824	int ret;
2825
2826	if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2827	dev_warn(nfc->dev,
2828	"DMA not enabled in configuration\n");
2829	return -ENOTSUPP;
2830	}
2831
2832	ret = dma_set_mask_and_coherent(dev: nfc->dev, DMA_BIT_MASK(32));
2833	if (ret)
2834	return ret;
2835
2836	nfc->dma_chan = dma_request_chan(dev: nfc->dev, name: "data");
2837	if (IS_ERR(ptr: nfc->dma_chan)) {
2838	ret = PTR_ERR(ptr: nfc->dma_chan);
2839	nfc->dma_chan = NULL;
2840	return dev_err_probe(dev: nfc->dev, err: ret, fmt: "DMA channel request failed\n");
2841	}
2842
2843	r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2844	if (!r) {
2845	ret = -ENXIO;
2846	goto release_channel;
2847	}
2848
2849	config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2850	config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2851	config.src_addr = r->start + NDDB;
2852	config.dst_addr = r->start + NDDB;
2853	config.src_maxburst = 32;
2854	config.dst_maxburst = 32;
2855	ret = dmaengine_slave_config(chan: nfc->dma_chan, config: &config);
2856	if (ret < 0) {
2857	dev_err(nfc->dev, "Failed to configure DMA channel\n");
2858	goto release_channel;
2859	}
2860
2861	/*
2862	* DMA must act on length multiple of 32 and this length may be
2863	* bigger than the destination buffer. Use this buffer instead
2864	* for DMA transfers and then copy the desired amount of data to
2865	* the provided buffer.
2866	*/
2867	nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
2868	if (!nfc->dma_buf) {
2869	ret = -ENOMEM;
2870	goto release_channel;
2871	}
2872
2873	nfc->use_dma = true;
2874
2875	return 0;
2876
2877	release_channel:
2878	dma_release_channel(chan: nfc->dma_chan);
2879	nfc->dma_chan = NULL;
2880
2881	return ret;
2882	}
2883
2884	static void marvell_nfc_reset(struct marvell_nfc *nfc)
2885	{
2886	/*
2887	* ECC operations and interruptions are only enabled when specifically
2888	* needed. ECC shall not be activated in the early stages (fails probe).
2889	* Arbiter flag, even if marked as "reserved", must be set (empirical).
2890	* SPARE_EN bit must always be set or ECC bytes will not be at the same
2891	* offset in the read page and this will fail the protection.
2892	*/
2893	writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2894	NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2895	writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2896	writel_relaxed(0, nfc->regs + NDECCCTRL);
2897	}
2898
2899	static int marvell_nfc_init(struct marvell_nfc *nfc)
2900	{
2901	struct device_node *np = nfc->dev->of_node;
2902
2903	/*
2904	* Some SoCs like A7k/A8k need to enable manually the NAND
2905	* controller, gated clocks and reset bits to avoid being bootloader
2906	* dependent. This is done through the use of the System Functions
2907	* registers.
2908	*/
2909	if (nfc->caps->need_system_controller) {
2910	struct regmap *sysctrl_base =
2911	syscon_regmap_lookup_by_phandle(np,
2912	property: "marvell,system-controller");
2913
2914	if (IS_ERR(ptr: sysctrl_base))
2915	return PTR_ERR(ptr: sysctrl_base);
2916
2917	regmap_write(map: sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2918	GENCONF_SOC_DEVICE_MUX_NFC_EN |
2919	GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2920	GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2921	GENCONF_SOC_DEVICE_MUX_NFC_INT_EN |
2922	GENCONF_SOC_DEVICE_MUX_NFC_DEVBUS_ARB_EN);
2923
2924	regmap_update_bits(map: sysctrl_base, GENCONF_CLK_GATING_CTRL,
2925	GENCONF_CLK_GATING_CTRL_ND_GATE,
2926	GENCONF_CLK_GATING_CTRL_ND_GATE);
2927	}
2928
2929	/* Configure the DMA if appropriate */
2930	if (!nfc->caps->is_nfcv2)
2931	marvell_nfc_init_dma(nfc);
2932
2933	marvell_nfc_reset(nfc);
2934
2935	return 0;
2936	}
2937
2938	static int marvell_nfc_probe(struct platform_device *pdev)
2939	{
2940	struct device *dev = &pdev->dev;
2941	struct marvell_nfc *nfc;
2942	int ret;
2943	int irq;
2944
2945	nfc = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct marvell_nfc),
2946	GFP_KERNEL);
2947	if (!nfc)
2948	return -ENOMEM;
2949
2950	nfc->dev = dev;
2951	nand_controller_init(nfc: &nfc->controller);
2952	nfc->controller.ops = &marvell_nand_controller_ops;
2953	INIT_LIST_HEAD(list: &nfc->chips);
2954
2955	nfc->regs = devm_platform_ioremap_resource(pdev, index: 0);
2956	if (IS_ERR(ptr: nfc->regs))
2957	return PTR_ERR(ptr: nfc->regs);
2958
2959	irq = platform_get_irq(pdev, 0);
2960	if (irq < 0)
2961	return irq;
2962
2963	nfc->core_clk = devm_clk_get(dev: &pdev->dev, id: "core");
2964
2965	/* Managed the legacy case (when the first clock was not named) */
2966	if (nfc->core_clk == ERR_PTR(error: -ENOENT))
2967	nfc->core_clk = devm_clk_get(dev: &pdev->dev, NULL);
2968
2969	if (IS_ERR(ptr: nfc->core_clk))
2970	return PTR_ERR(ptr: nfc->core_clk);
2971
2972	ret = clk_prepare_enable(clk: nfc->core_clk);
2973	if (ret)
2974	return ret;
2975
2976	nfc->reg_clk = devm_clk_get(dev: &pdev->dev, id: "reg");
2977	if (IS_ERR(ptr: nfc->reg_clk)) {
2978	if (PTR_ERR(ptr: nfc->reg_clk) != -ENOENT) {
2979	ret = PTR_ERR(ptr: nfc->reg_clk);
2980	goto unprepare_core_clk;
2981	}
2982
2983	nfc->reg_clk = NULL;
2984	}
2985
2986	ret = clk_prepare_enable(clk: nfc->reg_clk);
2987	if (ret)
2988	goto unprepare_core_clk;
2989
2990	marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2991	marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2992	ret = devm_request_irq(dev, irq, handler: marvell_nfc_isr,
2993	irqflags: 0, devname: "marvell-nfc", dev_id: nfc);
2994	if (ret)
2995	goto unprepare_reg_clk;
2996
2997	/* Get NAND controller capabilities */
2998	if (pdev->id_entry)
2999	nfc->caps = (void *)pdev->id_entry->driver_data;
3000	else
3001	nfc->caps = of_device_get_match_data(dev: &pdev->dev);
3002
3003	if (!nfc->caps) {
3004	dev_err(dev, "Could not retrieve NFC caps\n");
3005	ret = -EINVAL;
3006	goto unprepare_reg_clk;
3007	}
3008
3009	/* Init the controller and then probe the chips */
3010	ret = marvell_nfc_init(nfc);
3011	if (ret)
3012	goto unprepare_reg_clk;
3013
3014	platform_set_drvdata(pdev, data: nfc);
3015
3016	ret = marvell_nand_chips_init(dev, nfc);
3017	if (ret)
3018	goto release_dma;
3019
3020	return 0;
3021
3022	release_dma:
3023	if (nfc->use_dma)
3024	dma_release_channel(chan: nfc->dma_chan);
3025	unprepare_reg_clk:
3026	clk_disable_unprepare(clk: nfc->reg_clk);
3027	unprepare_core_clk:
3028	clk_disable_unprepare(clk: nfc->core_clk);
3029
3030	return ret;
3031	}
3032
3033	static void marvell_nfc_remove(struct platform_device *pdev)
3034	{
3035	struct marvell_nfc *nfc = platform_get_drvdata(pdev);
3036
3037	marvell_nand_chips_cleanup(nfc);
3038
3039	if (nfc->use_dma) {
3040	dmaengine_terminate_all(chan: nfc->dma_chan);
3041	dma_release_channel(chan: nfc->dma_chan);
3042	}
3043
3044	clk_disable_unprepare(clk: nfc->reg_clk);
3045	clk_disable_unprepare(clk: nfc->core_clk);
3046	}
3047
3048	static int __maybe_unused marvell_nfc_suspend(struct device *dev)
3049	{
3050	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3051	struct marvell_nand_chip *chip;
3052
3053	list_for_each_entry(chip, &nfc->chips, node)
3054	marvell_nfc_wait_ndrun(chip: &chip->chip);
3055
3056	clk_disable_unprepare(clk: nfc->reg_clk);
3057	clk_disable_unprepare(clk: nfc->core_clk);
3058
3059	return 0;
3060	}
3061
3062	static int __maybe_unused marvell_nfc_resume(struct device *dev)
3063	{
3064	struct marvell_nfc *nfc = dev_get_drvdata(dev);
3065	int ret;
3066
3067	ret = clk_prepare_enable(clk: nfc->core_clk);
3068	if (ret < 0)
3069	return ret;
3070
3071	ret = clk_prepare_enable(clk: nfc->reg_clk);
3072	if (ret < 0) {
3073	clk_disable_unprepare(clk: nfc->core_clk);
3074	return ret;
3075	}
3076
3077	/*
3078	* Reset nfc->selected_chip so the next command will cause the timing
3079	* registers to be restored in marvell_nfc_select_target().
3080	*/
3081	nfc->selected_chip = NULL;
3082
3083	/* Reset registers that have lost their contents */
3084	marvell_nfc_reset(nfc);
3085
3086	return 0;
3087	}
3088
3089	static const struct dev_pm_ops marvell_nfc_pm_ops = {
3090	SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3091	};
3092
3093	static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3094	.max_cs_nb = 4,
3095	.max_rb_nb = 2,
3096	.need_system_controller = true,
3097	.is_nfcv2 = true,
3098	};
3099
3100	static const struct marvell_nfc_caps marvell_ac5_caps = {
3101	.max_cs_nb = 2,
3102	.max_rb_nb = 1,
3103	.is_nfcv2 = true,
3104	.max_mode_number = 3,
3105	};
3106
3107	static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3108	.max_cs_nb = 4,
3109	.max_rb_nb = 2,
3110	.is_nfcv2 = true,
3111	};
3112
3113	static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3114	.max_cs_nb = 2,
3115	.max_rb_nb = 1,
3116	.use_dma = true,
3117	};
3118
3119	static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3120	.max_cs_nb = 4,
3121	.max_rb_nb = 2,
3122	.need_system_controller = true,
3123	.legacy_of_bindings = true,
3124	.is_nfcv2 = true,
3125	};
3126
3127	static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3128	.max_cs_nb = 4,
3129	.max_rb_nb = 2,
3130	.legacy_of_bindings = true,
3131	.is_nfcv2 = true,
3132	};
3133
3134	static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3135	.max_cs_nb = 2,
3136	.max_rb_nb = 1,
3137	.legacy_of_bindings = true,
3138	.use_dma = true,
3139	};
3140
3141	static const struct platform_device_id marvell_nfc_platform_ids[] = {
3142	{
3143	.name = "pxa3xx-nand",
3144	.driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3145	},
3146	{ /* sentinel */ },
3147	};
3148	MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3149
3150	static const struct of_device_id marvell_nfc_of_ids[] = {
3151	{
3152	.compatible = "marvell,armada-8k-nand-controller",
3153	.data = &marvell_armada_8k_nfc_caps,
3154	},
3155	{
3156	.compatible = "marvell,ac5-nand-controller",
3157	.data = &marvell_ac5_caps,
3158	},
3159	{
3160	.compatible = "marvell,armada370-nand-controller",
3161	.data = &marvell_armada370_nfc_caps,
3162	},
3163	{
3164	.compatible = "marvell,pxa3xx-nand-controller",
3165	.data = &marvell_pxa3xx_nfc_caps,
3166	},
3167	/* Support for old/deprecated bindings: */
3168	{
3169	.compatible = "marvell,armada-8k-nand",
3170	.data = &marvell_armada_8k_nfc_legacy_caps,
3171	},
3172	{
3173	.compatible = "marvell,armada370-nand",
3174	.data = &marvell_armada370_nfc_legacy_caps,
3175	},
3176	{
3177	.compatible = "marvell,pxa3xx-nand",
3178	.data = &marvell_pxa3xx_nfc_legacy_caps,
3179	},
3180	{ /* sentinel */ },
3181	};
3182	MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3183
3184	static struct platform_driver marvell_nfc_driver = {
3185	.driver = {
3186	.name = "marvell-nfc",
3187	.of_match_table = marvell_nfc_of_ids,
3188	.pm = &marvell_nfc_pm_ops,
3189	},
3190	.id_table = marvell_nfc_platform_ids,
3191	.probe = marvell_nfc_probe,
3192	.remove_new = marvell_nfc_remove,
3193	};
3194	module_platform_driver(marvell_nfc_driver);
3195
3196	MODULE_LICENSE("GPL");
3197	MODULE_DESCRIPTION("Marvell NAND controller driver");
3198