1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Driver for Pondicherry2 memory controller.
4	*
5	* Copyright (c) 2016, Intel Corporation.
6	*
7	* [Derived from sb_edac.c]
8	*
9	* Translation of system physical addresses to DIMM addresses
10	* is a two stage process:
11	*
12	* First the Pondicherry 2 memory controller handles slice and channel interleaving
13	* in "sys2pmi()". This is (almost) completley common between platforms.
14	*
15	* Then a platform specific dunit (DIMM unit) completes the process to provide DIMM,
16	* rank, bank, row and column using the appropriate "dunit_ops" functions/parameters.
17	*/
18
19	#include <linux/bitmap.h>
20	#include <linux/delay.h>
21	#include <linux/edac.h>
22	#include <linux/init.h>
23	#include <linux/math64.h>
24	#include <linux/mmzone.h>
25	#include <linux/mod_devicetable.h>
26	#include <linux/module.h>
27	#include <linux/pci.h>
28	#include <linux/pci_ids.h>
29	#include <linux/sizes.h>
30	#include <linux/slab.h>
31	#include <linux/smp.h>
32
33	#include <linux/platform_data/x86/p2sb.h>
34
35	#include <asm/cpu_device_id.h>
36	#include <asm/intel-family.h>
37	#include <asm/processor.h>
38	#include <asm/mce.h>
39
40	#include "edac_mc.h"
41	#include "edac_module.h"
42	#include "pnd2_edac.h"
43
44	#define EDAC_MOD_STR "pnd2_edac"
45
46	#define APL_NUM_CHANNELS 4
47	#define DNV_NUM_CHANNELS 2
48	#define DNV_MAX_DIMMS 2 /* Max DIMMs per channel */
49
50	enum type {
51	APL,
52	DNV, /* All requests go to PMI CH0 on each slice (CH1 disabled) */
53	};
54
55	struct dram_addr {
56	int chan;
57	int dimm;
58	int rank;
59	int bank;
60	int row;
61	int col;
62	};
63
64	struct pnd2_pvt {
65	int dimm_geom[APL_NUM_CHANNELS];
66	u64 tolm, tohm;
67	};
68
69	/*
70	* System address space is divided into multiple regions with
71	* different interleave rules in each. The as0/as1 regions
72	* have no interleaving at all. The as2 region is interleaved
73	* between two channels. The mot region is magic and may overlap
74	* other regions, with its interleave rules taking precedence.
75	* Addresses not in any of these regions are interleaved across
76	* all four channels.
77	*/
78	static struct region {
79	u64 base;
80	u64 limit;
81	u8 enabled;
82	} mot, as0, as1, as2;
83
84	static struct dunit_ops {
85	char *name;
86	enum type type;
87	int pmiaddr_shift;
88	int pmiidx_shift;
89	int channels;
90	int dimms_per_channel;
91	int (*rd_reg)(int port, int off, int op, void *data, size_t sz, char *name);
92	int (*get_registers)(void);
93	int (*check_ecc)(void);
94	void (*mk_region)(char *name, struct region *rp, void *asym);
95	void (*get_dimm_config)(struct mem_ctl_info *mci);
96	int (*pmi2mem)(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
97	struct dram_addr *daddr, char *msg);
98	} *ops;
99
100	static struct mem_ctl_info *pnd2_mci;
101
102	#define PND2_MSG_SIZE 256
103
104	/* Debug macros */
105	#define pnd2_printk(level, fmt, arg...) \
106	edac_printk(level, "pnd2", fmt, ##arg)
107
108	#define pnd2_mc_printk(mci, level, fmt, arg...) \
109	edac_mc_chipset_printk(mci, level, "pnd2", fmt, ##arg)
110
111	#define MOT_CHAN_INTLV_BIT_1SLC_2CH 12
112	#define MOT_CHAN_INTLV_BIT_2SLC_2CH 13
113	#define SELECTOR_DISABLED (-1)
114
115	#define PMI_ADDRESS_WIDTH 31
116	#define PND_MAX_PHYS_BIT 39
117
118	#define APL_ASYMSHIFT 28
119	#define DNV_ASYMSHIFT 31
120	#define CH_HASH_MASK_LSB 6
121	#define SLICE_HASH_MASK_LSB 6
122	#define MOT_SLC_INTLV_BIT 12
123	#define LOG2_PMI_ADDR_GRANULARITY 5
124	#define MOT_SHIFT 24
125
126	#define GET_BITFIELD(v, lo, hi) (((v) & GENMASK_ULL(hi, lo)) >> (lo))
127	#define U64_LSHIFT(val, s) ((u64)(val) << (s))
128
129	/*
130	* On Apollo Lake we access memory controller registers via a
131	* side-band mailbox style interface in a hidden PCI device
132	* configuration space.
133	*/
134	static struct pci_bus *p2sb_bus;
135	#define P2SB_DEVFN PCI_DEVFN(0xd, 0)
136	#define P2SB_ADDR_OFF 0xd0
137	#define P2SB_DATA_OFF 0xd4
138	#define P2SB_STAT_OFF 0xd8
139	#define P2SB_ROUT_OFF 0xda
140	#define P2SB_EADD_OFF 0xdc
141	#define P2SB_HIDE_OFF 0xe1
142
143	#define P2SB_BUSY 1
144
145	#define P2SB_READ(size, off, ptr) \
146	pci_bus_read_config_##size(p2sb_bus, P2SB_DEVFN, off, ptr)
147	#define P2SB_WRITE(size, off, val) \
148	pci_bus_write_config_##size(p2sb_bus, P2SB_DEVFN, off, val)
149
150	static bool p2sb_is_busy(u16 *status)
151	{
152	P2SB_READ(word, P2SB_STAT_OFF, status);
153
154	return !!(*status & P2SB_BUSY);
155	}
156
157	static int _apl_rd_reg(int port, int off, int op, u32 *data)
158	{
159	int retries = 0xff, ret;
160	u16 status;
161	u8 hidden;
162
163	/* Unhide the P2SB device, if it's hidden */
164	P2SB_READ(byte, P2SB_HIDE_OFF, &hidden);
165	if (hidden)
166	P2SB_WRITE(byte, P2SB_HIDE_OFF, 0);
167
168	if (p2sb_is_busy(status: &status)) {
169	ret = -EAGAIN;
170	goto out;
171	}
172
173	P2SB_WRITE(dword, P2SB_ADDR_OFF, (port << 24) | off);
174	P2SB_WRITE(dword, P2SB_DATA_OFF, 0);
175	P2SB_WRITE(dword, P2SB_EADD_OFF, 0);
176	P2SB_WRITE(word, P2SB_ROUT_OFF, 0);
177	P2SB_WRITE(word, P2SB_STAT_OFF, (op << 8) | P2SB_BUSY);
178
179	while (p2sb_is_busy(status: &status)) {
180	if (retries-- == 0) {
181	ret = -EBUSY;
182	goto out;
183	}
184	}
185
186	P2SB_READ(dword, P2SB_DATA_OFF, data);
187	ret = (status >> 1) & GENMASK(1, 0);
188	out:
189	/* Hide the P2SB device, if it was hidden before */
190	if (hidden)
191	P2SB_WRITE(byte, P2SB_HIDE_OFF, hidden);
192
193	return ret;
194	}
195
196	static int apl_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
197	{
198	int ret = 0;
199
200	edac_dbg(2, "Read %s port=%x off=%x op=%x\n", name, port, off, op);
201	switch (sz) {
202	case 8:
203	ret = _apl_rd_reg(port, off: off + 4, op, data: (u32 *)(data + 4));
204	fallthrough;
205	case 4:
206	ret |= _apl_rd_reg(port, off, op, data: (u32 *)data);
207	pnd2_printk(KERN_DEBUG, "%s=%x%08x ret=%d\n", name,
208	sz == 8 ? *((u32 *)(data + 4)) : 0, *((u32 *)data), ret);
209	break;
210	}
211
212	return ret;
213	}
214
215	static u64 get_mem_ctrl_hub_base_addr(void)
216	{
217	struct b_cr_mchbar_lo_pci lo;
218	struct b_cr_mchbar_hi_pci hi;
219	struct pci_dev *pdev;
220
221	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, device: 0x1980, NULL);
222	if (pdev) {
223	pci_read_config_dword(dev: pdev, where: 0x48, val: (u32 *)&lo);
224	pci_read_config_dword(dev: pdev, where: 0x4c, val: (u32 *)&hi);
225	pci_dev_put(dev: pdev);
226	} else {
227	return 0;
228	}
229
230	if (!lo.enable) {
231	edac_dbg(2, "MMIO via memory controller hub base address is disabled!\n");
232	return 0;
233	}
234
235	return U64_LSHIFT(hi.base, 32) | U64_LSHIFT(lo.base, 15);
236	}
237
238	#define DNV_MCHBAR_SIZE 0x8000
239	#define DNV_SB_PORT_SIZE 0x10000
240	static int dnv_rd_reg(int port, int off, int op, void *data, size_t sz, char *name)
241	{
242	struct pci_dev *pdev;
243	void __iomem *base;
244	struct resource r;
245	int ret;
246
247	if (op == 4) {
248	pdev = pci_get_device(PCI_VENDOR_ID_INTEL, device: 0x1980, NULL);
249	if (!pdev)
250	return -ENODEV;
251
252	pci_read_config_dword(dev: pdev, where: off, val: data);
253	pci_dev_put(dev: pdev);
254	} else {
255	/* MMIO via memory controller hub base address */
256	if (op == 0 && port == 0x4c) {
257	memset(&r, 0, sizeof(r));
258
259	r.start = get_mem_ctrl_hub_base_addr();
260	if (!r.start)
261	return -ENODEV;
262	r.end = r.start + DNV_MCHBAR_SIZE - 1;
263	} else {
264	/* MMIO via sideband register base address */
265	ret = p2sb_bar(NULL, devfn: 0, mem: &r);
266	if (ret)
267	return ret;
268
269	r.start += (port << 16);
270	r.end = r.start + DNV_SB_PORT_SIZE - 1;
271	}
272
273	base = ioremap(offset: r.start, size: resource_size(res: &r));
274	if (!base)
275	return -ENODEV;
276
277	if (sz == 8)
278	*(u64 *)data = readq(addr: base + off);
279	else
280	*(u32 *)data = readl(addr: base + off);
281
282	iounmap(addr: base);
283	}
284
285	edac_dbg(2, "Read %s=%.8x_%.8x\n", name,
286	(sz == 8) ? *(u32 *)(data + 4) : 0, *(u32 *)data);
287
288	return 0;
289	}
290
291	#define RD_REGP(regp, regname, port) \
292	ops->rd_reg(port, \
293	regname##_offset, \
294	regname##_r_opcode, \
295	regp, sizeof(struct regname), \
296	#regname)
297
298	#define RD_REG(regp, regname) \
299	ops->rd_reg(regname ## _port, \
300	regname##_offset, \
301	regname##_r_opcode, \
302	regp, sizeof(struct regname), \
303	#regname)
304
305	static u64 top_lm, top_hm;
306	static bool two_slices;
307	static bool two_channels; /* Both PMI channels in one slice enabled */
308
309	static u8 sym_chan_mask;
310	static u8 asym_chan_mask;
311	static unsigned long chan_mask;
312
313	static int slice_selector = -1;
314	static int chan_selector = -1;
315	static u64 slice_hash_mask;
316	static u64 chan_hash_mask;
317
318	static void mk_region(char *name, struct region *rp, u64 base, u64 limit)
319	{
320	rp->enabled = 1;
321	rp->base = base;
322	rp->limit = limit;
323	edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, limit);
324	}
325
326	static void mk_region_mask(char *name, struct region *rp, u64 base, u64 mask)
327	{
328	if (mask == 0) {
329	pr_info(FW_BUG "MOT mask cannot be zero\n");
330	return;
331	}
332	if (mask != GENMASK_ULL(PND_MAX_PHYS_BIT, __ffs(mask))) {
333	pr_info(FW_BUG "MOT mask is invalid\n");
334	return;
335	}
336	if (base & ~mask) {
337	pr_info(FW_BUG "MOT region base/mask alignment error\n");
338	return;
339	}
340	rp->base = base;
341	rp->limit = (base | ~mask) & GENMASK_ULL(PND_MAX_PHYS_BIT, 0);
342	rp->enabled = 1;
343	edac_dbg(2, "Region:%s [%llx, %llx]\n", name, base, rp->limit);
344	}
345
346	static bool in_region(struct region *rp, u64 addr)
347	{
348	if (!rp->enabled)
349	return false;
350
351	return rp->base <= addr && addr <= rp->limit;
352	}
353
354	static int gen_sym_mask(struct b_cr_slice_channel_hash *p)
355	{
356	int mask = 0;
357
358	if (!p->slice_0_mem_disabled)
359	mask |= p->sym_slice0_channel_enabled;
360
361	if (!p->slice_1_disabled)
362	mask |= p->sym_slice1_channel_enabled << 2;
363
364	if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
365	mask &= 0x5;
366
367	return mask;
368	}
369
370	static int gen_asym_mask(struct b_cr_slice_channel_hash *p,
371	struct b_cr_asym_mem_region0_mchbar *as0,
372	struct b_cr_asym_mem_region1_mchbar *as1,
373	struct b_cr_asym_2way_mem_region_mchbar *as2way)
374	{
375	const int intlv[] = { 0x5, 0xA, 0x3, 0xC };
376	int mask = 0;
377
378	if (as2way->asym_2way_interleave_enable)
379	mask = intlv[as2way->asym_2way_intlv_mode];
380	if (as0->slice0_asym_enable)
381	mask |= (1 << as0->slice0_asym_channel_select);
382	if (as1->slice1_asym_enable)
383	mask |= (4 << as1->slice1_asym_channel_select);
384	if (p->slice_0_mem_disabled)
385	mask &= 0xc;
386	if (p->slice_1_disabled)
387	mask &= 0x3;
388	if (p->ch_1_disabled || p->enable_pmi_dual_data_mode)
389	mask &= 0x5;
390
391	return mask;
392	}
393
394	static struct b_cr_tolud_pci tolud;
395	static struct b_cr_touud_lo_pci touud_lo;
396	static struct b_cr_touud_hi_pci touud_hi;
397	static struct b_cr_asym_mem_region0_mchbar asym0;
398	static struct b_cr_asym_mem_region1_mchbar asym1;
399	static struct b_cr_asym_2way_mem_region_mchbar asym_2way;
400	static struct b_cr_mot_out_base_mchbar mot_base;
401	static struct b_cr_mot_out_mask_mchbar mot_mask;
402	static struct b_cr_slice_channel_hash chash;
403
404	/* Apollo Lake dunit */
405	/*
406	* Validated on board with just two DIMMs in the [0] and [2] positions
407	* in this array. Other port number matches documentation, but caution
408	* advised.
409	*/
410	static const int apl_dports[APL_NUM_CHANNELS] = { 0x18, 0x10, 0x11, 0x19 };
411	static struct d_cr_drp0 drp0[APL_NUM_CHANNELS];
412
413	/* Denverton dunit */
414	static const int dnv_dports[DNV_NUM_CHANNELS] = { 0x10, 0x12 };
415	static struct d_cr_dsch dsch;
416	static struct d_cr_ecc_ctrl ecc_ctrl[DNV_NUM_CHANNELS];
417	static struct d_cr_drp drp[DNV_NUM_CHANNELS];
418	static struct d_cr_dmap dmap[DNV_NUM_CHANNELS];
419	static struct d_cr_dmap1 dmap1[DNV_NUM_CHANNELS];
420	static struct d_cr_dmap2 dmap2[DNV_NUM_CHANNELS];
421	static struct d_cr_dmap3 dmap3[DNV_NUM_CHANNELS];
422	static struct d_cr_dmap4 dmap4[DNV_NUM_CHANNELS];
423	static struct d_cr_dmap5 dmap5[DNV_NUM_CHANNELS];
424
425	static void apl_mk_region(char *name, struct region *rp, void *asym)
426	{
427	struct b_cr_asym_mem_region0_mchbar *a = asym;
428
429	mk_region(name, rp,
430	U64_LSHIFT(a->slice0_asym_base, APL_ASYMSHIFT),
431	U64_LSHIFT(a->slice0_asym_limit, APL_ASYMSHIFT) +
432	GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
433	}
434
435	static void dnv_mk_region(char *name, struct region *rp, void *asym)
436	{
437	struct b_cr_asym_mem_region_denverton *a = asym;
438
439	mk_region(name, rp,
440	U64_LSHIFT(a->slice_asym_base, DNV_ASYMSHIFT),
441	U64_LSHIFT(a->slice_asym_limit, DNV_ASYMSHIFT) +
442	GENMASK_ULL(DNV_ASYMSHIFT - 1, 0));
443	}
444
445	static int apl_get_registers(void)
446	{
447	int ret = -ENODEV;
448	int i;
449
450	if (RD_REG(&asym_2way, b_cr_asym_2way_mem_region_mchbar))
451	return -ENODEV;
452
453	/*
454	* RD_REGP() will fail for unpopulated or non-existent
455	* DIMM slots. Return success if we find at least one DIMM.
456	*/
457	for (i = 0; i < APL_NUM_CHANNELS; i++)
458	if (!RD_REGP(&drp0[i], d_cr_drp0, apl_dports[i]))
459	ret = 0;
460
461	return ret;
462	}
463
464	static int dnv_get_registers(void)
465	{
466	int i;
467
468	if (RD_REG(&dsch, d_cr_dsch))
469	return -ENODEV;
470
471	for (i = 0; i < DNV_NUM_CHANNELS; i++)
472	if (RD_REGP(&ecc_ctrl[i], d_cr_ecc_ctrl, dnv_dports[i]) ||
473	RD_REGP(&drp[i], d_cr_drp, dnv_dports[i]) ||
474	RD_REGP(&dmap[i], d_cr_dmap, dnv_dports[i]) ||
475	RD_REGP(&dmap1[i], d_cr_dmap1, dnv_dports[i]) ||
476	RD_REGP(&dmap2[i], d_cr_dmap2, dnv_dports[i]) ||
477	RD_REGP(&dmap3[i], d_cr_dmap3, dnv_dports[i]) ||
478	RD_REGP(&dmap4[i], d_cr_dmap4, dnv_dports[i]) ||
479	RD_REGP(&dmap5[i], d_cr_dmap5, dnv_dports[i]))
480	return -ENODEV;
481
482	return 0;
483	}
484
485	/*
486	* Read all the h/w config registers once here (they don't
487	* change at run time. Figure out which address ranges have
488	* which interleave characteristics.
489	*/
490	static int get_registers(void)
491	{
492	const int intlv[] = { 10, 11, 12, 12 };
493
494	if (RD_REG(&tolud, b_cr_tolud_pci) ||
495	RD_REG(&touud_lo, b_cr_touud_lo_pci) ||
496	RD_REG(&touud_hi, b_cr_touud_hi_pci) ||
497	RD_REG(&asym0, b_cr_asym_mem_region0_mchbar) ||
498	RD_REG(&asym1, b_cr_asym_mem_region1_mchbar) ||
499	RD_REG(&mot_base, b_cr_mot_out_base_mchbar) ||
500	RD_REG(&mot_mask, b_cr_mot_out_mask_mchbar) ||
501	RD_REG(&chash, b_cr_slice_channel_hash))
502	return -ENODEV;
503
504	if (ops->get_registers())
505	return -ENODEV;
506
507	if (ops->type == DNV) {
508	/* PMI channel idx (always 0) for asymmetric region */
509	asym0.slice0_asym_channel_select = 0;
510	asym1.slice1_asym_channel_select = 0;
511	/* PMI channel bitmap (always 1) for symmetric region */
512	chash.sym_slice0_channel_enabled = 0x1;
513	chash.sym_slice1_channel_enabled = 0x1;
514	}
515
516	if (asym0.slice0_asym_enable)
517	ops->mk_region("as0", &as0, &asym0);
518
519	if (asym1.slice1_asym_enable)
520	ops->mk_region("as1", &as1, &asym1);
521
522	if (asym_2way.asym_2way_interleave_enable) {
523	mk_region(name: "as2way", rp: &as2,
524	U64_LSHIFT(asym_2way.asym_2way_base, APL_ASYMSHIFT),
525	U64_LSHIFT(asym_2way.asym_2way_limit, APL_ASYMSHIFT) +
526	GENMASK_ULL(APL_ASYMSHIFT - 1, 0));
527	}
528
529	if (mot_base.imr_en) {
530	mk_region_mask(name: "mot", rp: &mot,
531	U64_LSHIFT(mot_base.mot_out_base, MOT_SHIFT),
532	U64_LSHIFT(mot_mask.mot_out_mask, MOT_SHIFT));
533	}
534
535	top_lm = U64_LSHIFT(tolud.tolud, 20);
536	top_hm = U64_LSHIFT(touud_hi.touud, 32) | U64_LSHIFT(touud_lo.touud, 20);
537
538	two_slices = !chash.slice_1_disabled &&
539	!chash.slice_0_mem_disabled &&
540	(chash.sym_slice0_channel_enabled != 0) &&
541	(chash.sym_slice1_channel_enabled != 0);
542	two_channels = !chash.ch_1_disabled &&
543	!chash.enable_pmi_dual_data_mode &&
544	((chash.sym_slice0_channel_enabled == 3) ||
545	(chash.sym_slice1_channel_enabled == 3));
546
547	sym_chan_mask = gen_sym_mask(p: &chash);
548	asym_chan_mask = gen_asym_mask(p: &chash, as0: &asym0, as1: &asym1, as2way: &asym_2way);
549	chan_mask = sym_chan_mask | asym_chan_mask;
550
551	if (two_slices && !two_channels) {
552	if (chash.hvm_mode)
553	slice_selector = 29;
554	else
555	slice_selector = intlv[chash.interleave_mode];
556	} else if (!two_slices && two_channels) {
557	if (chash.hvm_mode)
558	chan_selector = 29;
559	else
560	chan_selector = intlv[chash.interleave_mode];
561	} else if (two_slices && two_channels) {
562	if (chash.hvm_mode) {
563	slice_selector = 29;
564	chan_selector = 30;
565	} else {
566	slice_selector = intlv[chash.interleave_mode];
567	chan_selector = intlv[chash.interleave_mode] + 1;
568	}
569	}
570
571	if (two_slices) {
572	if (!chash.hvm_mode)
573	slice_hash_mask = chash.slice_hash_mask << SLICE_HASH_MASK_LSB;
574	if (!two_channels)
575	slice_hash_mask |= BIT_ULL(slice_selector);
576	}
577
578	if (two_channels) {
579	if (!chash.hvm_mode)
580	chan_hash_mask = chash.ch_hash_mask << CH_HASH_MASK_LSB;
581	if (!two_slices)
582	chan_hash_mask |= BIT_ULL(chan_selector);
583	}
584
585	return 0;
586	}
587
588	/* Get a contiguous memory address (remove the MMIO gap) */
589	static u64 remove_mmio_gap(u64 sys)
590	{
591	return (sys < SZ_4G) ? sys : sys - (SZ_4G - top_lm);
592	}
593
594	/* Squeeze out one address bit, shift upper part down to fill gap */
595	static void remove_addr_bit(u64 *addr, int bitidx)
596	{
597	u64 mask;
598
599	if (bitidx == -1)
600	return;
601
602	mask = BIT_ULL(bitidx) - 1;
603	*addr = ((*addr >> 1) & ~mask) | (*addr & mask);
604	}
605
606	/* XOR all the bits from addr specified in mask */
607	static int hash_by_mask(u64 addr, u64 mask)
608	{
609	u64 result = addr & mask;
610
611	result = (result >> 32) ^ result;
612	result = (result >> 16) ^ result;
613	result = (result >> 8) ^ result;
614	result = (result >> 4) ^ result;
615	result = (result >> 2) ^ result;
616	result = (result >> 1) ^ result;
617
618	return (int)result & 1;
619	}
620
621	/*
622	* First stage decode. Take the system address and figure out which
623	* second stage will deal with it based on interleave modes.
624	*/
625	static int sys2pmi(const u64 addr, u32 *pmiidx, u64 *pmiaddr, char *msg)
626	{
627	u64 contig_addr, contig_base, contig_offset, contig_base_adj;
628	int mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH :
629	MOT_CHAN_INTLV_BIT_1SLC_2CH;
630	int slice_intlv_bit_rm = SELECTOR_DISABLED;
631	int chan_intlv_bit_rm = SELECTOR_DISABLED;
632	/* Determine if address is in the MOT region. */
633	bool mot_hit = in_region(rp: &mot, addr);
634	/* Calculate the number of symmetric regions enabled. */
635	int sym_channels = hweight8(sym_chan_mask);
636
637	/*
638	* The amount we need to shift the asym base can be determined by the
639	* number of enabled symmetric channels.
640	* NOTE: This can only work because symmetric memory is not supposed
641	* to do a 3-way interleave.
642	*/
643	int sym_chan_shift = sym_channels >> 1;
644
645	/* Give up if address is out of range, or in MMIO gap */
646	if (addr >= BIT(PND_MAX_PHYS_BIT) ||
647	(addr >= top_lm && addr < SZ_4G) || addr >= top_hm) {
648	snprintf(buf: msg, PND2_MSG_SIZE, fmt: "Error address 0x%llx is not DRAM", addr);
649	return -EINVAL;
650	}
651
652	/* Get a contiguous memory address (remove the MMIO gap) */
653	contig_addr = remove_mmio_gap(sys: addr);
654
655	if (in_region(rp: &as0, addr)) {
656	*pmiidx = asym0.slice0_asym_channel_select;
657
658	contig_base = remove_mmio_gap(sys: as0.base);
659	contig_offset = contig_addr - contig_base;
660	contig_base_adj = (contig_base >> sym_chan_shift) *
661	((chash.sym_slice0_channel_enabled >> (*pmiidx & 1)) & 1);
662	contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull);
663	} else if (in_region(rp: &as1, addr)) {
664	*pmiidx = 2u + asym1.slice1_asym_channel_select;
665
666	contig_base = remove_mmio_gap(sys: as1.base);
667	contig_offset = contig_addr - contig_base;
668	contig_base_adj = (contig_base >> sym_chan_shift) *
669	((chash.sym_slice1_channel_enabled >> (*pmiidx & 1)) & 1);
670	contig_addr = contig_offset + ((sym_channels > 0) ? contig_base_adj : 0ull);
671	} else if (in_region(rp: &as2, addr) && (asym_2way.asym_2way_intlv_mode == 0x3ul)) {
672	bool channel1;
673
674	mot_intlv_bit = MOT_CHAN_INTLV_BIT_1SLC_2CH;
675	*pmiidx = (asym_2way.asym_2way_intlv_mode & 1) << 1;
676	channel1 = mot_hit ? ((bool)((addr >> mot_intlv_bit) & 1)) :
677	hash_by_mask(addr: contig_addr, mask: chan_hash_mask);
678	*pmiidx |= (u32)channel1;
679
680	contig_base = remove_mmio_gap(sys: as2.base);
681	chan_intlv_bit_rm = mot_hit ? mot_intlv_bit : chan_selector;
682	contig_offset = contig_addr - contig_base;
683	remove_addr_bit(addr: &contig_offset, bitidx: chan_intlv_bit_rm);
684	contig_addr = (contig_base >> sym_chan_shift) + contig_offset;
685	} else {
686	/* Otherwise we're in normal, boring symmetric mode. */
687	*pmiidx = 0u;
688
689	if (two_slices) {
690	bool slice1;
691
692	if (mot_hit) {
693	slice_intlv_bit_rm = MOT_SLC_INTLV_BIT;
694	slice1 = (addr >> MOT_SLC_INTLV_BIT) & 1;
695	} else {
696	slice_intlv_bit_rm = slice_selector;
697	slice1 = hash_by_mask(addr, mask: slice_hash_mask);
698	}
699
700	*pmiidx = (u32)slice1 << 1;
701	}
702
703	if (two_channels) {
704	bool channel1;
705
706	mot_intlv_bit = two_slices ? MOT_CHAN_INTLV_BIT_2SLC_2CH :
707	MOT_CHAN_INTLV_BIT_1SLC_2CH;
708
709	if (mot_hit) {
710	chan_intlv_bit_rm = mot_intlv_bit;
711	channel1 = (addr >> mot_intlv_bit) & 1;
712	} else {
713	chan_intlv_bit_rm = chan_selector;
714	channel1 = hash_by_mask(addr: contig_addr, mask: chan_hash_mask);
715	}
716
717	*pmiidx |= (u32)channel1;
718	}
719	}
720
721	/* Remove the chan_selector bit first */
722	remove_addr_bit(addr: &contig_addr, bitidx: chan_intlv_bit_rm);
723	/* Remove the slice bit (we remove it second because it must be lower */
724	remove_addr_bit(addr: &contig_addr, bitidx: slice_intlv_bit_rm);
725	*pmiaddr = contig_addr;
726
727	return 0;
728	}
729
730	/* Translate PMI address to memory (rank, row, bank, column) */
731	#define C(n) (BIT(4) | (n)) /* column */
732	#define B(n) (BIT(5) | (n)) /* bank */
733	#define R(n) (BIT(6) | (n)) /* row */
734	#define RS (BIT(7)) /* rank */
735
736	/* addrdec values */
737	#define AMAP_1KB 0
738	#define AMAP_2KB 1
739	#define AMAP_4KB 2
740	#define AMAP_RSVD 3
741
742	/* dden values */
743	#define DEN_4Gb 0
744	#define DEN_8Gb 2
745
746	/* dwid values */
747	#define X8 0
748	#define X16 1
749
750	static struct dimm_geometry {
751	u8 addrdec;
752	u8 dden;
753	u8 dwid;
754	u8 rowbits, colbits;
755	u16 bits[PMI_ADDRESS_WIDTH];
756	} dimms[] = {
757	{
758	.addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X16,
759	.rowbits = 15, .colbits = 10,
760	.bits = {
761	C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
762	R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
763	R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
764	0, 0, 0, 0
765	}
766	},
767	{
768	.addrdec = AMAP_1KB, .dden = DEN_4Gb, .dwid = X8,
769	.rowbits = 16, .colbits = 10,
770	.bits = {
771	C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
772	R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
773	R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
774	R(15), 0, 0, 0
775	}
776	},
777	{
778	.addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X16,
779	.rowbits = 16, .colbits = 10,
780	.bits = {
781	C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
782	R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
783	R(10), C(7), C(8), C(9), R(11), RS, R(12), R(13), R(14),
784	R(15), 0, 0, 0
785	}
786	},
787	{
788	.addrdec = AMAP_1KB, .dden = DEN_8Gb, .dwid = X8,
789	.rowbits = 16, .colbits = 11,
790	.bits = {
791	C(2), C(3), C(4), C(5), C(6), B(0), B(1), B(2), R(0),
792	R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8), R(9),
793	R(10), C(7), C(8), C(9), R(11), RS, C(11), R(12), R(13),
794	R(14), R(15), 0, 0
795	}
796	},
797	{
798	.addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X16,
799	.rowbits = 15, .colbits = 10,
800	.bits = {
801	C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
802	R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
803	R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
804	0, 0, 0, 0
805	}
806	},
807	{
808	.addrdec = AMAP_2KB, .dden = DEN_4Gb, .dwid = X8,
809	.rowbits = 16, .colbits = 10,
810	.bits = {
811	C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
812	R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
813	R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
814	R(15), 0, 0, 0
815	}
816	},
817	{
818	.addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X16,
819	.rowbits = 16, .colbits = 10,
820	.bits = {
821	C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
822	R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
823	R(9), R(10), C(8), C(9), R(11), RS, R(12), R(13), R(14),
824	R(15), 0, 0, 0
825	}
826	},
827	{
828	.addrdec = AMAP_2KB, .dden = DEN_8Gb, .dwid = X8,
829	.rowbits = 16, .colbits = 11,
830	.bits = {
831	C(2), C(3), C(4), C(5), C(6), C(7), B(0), B(1), B(2),
832	R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7), R(8),
833	R(9), R(10), C(8), C(9), R(11), RS, C(11), R(12), R(13),
834	R(14), R(15), 0, 0
835	}
836	},
837	{
838	.addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X16,
839	.rowbits = 15, .colbits = 10,
840	.bits = {
841	C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
842	B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
843	R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
844	0, 0, 0, 0
845	}
846	},
847	{
848	.addrdec = AMAP_4KB, .dden = DEN_4Gb, .dwid = X8,
849	.rowbits = 16, .colbits = 10,
850	.bits = {
851	C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
852	B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
853	R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
854	R(15), 0, 0, 0
855	}
856	},
857	{
858	.addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X16,
859	.rowbits = 16, .colbits = 10,
860	.bits = {
861	C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
862	B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
863	R(8), R(9), R(10), C(9), R(11), RS, R(12), R(13), R(14),
864	R(15), 0, 0, 0
865	}
866	},
867	{
868	.addrdec = AMAP_4KB, .dden = DEN_8Gb, .dwid = X8,
869	.rowbits = 16, .colbits = 11,
870	.bits = {
871	C(2), C(3), C(4), C(5), C(6), C(7), C(8), B(0), B(1),
872	B(2), R(0), R(1), R(2), R(3), R(4), R(5), R(6), R(7),
873	R(8), R(9), R(10), C(9), R(11), RS, C(11), R(12), R(13),
874	R(14), R(15), 0, 0
875	}
876	}
877	};
878
879	static int bank_hash(u64 pmiaddr, int idx, int shft)
880	{
881	int bhash = 0;
882
883	switch (idx) {
884	case 0:
885	bhash ^= ((pmiaddr >> (12 + shft)) ^ (pmiaddr >> (9 + shft))) & 1;
886	break;
887	case 1:
888	bhash ^= (((pmiaddr >> (10 + shft)) ^ (pmiaddr >> (8 + shft))) & 1) << 1;
889	bhash ^= ((pmiaddr >> 22) & 1) << 1;
890	break;
891	case 2:
892	bhash ^= (((pmiaddr >> (13 + shft)) ^ (pmiaddr >> (11 + shft))) & 1) << 2;
893	break;
894	}
895
896	return bhash;
897	}
898
899	static int rank_hash(u64 pmiaddr)
900	{
901	return ((pmiaddr >> 16) ^ (pmiaddr >> 10)) & 1;
902	}
903
904	/* Second stage decode. Compute rank, bank, row & column. */
905	static int apl_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
906	struct dram_addr *daddr, char *msg)
907	{
908	struct d_cr_drp0 *cr_drp0 = &drp0[pmiidx];
909	struct pnd2_pvt *pvt = mci->pvt_info;
910	int g = pvt->dimm_geom[pmiidx];
911	struct dimm_geometry *d = &dimms[g];
912	int column = 0, bank = 0, row = 0, rank = 0;
913	int i, idx, type, skiprs = 0;
914
915	for (i = 0; i < PMI_ADDRESS_WIDTH; i++) {
916	int bit = (pmiaddr >> i) & 1;
917
918	if (i + skiprs >= PMI_ADDRESS_WIDTH) {
919	snprintf(buf: msg, PND2_MSG_SIZE, fmt: "Bad dimm_geometry[] table\n");
920	return -EINVAL;
921	}
922
923	type = d->bits[i + skiprs] & ~0xf;
924	idx = d->bits[i + skiprs] & 0xf;
925
926	/*
927	* On single rank DIMMs ignore the rank select bit
928	* and shift remainder of "bits[]" down one place.
929	*/
930	if (type == RS && (cr_drp0->rken0 + cr_drp0->rken1) == 1) {
931	skiprs = 1;
932	type = d->bits[i + skiprs] & ~0xf;
933	idx = d->bits[i + skiprs] & 0xf;
934	}
935
936	switch (type) {
937	case C(0):
938	column |= (bit << idx);
939	break;
940	case B(0):
941	bank |= (bit << idx);
942	if (cr_drp0->bahen)
943	bank ^= bank_hash(pmiaddr, idx, shft: d->addrdec);
944	break;
945	case R(0):
946	row |= (bit << idx);
947	break;
948	case RS:
949	rank = bit;
950	if (cr_drp0->rsien)
951	rank ^= rank_hash(pmiaddr);
952	break;
953	default:
954	if (bit) {
955	snprintf(buf: msg, PND2_MSG_SIZE, fmt: "Bad translation\n");
956	return -EINVAL;
957	}
958	goto done;
959	}
960	}
961
962	done:
963	daddr->col = column;
964	daddr->bank = bank;
965	daddr->row = row;
966	daddr->rank = rank;
967	daddr->dimm = 0;
968
969	return 0;
970	}
971
972	/* Pluck bit "in" from pmiaddr and return value shifted to bit "out" */
973	#define dnv_get_bit(pmi, in, out) ((int)(((pmi) >> (in)) & 1u) << (out))
974
975	static int dnv_pmi2mem(struct mem_ctl_info *mci, u64 pmiaddr, u32 pmiidx,
976	struct dram_addr *daddr, char *msg)
977	{
978	/* Rank 0 or 1 */
979	daddr->rank = dnv_get_bit(pmiaddr, dmap[pmiidx].rs0 + 13, 0);
980	/* Rank 2 or 3 */
981	daddr->rank |= dnv_get_bit(pmiaddr, dmap[pmiidx].rs1 + 13, 1);
982
983	/*
984	* Normally ranks 0,1 are DIMM0, and 2,3 are DIMM1, but we
985	* flip them if DIMM1 is larger than DIMM0.
986	*/
987	daddr->dimm = (daddr->rank >= 2) ^ drp[pmiidx].dimmflip;
988
989	daddr->bank = dnv_get_bit(pmiaddr, dmap[pmiidx].ba0 + 6, 0);
990	daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].ba1 + 6, 1);
991	daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg0 + 6, 2);
992	if (dsch.ddr4en)
993	daddr->bank |= dnv_get_bit(pmiaddr, dmap[pmiidx].bg1 + 6, 3);
994	if (dmap1[pmiidx].bxor) {
995	if (dsch.ddr4en) {
996	daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 0);
997	daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 1);
998	if (dsch.chan_width == 0)
999	/* 64/72 bit dram channel width */
1000	daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2);
1001	else
1002	/* 32/40 bit dram channel width */
1003	daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2);
1004	daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 3);
1005	} else {
1006	daddr->bank ^= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 0);
1007	daddr->bank ^= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 1);
1008	if (dsch.chan_width == 0)
1009	daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 2);
1010	else
1011	daddr->bank ^= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 2);
1012	}
1013	}
1014
1015	daddr->row = dnv_get_bit(pmiaddr, dmap2[pmiidx].row0 + 6, 0);
1016	daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row1 + 6, 1);
1017	daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row2 + 6, 2);
1018	daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row3 + 6, 3);
1019	daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row4 + 6, 4);
1020	daddr->row |= dnv_get_bit(pmiaddr, dmap2[pmiidx].row5 + 6, 5);
1021	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row6 + 6, 6);
1022	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row7 + 6, 7);
1023	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row8 + 6, 8);
1024	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row9 + 6, 9);
1025	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row10 + 6, 10);
1026	daddr->row |= dnv_get_bit(pmiaddr, dmap3[pmiidx].row11 + 6, 11);
1027	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row12 + 6, 12);
1028	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row13 + 6, 13);
1029	if (dmap4[pmiidx].row14 != 31)
1030	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row14 + 6, 14);
1031	if (dmap4[pmiidx].row15 != 31)
1032	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row15 + 6, 15);
1033	if (dmap4[pmiidx].row16 != 31)
1034	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row16 + 6, 16);
1035	if (dmap4[pmiidx].row17 != 31)
1036	daddr->row |= dnv_get_bit(pmiaddr, dmap4[pmiidx].row17 + 6, 17);
1037
1038	daddr->col = dnv_get_bit(pmiaddr, dmap5[pmiidx].ca3 + 6, 3);
1039	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca4 + 6, 4);
1040	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca5 + 6, 5);
1041	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca6 + 6, 6);
1042	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca7 + 6, 7);
1043	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca8 + 6, 8);
1044	daddr->col |= dnv_get_bit(pmiaddr, dmap5[pmiidx].ca9 + 6, 9);
1045	if (!dsch.ddr4en && dmap1[pmiidx].ca11 != 0x3f)
1046	daddr->col |= dnv_get_bit(pmiaddr, dmap1[pmiidx].ca11 + 13, 11);
1047
1048	return 0;
1049	}
1050
1051	static int check_channel(int ch)
1052	{
1053	if (drp0[ch].dramtype != 0) {
1054	pnd2_printk(KERN_INFO, "Unsupported DIMM in channel %d\n", ch);
1055	return 1;
1056	} else if (drp0[ch].eccen == 0) {
1057	pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch);
1058	return 1;
1059	}
1060	return 0;
1061	}
1062
1063	static int apl_check_ecc_active(void)
1064	{
1065	int i, ret = 0;
1066
1067	/* Check dramtype and ECC mode for each present DIMM */
1068	for_each_set_bit(i, &chan_mask, APL_NUM_CHANNELS)
1069	ret += check_channel(ch: i);
1070
1071	return ret ? -EINVAL : 0;
1072	}
1073
1074	#define DIMMS_PRESENT(d) ((d)->rken0 + (d)->rken1 + (d)->rken2 + (d)->rken3)
1075
1076	static int check_unit(int ch)
1077	{
1078	struct d_cr_drp *d = &drp[ch];
1079
1080	if (DIMMS_PRESENT(d) && !ecc_ctrl[ch].eccen) {
1081	pnd2_printk(KERN_INFO, "ECC disabled on channel %d\n", ch);
1082	return 1;
1083	}
1084	return 0;
1085	}
1086
1087	static int dnv_check_ecc_active(void)
1088	{
1089	int i, ret = 0;
1090
1091	for (i = 0; i < DNV_NUM_CHANNELS; i++)
1092	ret += check_unit(ch: i);
1093	return ret ? -EINVAL : 0;
1094	}
1095
1096	static int get_memory_error_data(struct mem_ctl_info *mci, u64 addr,
1097	struct dram_addr *daddr, char *msg)
1098	{
1099	u64 pmiaddr;
1100	u32 pmiidx;
1101	int ret;
1102
1103	ret = sys2pmi(addr, pmiidx: &pmiidx, pmiaddr: &pmiaddr, msg);
1104	if (ret)
1105	return ret;
1106
1107	pmiaddr >>= ops->pmiaddr_shift;
1108	/* pmi channel idx to dimm channel idx */
1109	pmiidx >>= ops->pmiidx_shift;
1110	daddr->chan = pmiidx;
1111
1112	ret = ops->pmi2mem(mci, pmiaddr, pmiidx, daddr, msg);
1113	if (ret)
1114	return ret;
1115
1116	edac_dbg(0, "SysAddr=%llx PmiAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n",
1117	addr, pmiaddr, daddr->chan, daddr->dimm, daddr->rank, daddr->bank, daddr->row, daddr->col);
1118
1119	return 0;
1120	}
1121
1122	static void pnd2_mce_output_error(struct mem_ctl_info *mci, const struct mce *m,
1123	struct dram_addr *daddr)
1124	{
1125	enum hw_event_mc_err_type tp_event;
1126	char *optype, msg[PND2_MSG_SIZE];
1127	bool ripv = m->mcgstatus & MCG_STATUS_RIPV;
1128	bool overflow = m->status & MCI_STATUS_OVER;
1129	bool uc_err = m->status & MCI_STATUS_UC;
1130	bool recov = m->status & MCI_STATUS_S;
1131	u32 core_err_cnt = GET_BITFIELD(m->status, 38, 52);
1132	u32 mscod = GET_BITFIELD(m->status, 16, 31);
1133	u32 errcode = GET_BITFIELD(m->status, 0, 15);
1134	u32 optypenum = GET_BITFIELD(m->status, 4, 6);
1135	int rc;
1136
1137	tp_event = uc_err ? (ripv ? HW_EVENT_ERR_UNCORRECTED : HW_EVENT_ERR_FATAL) :
1138	HW_EVENT_ERR_CORRECTED;
1139
1140	/*
1141	* According with Table 15-9 of the Intel Architecture spec vol 3A,
1142	* memory errors should fit in this mask:
1143	* 000f 0000 1mmm cccc (binary)
1144	* where:
1145	* f = Correction Report Filtering Bit. If 1, subsequent errors
1146	* won't be shown
1147	* mmm = error type
1148	* cccc = channel
1149	* If the mask doesn't match, report an error to the parsing logic
1150	*/
1151	if (!((errcode & 0xef80) == 0x80)) {
1152	optype = "Can't parse: it is not a mem";
1153	} else {
1154	switch (optypenum) {
1155	case 0:
1156	optype = "generic undef request error";
1157	break;
1158	case 1:
1159	optype = "memory read error";
1160	break;
1161	case 2:
1162	optype = "memory write error";
1163	break;
1164	case 3:
1165	optype = "addr/cmd error";
1166	break;
1167	case 4:
1168	optype = "memory scrubbing error";
1169	break;
1170	default:
1171	optype = "reserved";
1172	break;
1173	}
1174	}
1175
1176	/* Only decode errors with an valid address (ADDRV) */
1177	if (!(m->status & MCI_STATUS_ADDRV))
1178	return;
1179
1180	rc = get_memory_error_data(mci, addr: m->addr, daddr, msg);
1181	if (rc)
1182	goto address_error;
1183
1184	snprintf(buf: msg, size: sizeof(msg),
1185	fmt: "%s%s err_code:%04x:%04x channel:%d DIMM:%d rank:%d row:%d bank:%d col:%d",
1186	overflow ? " OVERFLOW" : "", (uc_err && recov) ? " recoverable" : "", mscod,
1187	errcode, daddr->chan, daddr->dimm, daddr->rank, daddr->row, daddr->bank, daddr->col);
1188
1189	edac_dbg(0, "%s\n", msg);
1190
1191	/* Call the helper to output message */
1192	edac_mc_handle_error(type: tp_event, mci, error_count: core_err_cnt, page_frame_number: m->addr >> PAGE_SHIFT,
1193	offset_in_page: m->addr & ~PAGE_MASK, syndrome: 0, top_layer: daddr->chan, mid_layer: daddr->dimm, low_layer: -1, msg: optype, other_detail: msg);
1194
1195	return;
1196
1197	address_error:
1198	edac_mc_handle_error(type: tp_event, mci, error_count: core_err_cnt, page_frame_number: 0, offset_in_page: 0, syndrome: 0, top_layer: -1, mid_layer: -1, low_layer: -1, msg, other_detail: "");
1199	}
1200
1201	static void apl_get_dimm_config(struct mem_ctl_info *mci)
1202	{
1203	struct pnd2_pvt *pvt = mci->pvt_info;
1204	struct dimm_info *dimm;
1205	struct d_cr_drp0 *d;
1206	u64 capacity;
1207	int i, g;
1208
1209	for_each_set_bit(i, &chan_mask, APL_NUM_CHANNELS) {
1210	dimm = edac_get_dimm(mci, layer0: i, layer1: 0, layer2: 0);
1211	if (!dimm) {
1212	edac_dbg(0, "No allocated DIMM for channel %d\n", i);
1213	continue;
1214	}
1215
1216	d = &drp0[i];
1217	for (g = 0; g < ARRAY_SIZE(dimms); g++)
1218	if (dimms[g].addrdec == d->addrdec &&
1219	dimms[g].dden == d->dden &&
1220	dimms[g].dwid == d->dwid)
1221	break;
1222
1223	if (g == ARRAY_SIZE(dimms)) {
1224	edac_dbg(0, "Channel %d: unrecognized DIMM\n", i);
1225	continue;
1226	}
1227
1228	pvt->dimm_geom[i] = g;
1229	capacity = (d->rken0 + d->rken1) * 8 * BIT(dimms[g].rowbits + dimms[g].colbits);
1230	edac_dbg(0, "Channel %d: %lld MByte DIMM\n", i, capacity >> (20 - 3));
1231	dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3));
1232	dimm->grain = 32;
1233	dimm->dtype = (d->dwid == 0) ? DEV_X8 : DEV_X16;
1234	dimm->mtype = MEM_DDR3;
1235	dimm->edac_mode = EDAC_SECDED;
1236	snprintf(buf: dimm->label, size: sizeof(dimm->label), fmt: "Slice#%d_Chan#%d", i / 2, i % 2);
1237	}
1238	}
1239
1240	static const int dnv_dtypes[] = {
1241	DEV_X8, DEV_X4, DEV_X16, DEV_UNKNOWN
1242	};
1243
1244	static void dnv_get_dimm_config(struct mem_ctl_info *mci)
1245	{
1246	int i, j, ranks_of_dimm[DNV_MAX_DIMMS], banks, rowbits, colbits, memtype;
1247	struct dimm_info *dimm;
1248	struct d_cr_drp *d;
1249	u64 capacity;
1250
1251	if (dsch.ddr4en) {
1252	memtype = MEM_DDR4;
1253	banks = 16;
1254	colbits = 10;
1255	} else {
1256	memtype = MEM_DDR3;
1257	banks = 8;
1258	}
1259
1260	for (i = 0; i < DNV_NUM_CHANNELS; i++) {
1261	if (dmap4[i].row14 == 31)
1262	rowbits = 14;
1263	else if (dmap4[i].row15 == 31)
1264	rowbits = 15;
1265	else if (dmap4[i].row16 == 31)
1266	rowbits = 16;
1267	else if (dmap4[i].row17 == 31)
1268	rowbits = 17;
1269	else
1270	rowbits = 18;
1271
1272	if (memtype == MEM_DDR3) {
1273	if (dmap1[i].ca11 != 0x3f)
1274	colbits = 12;
1275	else
1276	colbits = 10;
1277	}
1278
1279	d = &drp[i];
1280	/* DIMM0 is present if rank0 and/or rank1 is enabled */
1281	ranks_of_dimm[0] = d->rken0 + d->rken1;
1282	/* DIMM1 is present if rank2 and/or rank3 is enabled */
1283	ranks_of_dimm[1] = d->rken2 + d->rken3;
1284
1285	for (j = 0; j < DNV_MAX_DIMMS; j++) {
1286	if (!ranks_of_dimm[j])
1287	continue;
1288
1289	dimm = edac_get_dimm(mci, layer0: i, layer1: j, layer2: 0);
1290	if (!dimm) {
1291	edac_dbg(0, "No allocated DIMM for channel %d DIMM %d\n", i, j);
1292	continue;
1293	}
1294
1295	capacity = ranks_of_dimm[j] * banks * BIT(rowbits + colbits);
1296	edac_dbg(0, "Channel %d DIMM %d: %lld MByte DIMM\n", i, j, capacity >> (20 - 3));
1297	dimm->nr_pages = MiB_TO_PAGES(capacity >> (20 - 3));
1298	dimm->grain = 32;
1299	dimm->dtype = dnv_dtypes[j ? d->dimmdwid0 : d->dimmdwid1];
1300	dimm->mtype = memtype;
1301	dimm->edac_mode = EDAC_SECDED;
1302	snprintf(buf: dimm->label, size: sizeof(dimm->label), fmt: "Chan#%d_DIMM#%d", i, j);
1303	}
1304	}
1305	}
1306
1307	static int pnd2_register_mci(struct mem_ctl_info **ppmci)
1308	{
1309	struct edac_mc_layer layers[2];
1310	struct mem_ctl_info *mci;
1311	struct pnd2_pvt *pvt;
1312	int rc;
1313
1314	rc = ops->check_ecc();
1315	if (rc < 0)
1316	return rc;
1317
1318	/* Allocate a new MC control structure */
1319	layers[0].type = EDAC_MC_LAYER_CHANNEL;
1320	layers[0].size = ops->channels;
1321	layers[0].is_virt_csrow = false;
1322	layers[1].type = EDAC_MC_LAYER_SLOT;
1323	layers[1].size = ops->dimms_per_channel;
1324	layers[1].is_virt_csrow = true;
1325	mci = edac_mc_alloc(mc_num: 0, ARRAY_SIZE(layers), layers, sz_pvt: sizeof(*pvt));
1326	if (!mci)
1327	return -ENOMEM;
1328
1329	pvt = mci->pvt_info;
1330	memset(pvt, 0, sizeof(*pvt));
1331
1332	mci->mod_name = EDAC_MOD_STR;
1333	mci->dev_name = ops->name;
1334	mci->ctl_name = "Pondicherry2";
1335
1336	/* Get dimm basic config and the memory layout */
1337	ops->get_dimm_config(mci);
1338
1339	if (edac_mc_add_mc(mci)) {
1340	edac_dbg(0, "MC: failed edac_mc_add_mc()\n");
1341	edac_mc_free(mci);
1342	return -EINVAL;
1343	}
1344
1345	*ppmci = mci;
1346
1347	return 0;
1348	}
1349
1350	static void pnd2_unregister_mci(struct mem_ctl_info *mci)
1351	{
1352	if (unlikely(!mci || !mci->pvt_info)) {
1353	pnd2_printk(KERN_ERR, "Couldn't find mci handler\n");
1354	return;
1355	}
1356
1357	/* Remove MC sysfs nodes */
1358	edac_mc_del_mc(NULL);
1359	edac_dbg(1, "%s: free mci struct\n", mci->ctl_name);
1360	edac_mc_free(mci);
1361	}
1362
1363	/*
1364	* Callback function registered with core kernel mce code.
1365	* Called once for each logged error.
1366	*/
1367	static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, void *data)
1368	{
1369	struct mce *mce = (struct mce *)data;
1370	struct mem_ctl_info *mci;
1371	struct dram_addr daddr;
1372	char *type;
1373
1374	mci = pnd2_mci;
1375	if (!mci || (mce->kflags & MCE_HANDLED_CEC))
1376	return NOTIFY_DONE;
1377
1378	/*
1379	* Just let mcelog handle it if the error is
1380	* outside the memory controller. A memory error
1381	* is indicated by bit 7 = 1 and bits = 8-11,13-15 = 0.
1382	* bit 12 has an special meaning.
1383	*/
1384	if ((mce->status & 0xefff) >> 7 != 1)
1385	return NOTIFY_DONE;
1386
1387	if (mce->mcgstatus & MCG_STATUS_MCIP)
1388	type = "Exception";
1389	else
1390	type = "Event";
1391
1392	pnd2_mc_printk(mci, KERN_INFO, "HANDLING MCE MEMORY ERROR\n");
1393	pnd2_mc_printk(mci, KERN_INFO, "CPU %u: Machine Check %s: %llx Bank %u: %llx\n",
1394	mce->extcpu, type, mce->mcgstatus, mce->bank, mce->status);
1395	pnd2_mc_printk(mci, KERN_INFO, "TSC %llx ", mce->tsc);
1396	pnd2_mc_printk(mci, KERN_INFO, "ADDR %llx ", mce->addr);
1397	pnd2_mc_printk(mci, KERN_INFO, "MISC %llx ", mce->misc);
1398	pnd2_mc_printk(mci, KERN_INFO, "PROCESSOR %u:%x TIME %llu SOCKET %u APIC %x\n",
1399	mce->cpuvendor, mce->cpuid, mce->time, mce->socketid, mce->apicid);
1400
1401	pnd2_mce_output_error(mci, m: mce, daddr: &daddr);
1402
1403	/* Advice mcelog that the error were handled */
1404	mce->kflags |= MCE_HANDLED_EDAC;
1405	return NOTIFY_OK;
1406	}
1407
1408	static struct notifier_block pnd2_mce_dec = {
1409	.notifier_call = pnd2_mce_check_error,
1410	.priority = MCE_PRIO_EDAC,
1411	};
1412
1413	#ifdef CONFIG_EDAC_DEBUG
1414	/*
1415	* Write an address to this file to exercise the address decode
1416	* logic in this driver.
1417	*/
1418	static u64 pnd2_fake_addr;
1419	#define PND2_BLOB_SIZE 1024
1420	static char pnd2_result[PND2_BLOB_SIZE];
1421	static struct dentry *pnd2_test;
1422	static struct debugfs_blob_wrapper pnd2_blob = {
1423	.data = pnd2_result,
1424	.size = 0
1425	};
1426
1427	static int debugfs_u64_set(void *data, u64 val)
1428	{
1429	struct dram_addr daddr;
1430	struct mce m;
1431
1432	*(u64 *)data = val;
1433	m.mcgstatus = 0;
1434	/* ADDRV + MemRd + Unknown channel */
1435	m.status = MCI_STATUS_ADDRV + 0x9f;
1436	m.addr = val;
1437	pnd2_mce_output_error(mci: pnd2_mci, m: &m, daddr: &daddr);
1438	snprintf(buf: pnd2_blob.data, PND2_BLOB_SIZE,
1439	fmt: "SysAddr=%llx Channel=%d DIMM=%d Rank=%d Bank=%d Row=%d Column=%d\n",
1440	m.addr, daddr.chan, daddr.dimm, daddr.rank, daddr.bank, daddr.row, daddr.col);
1441	pnd2_blob.size = strlen(pnd2_blob.data);
1442
1443	return 0;
1444	}
1445	DEFINE_DEBUGFS_ATTRIBUTE(fops_u64_wo, NULL, debugfs_u64_set, "%llu\n");
1446
1447	static void setup_pnd2_debug(void)
1448	{
1449	pnd2_test = edac_debugfs_create_dir(dirname: "pnd2_test");
1450	edac_debugfs_create_file(name: "pnd2_debug_addr", mode: 0200, parent: pnd2_test,
1451	data: &pnd2_fake_addr, fops: &fops_u64_wo);
1452	debugfs_create_blob(name: "pnd2_debug_results", mode: 0400, parent: pnd2_test, blob: &pnd2_blob);
1453	}
1454
1455	static void teardown_pnd2_debug(void)
1456	{
1457	debugfs_remove_recursive(dentry: pnd2_test);
1458	}
1459	#else
1460	static void setup_pnd2_debug(void) {}
1461	static void teardown_pnd2_debug(void) {}
1462	#endif /* CONFIG_EDAC_DEBUG */
1463
1464
1465	static int pnd2_probe(void)
1466	{
1467	int rc;
1468
1469	edac_dbg(2, "\n");
1470	rc = get_registers();
1471	if (rc)
1472	return rc;
1473
1474	return pnd2_register_mci(ppmci: &pnd2_mci);
1475	}
1476
1477	static void pnd2_remove(void)
1478	{
1479	edac_dbg(0, "\n");
1480	pnd2_unregister_mci(mci: pnd2_mci);
1481	}
1482
1483	static struct dunit_ops apl_ops = {
1484	.name = "pnd2/apl",
1485	.type = APL,
1486	.pmiaddr_shift = LOG2_PMI_ADDR_GRANULARITY,
1487	.pmiidx_shift = 0,
1488	.channels = APL_NUM_CHANNELS,
1489	.dimms_per_channel = 1,
1490	.rd_reg = apl_rd_reg,
1491	.get_registers = apl_get_registers,
1492	.check_ecc = apl_check_ecc_active,
1493	.mk_region = apl_mk_region,
1494	.get_dimm_config = apl_get_dimm_config,
1495	.pmi2mem = apl_pmi2mem,
1496	};
1497
1498	static struct dunit_ops dnv_ops = {
1499	.name = "pnd2/dnv",
1500	.type = DNV,
1501	.pmiaddr_shift = 0,
1502	.pmiidx_shift = 1,
1503	.channels = DNV_NUM_CHANNELS,
1504	.dimms_per_channel = 2,
1505	.rd_reg = dnv_rd_reg,
1506	.get_registers = dnv_get_registers,
1507	.check_ecc = dnv_check_ecc_active,
1508	.mk_region = dnv_mk_region,
1509	.get_dimm_config = dnv_get_dimm_config,
1510	.pmi2mem = dnv_pmi2mem,
1511	};
1512
1513	static const struct x86_cpu_id pnd2_cpuids[] = {
1514	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT, &apl_ops),
1515	X86_MATCH_INTEL_FAM6_MODEL(ATOM_GOLDMONT_D, &dnv_ops),
1516	{ }
1517	};
1518	MODULE_DEVICE_TABLE(x86cpu, pnd2_cpuids);
1519
1520	static int __init pnd2_init(void)
1521	{
1522	const struct x86_cpu_id *id;
1523	const char *owner;
1524	int rc;
1525
1526	edac_dbg(2, "\n");
1527
1528	if (ghes_get_devices())
1529	return -EBUSY;
1530
1531	owner = edac_get_owner();
1532	if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR)))
1533	return -EBUSY;
1534
1535	if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR))
1536	return -ENODEV;
1537
1538	id = x86_match_cpu(match: pnd2_cpuids);
1539	if (!id)
1540	return -ENODEV;
1541
1542	ops = (struct dunit_ops *)id->driver_data;
1543
1544	if (ops->type == APL) {
1545	p2sb_bus = pci_find_bus(domain: 0, busnr: 0);
1546	if (!p2sb_bus)
1547	return -ENODEV;
1548	}
1549
1550	/* Ensure that the OPSTATE is set correctly for POLL or NMI */
1551	opstate_init();
1552
1553	rc = pnd2_probe();
1554	if (rc < 0) {
1555	pnd2_printk(KERN_ERR, "Failed to register device with error %d.\n", rc);
1556	return rc;
1557	}
1558
1559	if (!pnd2_mci)
1560	return -ENODEV;
1561
1562	mce_register_decode_chain(nb: &pnd2_mce_dec);
1563	setup_pnd2_debug();
1564
1565	return 0;
1566	}
1567
1568	static void __exit pnd2_exit(void)
1569	{
1570	edac_dbg(2, "\n");
1571	teardown_pnd2_debug();
1572	mce_unregister_decode_chain(nb: &pnd2_mce_dec);
1573	pnd2_remove();
1574	}
1575
1576	module_init(pnd2_init);
1577	module_exit(pnd2_exit);
1578
1579	module_param(edac_op_state, int, 0444);
1580	MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");
1581
1582	MODULE_LICENSE("GPL v2");
1583	MODULE_AUTHOR("Tony Luck");
1584	MODULE_DESCRIPTION("MC Driver for Intel SoC using Pondicherry memory controller");
1585