// SPDX-License-Identifier: Intel /* * Copyright (C) 2013, Intel Corporation * Copyright (C) 2015, Bin Meng * * Ported from Intel released Quark UEFI BIOS * QuarkSocPkg/QuarkNorthCluster/MemoryInit/Pei */ #include #include #include #include #include #include "mrc_util.h" #include "hte.h" #include "smc.h" /* t_ck clock period in picoseconds per speed index 800, 1066, 1333 */ static const uint32_t t_ck[3] = { 2500, 1875, 1500 }; /* Global variables */ static const uint16_t ddr_wclk[] = {193, 158}; #ifdef BACKUP_WCTL static const uint16_t ddr_wctl[] = {1, 217}; #endif #ifdef BACKUP_WCMD static const uint16_t ddr_wcmd[] = {1, 220}; #endif #ifdef BACKUP_RCVN static const uint16_t ddr_rcvn[] = {129, 498}; #endif #ifdef BACKUP_WDQS static const uint16_t ddr_wdqs[] = {65, 289}; #endif #ifdef BACKUP_RDQS static const uint8_t ddr_rdqs[] = {32, 24}; #endif #ifdef BACKUP_WDQ static const uint16_t ddr_wdq[] = {32, 257}; #endif /* Stop self refresh driven by MCU */ void clear_self_refresh(struct mrc_params *mrc_params) { ENTERFN(); /* clear the PMSTS Channel Self Refresh bits */ mrc_write_mask(MEM_CTLR, PMSTS, PMSTS_DISR, PMSTS_DISR); LEAVEFN(); } /* It will initialize timing registers in the MCU (DTR0..DTR4) */ void prog_ddr_timing_control(struct mrc_params *mrc_params) { uint8_t tcl, wl; uint8_t trp, trcd, tras, twr, twtr, trrd, trtp, tfaw; uint32_t tck; u32 dtr0, dtr1, dtr2, dtr3, dtr4; u32 tmp1, tmp2; ENTERFN(); /* mcu_init starts */ mrc_post_code(0x02, 0x00); dtr0 = msg_port_read(MEM_CTLR, DTR0); dtr1 = msg_port_read(MEM_CTLR, DTR1); dtr2 = msg_port_read(MEM_CTLR, DTR2); dtr3 = msg_port_read(MEM_CTLR, DTR3); dtr4 = msg_port_read(MEM_CTLR, DTR4); tck = t_ck[mrc_params->ddr_speed]; /* Clock in picoseconds */ tcl = mrc_params->params.cl; /* CAS latency in clocks */ trp = tcl; /* Per CAT MRC */ trcd = tcl; /* Per CAT MRC */ tras = MCEIL(mrc_params->params.ras, tck); /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */ twr = MCEIL(15000, tck); twtr = MCEIL(mrc_params->params.wtr, tck); trrd = MCEIL(mrc_params->params.rrd, tck); trtp = 4; /* Valid for 800 and 1066, use 5 for 1333 */ tfaw = MCEIL(mrc_params->params.faw, tck); wl = 5 + mrc_params->ddr_speed; dtr0 &= ~DTR0_DFREQ_MASK; dtr0 |= mrc_params->ddr_speed; dtr0 &= ~DTR0_TCL_MASK; tmp1 = tcl - 5; dtr0 |= ((tcl - 5) << 12); dtr0 &= ~DTR0_TRP_MASK; dtr0 |= ((trp - 5) << 4); /* 5 bit DRAM Clock */ dtr0 &= ~DTR0_TRCD_MASK; dtr0 |= ((trcd - 5) << 8); /* 5 bit DRAM Clock */ dtr1 &= ~DTR1_TWCL_MASK; tmp2 = wl - 3; dtr1 |= (wl - 3); dtr1 &= ~DTR1_TWTP_MASK; dtr1 |= ((wl + 4 + twr - 14) << 8); /* Change to tWTP */ dtr1 &= ~DTR1_TRTP_MASK; dtr1 |= ((MMAX(trtp, 4) - 3) << 28); /* 4 bit DRAM Clock */ dtr1 &= ~DTR1_TRRD_MASK; dtr1 |= ((trrd - 4) << 24); /* 4 bit DRAM Clock */ dtr1 &= ~DTR1_TCMD_MASK; dtr1 |= (1 << 4); dtr1 &= ~DTR1_TRAS_MASK; dtr1 |= ((tras - 14) << 20); /* 6 bit DRAM Clock */ dtr1 &= ~DTR1_TFAW_MASK; dtr1 |= ((((tfaw + 1) >> 1) - 5) << 16);/* 4 bit DRAM Clock */ /* Set 4 Clock CAS to CAS delay (multi-burst) */ dtr1 &= ~DTR1_TCCD_MASK; dtr2 &= ~DTR2_TRRDR_MASK; dtr2 |= 1; dtr2 &= ~DTR2_TWWDR_MASK; dtr2 |= (2 << 8); dtr2 &= ~DTR2_TRWDR_MASK; dtr2 |= (2 << 16); dtr3 &= ~DTR3_TWRDR_MASK; dtr3 |= 2; dtr3 &= ~DTR3_TXXXX_MASK; dtr3 |= (2 << 4); dtr3 &= ~DTR3_TRWSR_MASK; if (mrc_params->ddr_speed == DDRFREQ_800) { /* Extended RW delay (+1) */ dtr3 |= ((tcl - 5 + 1) << 8); } else if (mrc_params->ddr_speed == DDRFREQ_1066) { /* Extended RW delay (+1) */ dtr3 |= ((tcl - 5 + 1) << 8); } dtr3 &= ~DTR3_TWRSR_MASK; dtr3 |= ((4 + wl + twtr - 11) << 13); dtr3 &= ~DTR3_TXP_MASK; if (mrc_params->ddr_speed == DDRFREQ_800) dtr3 |= ((MMAX(0, 1 - 1)) << 22); else dtr3 |= ((MMAX(0, 2 - 1)) << 22); dtr4 &= ~DTR4_WRODTSTRT_MASK; dtr4 |= 1; dtr4 &= ~DTR4_WRODTSTOP_MASK; dtr4 |= (1 << 4); dtr4 &= ~DTR4_XXXX1_MASK; dtr4 |= ((1 + tmp1 - tmp2 + 2) << 8); dtr4 &= ~DTR4_XXXX2_MASK; dtr4 |= ((1 + tmp1 - tmp2 + 2) << 12); dtr4 &= ~(DTR4_ODTDIS | DTR4_TRGSTRDIS); msg_port_write(MEM_CTLR, DTR0, dtr0); msg_port_write(MEM_CTLR, DTR1, dtr1); msg_port_write(MEM_CTLR, DTR2, dtr2); msg_port_write(MEM_CTLR, DTR3, dtr3); msg_port_write(MEM_CTLR, DTR4, dtr4); LEAVEFN(); } /* Configure MCU before jedec init sequence */ void prog_decode_before_jedec(struct mrc_params *mrc_params) { u32 drp; u32 drfc; u32 dcal; u32 dsch; u32 dpmc0; ENTERFN(); /* Disable power saving features */ dpmc0 = msg_port_read(MEM_CTLR, DPMC0); dpmc0 |= (DPMC0_CLKGTDIS | DPMC0_DISPWRDN); dpmc0 &= ~DPMC0_PCLSTO_MASK; dpmc0 &= ~DPMC0_DYNSREN; msg_port_write(MEM_CTLR, DPMC0, dpmc0); /* Disable out of order transactions */ dsch = msg_port_read(MEM_CTLR, DSCH); dsch |= (DSCH_OOODIS | DSCH_NEWBYPDIS); msg_port_write(MEM_CTLR, DSCH, dsch); /* Disable issuing the REF command */ drfc = msg_port_read(MEM_CTLR, DRFC); drfc &= ~DRFC_TREFI_MASK; msg_port_write(MEM_CTLR, DRFC, drfc); /* Disable ZQ calibration short */ dcal = msg_port_read(MEM_CTLR, DCAL); dcal &= ~DCAL_ZQCINT_MASK; dcal &= ~DCAL_SRXZQCL_MASK; msg_port_write(MEM_CTLR, DCAL, dcal); /* * Training performed in address mode 0, rank population has limited * impact, however simulator complains if enabled non-existing rank. */ drp = 0; if (mrc_params->rank_enables & 1) drp |= DRP_RKEN0; if (mrc_params->rank_enables & 2) drp |= DRP_RKEN1; msg_port_write(MEM_CTLR, DRP, drp); LEAVEFN(); } /* * After Cold Reset, BIOS should set COLDWAKE bit to 1 before * sending the WAKE message to the Dunit. * * For Standby Exit, or any other mode in which the DRAM is in * SR, this bit must be set to 0. */ void perform_ddr_reset(struct mrc_params *mrc_params) { ENTERFN(); /* Set COLDWAKE bit before sending the WAKE message */ mrc_write_mask(MEM_CTLR, DRMC, DRMC_COLDWAKE, DRMC_COLDWAKE); /* Send wake command to DUNIT (MUST be done before JEDEC) */ dram_wake_command(); /* Set default value */ msg_port_write(MEM_CTLR, DRMC, mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0); LEAVEFN(); } /* * This function performs some initialization on the DDRIO unit. * This function is dependent on BOARD_ID, DDR_SPEED, and CHANNEL_ENABLES. */ void ddrphy_init(struct mrc_params *mrc_params) { uint32_t temp; uint8_t ch; /* channel counter */ uint8_t rk; /* rank counter */ uint8_t bl_grp; /* byte lane group counter (2 BLs per module) */ uint8_t bl_divisor = 1; /* byte lane divisor */ /* For DDR3 --> 0 == 800, 1 == 1066, 2 == 1333 */ uint8_t speed = mrc_params->ddr_speed & 3; uint8_t cas; uint8_t cwl; ENTERFN(); cas = mrc_params->params.cl; cwl = 5 + mrc_params->ddr_speed; /* ddrphy_init starts */ mrc_post_code(0x03, 0x00); /* * HSD#231531 * Make sure IOBUFACT is deasserted before initializing the DDR PHY * * HSD#234845 * Make sure WRPTRENABLE is deasserted before initializing the DDR PHY */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* Deassert DDRPHY Initialization Complete */ mrc_alt_write_mask(DDRPHY, CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, ~(1 << 20), 1 << 20); /* SPID_INIT_COMPLETE=0 */ /* Deassert IOBUFACT */ mrc_alt_write_mask(DDRPHY, CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, ~(1 << 2), 1 << 2); /* IOBUFACTRST_N=0 */ /* Disable WRPTR */ mrc_alt_write_mask(DDRPHY, CMDPTRREG + ch * DDRIOCCC_CH_OFFSET, ~(1 << 0), 1 << 0); /* WRPTRENABLE=0 */ } } /* Put PHY in reset */ mrc_alt_write_mask(DDRPHY, MASTERRSTN, 0, 1); /* Initialize DQ01, DQ23, CMD, CLK-CTL, COMP modules */ /* STEP0 */ mrc_post_code(0x03, 0x10); for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* DQ01-DQ23 */ for (bl_grp = 0; bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; bl_grp++) { /* Analog MUX select - IO2xCLKSEL */ mrc_alt_write_mask(DDRPHY, DQOBSCKEBBCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, bl_grp ? 0 : (1 << 22), 1 << 22); /* ODT Strength */ switch (mrc_params->rd_odt_value) { case 1: temp = 0x3; break; /* 60 ohm */ case 2: temp = 0x3; break; /* 120 ohm */ case 3: temp = 0x3; break; /* 180 ohm */ default: temp = 0x3; break; /* 120 ohm */ } /* ODT strength */ mrc_alt_write_mask(DDRPHY, B0RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp << 5, 0x60); /* ODT strength */ mrc_alt_write_mask(DDRPHY, B1RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp << 5, 0x60); /* Dynamic ODT/DIFFAMP */ temp = (cas << 24) | (cas << 16) | (cas << 8) | (cas << 0); switch (speed) { case 0: temp -= 0x01010101; break; /* 800 */ case 1: temp -= 0x02020202; break; /* 1066 */ case 2: temp -= 0x03030303; break; /* 1333 */ case 3: temp -= 0x04040404; break; /* 1600 */ } /* Launch Time: ODT, DIFFAMP, ODT, DIFFAMP */ mrc_alt_write_mask(DDRPHY, B01LATCTL1 + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0x1f1f1f1f); switch (speed) { /* HSD#234715 */ case 0: temp = (0x06 << 16) | (0x07 << 8); break; /* 800 */ case 1: temp = (0x07 << 16) | (0x08 << 8); break; /* 1066 */ case 2: temp = (0x09 << 16) | (0x0a << 8); break; /* 1333 */ case 3: temp = (0x0a << 16) | (0x0b << 8); break; /* 1600 */ } /* On Duration: ODT, DIFFAMP */ mrc_alt_write_mask(DDRPHY, B0ONDURCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0x003f3f00); /* On Duration: ODT, DIFFAMP */ mrc_alt_write_mask(DDRPHY, B1ONDURCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0x003f3f00); switch (mrc_params->rd_odt_value) { case 0: /* override DIFFAMP=on, ODT=off */ temp = (0x3f << 16) | (0x3f << 10); break; default: /* override DIFFAMP=on, ODT=on */ temp = (0x3f << 16) | (0x2a << 10); break; } /* Override: DIFFAMP, ODT */ mrc_alt_write_mask(DDRPHY, B0OVRCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0x003ffc00); /* Override: DIFFAMP, ODT */ mrc_alt_write_mask(DDRPHY, B1OVRCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0x003ffc00); /* DLL Setup */ /* 1xCLK Domain Timings: tEDP,RCVEN,WDQS (PO) */ mrc_alt_write_mask(DDRPHY, B0LATCTL0 + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, ((cas + 7) << 16) | ((cas - 4) << 8) | ((cwl - 2) << 0), 0x003f1f1f); mrc_alt_write_mask(DDRPHY, B1LATCTL0 + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, ((cas + 7) << 16) | ((cas - 4) << 8) | ((cwl - 2) << 0), 0x003f1f1f); /* RCVEN Bypass (PO) */ mrc_alt_write_mask(DDRPHY, B0RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0, 0x81); mrc_alt_write_mask(DDRPHY, B1RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0, 0x81); /* TX */ mrc_alt_write_mask(DDRPHY, DQCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 1 << 16, 1 << 16); mrc_alt_write_mask(DDRPHY, B01PTRCTL1 + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 1 << 8, 1 << 8); /* RX (PO) */ /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */ mrc_alt_write_mask(DDRPHY, B0VREFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, (0x03 << 2) | (0x0 << 1) | (0x0 << 0), 0xff); /* Internal Vref Code, Enable#, Ext_or_Int (1=Ext) */ mrc_alt_write_mask(DDRPHY, B1VREFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, (0x03 << 2) | (0x0 << 1) | (0x0 << 0), 0xff); /* Per-Bit De-Skew Enable */ mrc_alt_write_mask(DDRPHY, B0RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0, 0x10); /* Per-Bit De-Skew Enable */ mrc_alt_write_mask(DDRPHY, B1RXIOBUFCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0, 0x10); } /* CLKEBB */ mrc_alt_write_mask(DDRPHY, CMDOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET, 0, 1 << 23); /* Enable tristate control of cmd/address bus */ mrc_alt_write_mask(DDRPHY, CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 0, 0x03); /* ODT RCOMP */ mrc_alt_write_mask(DDRPHY, CMDRCOMPODT + ch * DDRIOCCC_CH_OFFSET, (0x03 << 5) | (0x03 << 0), 0x3ff); /* CMDPM* registers must be programmed in this order */ /* Turn On Delays: SFR (regulator), MPLL */ mrc_alt_write_mask(DDRPHY, CMDPMDLYREG4 + ch * DDRIOCCC_CH_OFFSET, 0xffffffff, 0xffffffff); /* * Delays: ASSERT_IOBUFACT_to_ALLON0_for_PM_MSG_3, * VREG (MDLL) Turn On, ALLON0_to_DEASSERT_IOBUFACT * for_PM_MSG_gt0, MDLL Turn On */ mrc_alt_write_mask(DDRPHY, CMDPMDLYREG3 + ch * DDRIOCCC_CH_OFFSET, 0xfffff616, 0xffffffff); /* MPLL Divider Reset Delays */ mrc_alt_write_mask(DDRPHY, CMDPMDLYREG2 + ch * DDRIOCCC_CH_OFFSET, 0xffffffff, 0xffffffff); /* Turn Off Delays: VREG, Staggered MDLL, MDLL, PI */ mrc_alt_write_mask(DDRPHY, CMDPMDLYREG1 + ch * DDRIOCCC_CH_OFFSET, 0xffffffff, 0xffffffff); /* Turn On Delays: MPLL, Staggered MDLL, PI, IOBUFACT */ mrc_alt_write_mask(DDRPHY, CMDPMDLYREG0 + ch * DDRIOCCC_CH_OFFSET, 0xffffffff, 0xffffffff); /* Allow PUnit signals */ mrc_alt_write_mask(DDRPHY, CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, (0x6 << 8) | (0x1 << 6) | (0x4 << 0), 0xffe00f4f); /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */ mrc_alt_write_mask(DDRPHY, CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, (0x3 << 4) | (0x7 << 0), 0x7f); /* CLK-CTL */ mrc_alt_write_mask(DDRPHY, CCOBSCKEBBCTL + ch * DDRIOCCC_CH_OFFSET, 0, 1 << 24); /* CLKEBB */ /* Buffer Enable: CS,CKE,ODT,CLK */ mrc_alt_write_mask(DDRPHY, CCCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 0x1f, 0x000ffff1); /* ODT RCOMP */ mrc_alt_write_mask(DDRPHY, CCRCOMPODT + ch * DDRIOCCC_CH_OFFSET, (0x03 << 8) | (0x03 << 0), 0x00001f1f); /* DLL_VREG Bias Trim, VREF Tuning for DLL_VREG */ mrc_alt_write_mask(DDRPHY, CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, (0x3 << 4) | (0x7 << 0), 0x7f); /* * COMP (RON channel specific) * - DQ/DQS/DM RON: 32 Ohm * - CTRL/CMD RON: 27 Ohm * - CLK RON: 26 Ohm */ /* RCOMP Vref PU/PD */ mrc_alt_write_mask(DDRPHY, DQVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x08 << 24) | (0x03 << 16), 0x3f3f0000); /* RCOMP Vref PU/PD */ mrc_alt_write_mask(DDRPHY, CMDVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x0C << 24) | (0x03 << 16), 0x3f3f0000); /* RCOMP Vref PU/PD */ mrc_alt_write_mask(DDRPHY, CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x0F << 24) | (0x03 << 16), 0x3f3f0000); /* RCOMP Vref PU/PD */ mrc_alt_write_mask(DDRPHY, DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x08 << 24) | (0x03 << 16), 0x3f3f0000); /* RCOMP Vref PU/PD */ mrc_alt_write_mask(DDRPHY, CTLVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x0C << 24) | (0x03 << 16), 0x3f3f0000); /* DQS Swapped Input Enable */ mrc_alt_write_mask(DDRPHY, COMPEN1CH0 + ch * DDRCOMP_CH_OFFSET, (1 << 19) | (1 << 17), 0xc00ac000); /* ODT VREF = 1.5 x 274/360+274 = 0.65V (code of ~50) */ /* ODT Vref PU/PD */ mrc_alt_write_mask(DDRPHY, DQVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x32 << 8) | (0x03 << 0), 0x00003f3f); /* ODT Vref PU/PD */ mrc_alt_write_mask(DDRPHY, DQSVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x32 << 8) | (0x03 << 0), 0x00003f3f); /* ODT Vref PU/PD */ mrc_alt_write_mask(DDRPHY, CLKVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x0E << 8) | (0x05 << 0), 0x00003f3f); /* * Slew rate settings are frequency specific, * numbers below are for 800Mhz (speed == 0) * - DQ/DQS/DM/CLK SR: 4V/ns, * - CTRL/CMD SR: 1.5V/ns */ temp = (0x0e << 16) | (0x0e << 12) | (0x08 << 8) | (0x0b << 4) | (0x0b << 0); /* DCOMP Delay Select: CTL,CMD,CLK,DQS,DQ */ mrc_alt_write_mask(DDRPHY, DLYSELCH0 + ch * DDRCOMP_CH_OFFSET, temp, 0x000fffff); /* TCO Vref CLK,DQS,DQ */ mrc_alt_write_mask(DDRPHY, TCOVREFCH0 + ch * DDRCOMP_CH_OFFSET, (0x05 << 16) | (0x05 << 8) | (0x05 << 0), 0x003f3f3f); /* ODTCOMP CMD/CTL PU/PD */ mrc_alt_write_mask(DDRPHY, CCBUFODTCH0 + ch * DDRCOMP_CH_OFFSET, (0x03 << 8) | (0x03 << 0), 0x00001f1f); /* COMP */ mrc_alt_write_mask(DDRPHY, COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, 0, 0xc0000100); #ifdef BACKUP_COMPS /* DQ COMP Overrides */ /* RCOMP PU */ mrc_alt_write_mask(DDRPHY, DQDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* RCOMP PD */ mrc_alt_write_mask(DDRPHY, DQDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* DCOMP PU */ mrc_alt_write_mask(DDRPHY, DQDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x10 << 16), 0x801f0000); /* DCOMP PD */ mrc_alt_write_mask(DDRPHY, DQDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x10 << 16), 0x801f0000); /* ODTCOMP PU */ mrc_alt_write_mask(DDRPHY, DQODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* ODTCOMP PD */ mrc_alt_write_mask(DDRPHY, DQODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* DQS COMP Overrides */ /* RCOMP PU */ mrc_alt_write_mask(DDRPHY, DQSDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* RCOMP PD */ mrc_alt_write_mask(DDRPHY, DQSDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* DCOMP PU */ mrc_alt_write_mask(DDRPHY, DQSDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x10 << 16), 0x801f0000); /* DCOMP PD */ mrc_alt_write_mask(DDRPHY, DQSDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x10 << 16), 0x801f0000); /* ODTCOMP PU */ mrc_alt_write_mask(DDRPHY, DQSODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* ODTCOMP PD */ mrc_alt_write_mask(DDRPHY, DQSODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* CLK COMP Overrides */ /* RCOMP PU */ mrc_alt_write_mask(DDRPHY, CLKDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0c << 16), 0x801f0000); /* RCOMP PD */ mrc_alt_write_mask(DDRPHY, CLKDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0c << 16), 0x801f0000); /* DCOMP PU */ mrc_alt_write_mask(DDRPHY, CLKDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x07 << 16), 0x801f0000); /* DCOMP PD */ mrc_alt_write_mask(DDRPHY, CLKDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x07 << 16), 0x801f0000); /* ODTCOMP PU */ mrc_alt_write_mask(DDRPHY, CLKODTPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* ODTCOMP PD */ mrc_alt_write_mask(DDRPHY, CLKODTPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0b << 16), 0x801f0000); /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, 1 << 31, 1 << 31); /* CMD COMP Overrides */ /* RCOMP PU */ mrc_alt_write_mask(DDRPHY, CMDDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0d << 16), 0x803f0000); /* RCOMP PD */ mrc_alt_write_mask(DDRPHY, CMDDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0d << 16), 0x803f0000); /* DCOMP PU */ mrc_alt_write_mask(DDRPHY, CMDDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* DCOMP PD */ mrc_alt_write_mask(DDRPHY, CMDDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* CTL COMP Overrides */ /* RCOMP PU */ mrc_alt_write_mask(DDRPHY, CTLDRVPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0d << 16), 0x803f0000); /* RCOMP PD */ mrc_alt_write_mask(DDRPHY, CTLDRVPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0d << 16), 0x803f0000); /* DCOMP PU */ mrc_alt_write_mask(DDRPHY, CTLDLYPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); /* DCOMP PD */ mrc_alt_write_mask(DDRPHY, CTLDLYPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x0a << 16), 0x801f0000); #else /* DQ TCOCOMP Overrides */ /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, DQTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, DQTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); /* DQS TCOCOMP Overrides */ /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, DQSTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, DQSTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); /* CLK TCOCOMP Overrides */ /* TCOCOMP PU */ mrc_alt_write_mask(DDRPHY, CLKTCOPUCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); /* TCOCOMP PD */ mrc_alt_write_mask(DDRPHY, CLKTCOPDCTLCH0 + ch * DDRCOMP_CH_OFFSET, (1 << 31) | (0x1f << 16), 0x801f0000); #endif /* program STATIC delays */ #ifdef BACKUP_WCMD set_wcmd(ch, ddr_wcmd[PLATFORM_ID]); #else set_wcmd(ch, ddr_wclk[PLATFORM_ID] + HALF_CLK); #endif for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { set_wclk(ch, rk, ddr_wclk[PLATFORM_ID]); #ifdef BACKUP_WCTL set_wctl(ch, rk, ddr_wctl[PLATFORM_ID]); #else set_wctl(ch, rk, ddr_wclk[PLATFORM_ID] + HALF_CLK); #endif } } } } /* COMP (non channel specific) */ /* RCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQANADRVPUCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQANADRVPDCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CMDANADRVPUCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CMDANADRVPDCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CLKANADRVPUCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CLKANADRVPDCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQSANADRVPUCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQSANADRVPDCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CTLANADRVPUCTL, 1 << 30, 1 << 30); /* RCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CTLANADRVPDCTL, 1 << 30, 1 << 30); /* ODT: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQANAODTPUCTL, 1 << 30, 1 << 30); /* ODT: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQANAODTPDCTL, 1 << 30, 1 << 30); /* ODT: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CLKANAODTPUCTL, 1 << 30, 1 << 30); /* ODT: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CLKANAODTPDCTL, 1 << 30, 1 << 30); /* ODT: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQSANAODTPUCTL, 1 << 30, 1 << 30); /* ODT: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQSANAODTPDCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQANADLYPUCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQANADLYPDCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CMDANADLYPUCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CMDANADLYPDCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CLKANADLYPUCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CLKANADLYPDCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQSANADLYPUCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQSANADLYPDCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CTLANADLYPUCTL, 1 << 30, 1 << 30); /* DCOMP: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CTLANADLYPDCTL, 1 << 30, 1 << 30); /* TCO: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQANATCOPUCTL, 1 << 30, 1 << 30); /* TCO: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQANATCOPDCTL, 1 << 30, 1 << 30); /* TCO: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, CLKANATCOPUCTL, 1 << 30, 1 << 30); /* TCO: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, CLKANATCOPDCTL, 1 << 30, 1 << 30); /* TCO: Dither PU Enable */ mrc_alt_write_mask(DDRPHY, DQSANATCOPUCTL, 1 << 30, 1 << 30); /* TCO: Dither PD Enable */ mrc_alt_write_mask(DDRPHY, DQSANATCOPDCTL, 1 << 30, 1 << 30); /* TCOCOMP: Pulse Count */ mrc_alt_write_mask(DDRPHY, TCOCNTCTRL, 1, 3); /* ODT: CMD/CTL PD/PU */ mrc_alt_write_mask(DDRPHY, CHNLBUFSTATIC, (0x03 << 24) | (0x03 << 16), 0x1f1f0000); /* Set 1us counter */ mrc_alt_write_mask(DDRPHY, MSCNTR, 0x64, 0xff); mrc_alt_write_mask(DDRPHY, LATCH1CTL, 0x1 << 28, 0x70000000); /* Release PHY from reset */ mrc_alt_write_mask(DDRPHY, MASTERRSTN, 1, 1); /* STEP1 */ mrc_post_code(0x03, 0x11); for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* DQ01-DQ23 */ for (bl_grp = 0; bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; bl_grp++) { mrc_alt_write_mask(DDRPHY, DQMDLLCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 1 << 13, 1 << 13); /* Enable VREG */ delay_n(3); } /* ECC */ mrc_alt_write_mask(DDRPHY, ECCMDLLCTL, 1 << 13, 1 << 13); /* Enable VREG */ delay_n(3); /* CMD */ mrc_alt_write_mask(DDRPHY, CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 1 << 13, 1 << 13); /* Enable VREG */ delay_n(3); /* CLK-CTL */ mrc_alt_write_mask(DDRPHY, CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 1 << 13, 1 << 13); /* Enable VREG */ delay_n(3); } } /* STEP2 */ mrc_post_code(0x03, 0x12); delay_n(200); for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* DQ01-DQ23 */ for (bl_grp = 0; bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; bl_grp++) { mrc_alt_write_mask(DDRPHY, DQMDLLCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 1 << 17, 1 << 17); /* Enable MCDLL */ delay_n(50); } /* ECC */ mrc_alt_write_mask(DDRPHY, ECCMDLLCTL, 1 << 17, 1 << 17); /* Enable MCDLL */ delay_n(50); /* CMD */ mrc_alt_write_mask(DDRPHY, CMDMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 1 << 18, 1 << 18); /* Enable MCDLL */ delay_n(50); /* CLK-CTL */ mrc_alt_write_mask(DDRPHY, CCMDLLCTL + ch * DDRIOCCC_CH_OFFSET, 1 << 18, 1 << 18); /* Enable MCDLL */ delay_n(50); } } /* STEP3: */ mrc_post_code(0x03, 0x13); delay_n(100); for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* DQ01-DQ23 */ for (bl_grp = 0; bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; bl_grp++) { #ifdef FORCE_16BIT_DDRIO temp = (bl_grp && (mrc_params->channel_width == X16)) ? 0x11ff : 0xffff; #else temp = 0xffff; #endif /* Enable TXDLL */ mrc_alt_write_mask(DDRPHY, DQDLLTXCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, temp, 0xffff); delay_n(3); /* Enable RXDLL */ mrc_alt_write_mask(DDRPHY, DQDLLRXCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0xf, 0xf); delay_n(3); /* Enable RXDLL Overrides BL0 */ mrc_alt_write_mask(DDRPHY, B0OVRCTL + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0xf, 0xf); } /* ECC */ temp = 0xffff; mrc_alt_write_mask(DDRPHY, ECCDLLTXCTL, temp, 0xffff); delay_n(3); /* CMD (PO) */ mrc_alt_write_mask(DDRPHY, CMDDLLTXCTL + ch * DDRIOCCC_CH_OFFSET, temp, 0xffff); delay_n(3); } } /* STEP4 */ mrc_post_code(0x03, 0x14); for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* Host To Memory Clock Alignment (HMC) for 800/1066 */ for (bl_grp = 0; bl_grp < (NUM_BYTE_LANES / bl_divisor) / 2; bl_grp++) { /* CLK_ALIGN_MOD_ID */ mrc_alt_write_mask(DDRPHY, DQCLKALIGNREG2 + bl_grp * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, bl_grp ? 3 : 1, 0xf); } mrc_alt_write_mask(DDRPHY, ECCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 0x2, 0xf); mrc_alt_write_mask(DDRPHY, CMDCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 0x0, 0xf); mrc_alt_write_mask(DDRPHY, CCCLKALIGNREG2 + ch * DDRIODQ_CH_OFFSET, 0x2, 0xf); mrc_alt_write_mask(DDRPHY, CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET, 0x20, 0x30); /* * NUM_SAMPLES, MAX_SAMPLES, * MACRO_PI_STEP, MICRO_PI_STEP */ mrc_alt_write_mask(DDRPHY, CMDCLKALIGNREG1 + ch * DDRIOCCC_CH_OFFSET, (0x18 << 16) | (0x10 << 8) | (0x8 << 2) | (0x1 << 0), 0x007f7fff); /* TOTAL_NUM_MODULES, FIRST_U_PARTITION */ mrc_alt_write_mask(DDRPHY, CMDCLKALIGNREG2 + ch * DDRIOCCC_CH_OFFSET, (0x10 << 16) | (0x4 << 8) | (0x2 << 4), 0x001f0ff0); #ifdef HMC_TEST /* START_CLK_ALIGN=1 */ mrc_alt_write_mask(DDRPHY, CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET, 1 << 24, 1 << 24); while (msg_port_alt_read(DDRPHY, CMDCLKALIGNREG0 + ch * DDRIOCCC_CH_OFFSET) & (1 << 24)) ; /* wait for START_CLK_ALIGN=0 */ #endif /* Set RD/WR Pointer Seperation & COUNTEN & FIFOPTREN */ mrc_alt_write_mask(DDRPHY, CMDPTRREG + ch * DDRIOCCC_CH_OFFSET, 1, 1); /* WRPTRENABLE=1 */ /* COMP initial */ /* enable bypass for CLK buffer (PO) */ mrc_alt_write_mask(DDRPHY, COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, 1 << 5, 1 << 5); /* Initial COMP Enable */ mrc_alt_write_mask(DDRPHY, CMPCTRL, 1, 1); /* wait for Initial COMP Enable = 0 */ while (msg_port_alt_read(DDRPHY, CMPCTRL) & 1) ; /* disable bypass for CLK buffer (PO) */ mrc_alt_write_mask(DDRPHY, COMPEN0CH0 + ch * DDRCOMP_CH_OFFSET, ~(1 << 5), 1 << 5); /* IOBUFACT */ /* STEP4a */ mrc_alt_write_mask(DDRPHY, CMDCFGREG0 + ch * DDRIOCCC_CH_OFFSET, 1 << 2, 1 << 2); /* IOBUFACTRST_N=1 */ /* DDRPHY initialization complete */ mrc_alt_write_mask(DDRPHY, CMDPMCONFIG0 + ch * DDRIOCCC_CH_OFFSET, 1 << 20, 1 << 20); /* SPID_INIT_COMPLETE=1 */ } } LEAVEFN(); } /* This function performs JEDEC initialization on all enabled channels */ void perform_jedec_init(struct mrc_params *mrc_params) { uint8_t twr, wl, rank; uint32_t tck; u32 dtr0; u32 drp; u32 drmc; u32 mrs0_cmd = 0; u32 emrs1_cmd = 0; u32 emrs2_cmd = 0; u32 emrs3_cmd = 0; ENTERFN(); /* jedec_init starts */ mrc_post_code(0x04, 0x00); /* DDR3_RESET_SET=0, DDR3_RESET_RESET=1 */ mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 2, 0x102); /* Assert RESET# for 200us */ delay_u(200); /* DDR3_RESET_SET=1, DDR3_RESET_RESET=0 */ mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL, 0x100, 0x102); dtr0 = msg_port_read(MEM_CTLR, DTR0); /* * Set CKEVAL for populated ranks * then send NOP to each rank (#4550197) */ drp = msg_port_read(MEM_CTLR, DRP); drp &= 0x3; drmc = msg_port_read(MEM_CTLR, DRMC); drmc &= 0xfffffffc; drmc |= (DRMC_CKEMODE | drp); msg_port_write(MEM_CTLR, DRMC, drmc); for (rank = 0; rank < NUM_RANKS; rank++) { /* Skip to next populated rank */ if ((mrc_params->rank_enables & (1 << rank)) == 0) continue; dram_init_command(DCMD_NOP(rank)); } msg_port_write(MEM_CTLR, DRMC, (mrc_params->rd_odt_value == 0 ? DRMC_ODTMODE : 0)); /* * setup for emrs 2 * BIT[15:11] --> Always "0" * BIT[10:09] --> Rtt_WR: want "Dynamic ODT Off" (0) * BIT[08] --> Always "0" * BIT[07] --> SRT: use sr_temp_range * BIT[06] --> ASR: want "Manual SR Reference" (0) * BIT[05:03] --> CWL: use oem_tCWL * BIT[02:00] --> PASR: want "Full Array" (0) */ emrs2_cmd |= (2 << 3); wl = 5 + mrc_params->ddr_speed; emrs2_cmd |= ((wl - 5) << 9); emrs2_cmd |= (mrc_params->sr_temp_range << 13); /* * setup for emrs 3 * BIT[15:03] --> Always "0" * BIT[02] --> MPR: want "Normal Operation" (0) * BIT[01:00] --> MPR_Loc: want "Predefined Pattern" (0) */ emrs3_cmd |= (3 << 3); /* * setup for emrs 1 * BIT[15:13] --> Always "0" * BIT[12:12] --> Qoff: want "Output Buffer Enabled" (0) * BIT[11:11] --> TDQS: want "Disabled" (0) * BIT[10:10] --> Always "0" * BIT[09,06,02] --> Rtt_nom: use rtt_nom_value * BIT[08] --> Always "0" * BIT[07] --> WR_LVL: want "Disabled" (0) * BIT[05,01] --> DIC: use ron_value * BIT[04:03] --> AL: additive latency want "0" (0) * BIT[00] --> DLL: want "Enable" (0) * * (BIT5|BIT1) set Ron value * 00 --> RZQ/6 (40ohm) * 01 --> RZQ/7 (34ohm) * 1* --> RESERVED * * (BIT9|BIT6|BIT2) set Rtt_nom value * 000 --> Disabled * 001 --> RZQ/4 ( 60ohm) * 010 --> RZQ/2 (120ohm) * 011 --> RZQ/6 ( 40ohm) * 1** --> RESERVED */ emrs1_cmd |= (1 << 3); emrs1_cmd &= ~(1 << 6); if (mrc_params->ron_value == 0) emrs1_cmd |= (1 << 7); else emrs1_cmd &= ~(1 << 7); if (mrc_params->rtt_nom_value == 0) emrs1_cmd |= (DDR3_EMRS1_RTTNOM_40 << 6); else if (mrc_params->rtt_nom_value == 1) emrs1_cmd |= (DDR3_EMRS1_RTTNOM_60 << 6); else if (mrc_params->rtt_nom_value == 2) emrs1_cmd |= (DDR3_EMRS1_RTTNOM_120 << 6); /* save MRS1 value (excluding control fields) */ mrc_params->mrs1 = emrs1_cmd >> 6; /* * setup for mrs 0 * BIT[15:13] --> Always "0" * BIT[12] --> PPD: for Quark (1) * BIT[11:09] --> WR: use oem_tWR * BIT[08] --> DLL: want "Reset" (1, self clearing) * BIT[07] --> MODE: want "Normal" (0) * BIT[06:04,02] --> CL: use oem_tCAS * BIT[03] --> RD_BURST_TYPE: want "Interleave" (1) * BIT[01:00] --> BL: want "8 Fixed" (0) * WR: * 0 --> 16 * 1 --> 5 * 2 --> 6 * 3 --> 7 * 4 --> 8 * 5 --> 10 * 6 --> 12 * 7 --> 14 * CL: * BIT[02:02] "0" if oem_tCAS <= 11 (1866?) * BIT[06:04] use oem_tCAS-4 */ mrs0_cmd |= (1 << 14); mrs0_cmd |= (1 << 18); mrs0_cmd |= ((((dtr0 >> 12) & 7) + 1) << 10); tck = t_ck[mrc_params->ddr_speed]; /* Per JEDEC: tWR=15000ps DDR2/3 from 800-1600 */ twr = MCEIL(15000, tck); mrs0_cmd |= ((twr - 4) << 15); for (rank = 0; rank < NUM_RANKS; rank++) { /* Skip to next populated rank */ if ((mrc_params->rank_enables & (1 << rank)) == 0) continue; emrs2_cmd |= (rank << 22); dram_init_command(emrs2_cmd); emrs3_cmd |= (rank << 22); dram_init_command(emrs3_cmd); emrs1_cmd |= (rank << 22); dram_init_command(emrs1_cmd); mrs0_cmd |= (rank << 22); dram_init_command(mrs0_cmd); dram_init_command(DCMD_ZQCL(rank)); } LEAVEFN(); } /* * Dunit Initialization Complete * * Indicates that initialization of the Dunit has completed. * * Memory accesses are permitted and maintenance operation begins. * Until this bit is set to a 1, the memory controller will not accept * DRAM requests from the MEMORY_MANAGER or HTE. */ void set_ddr_init_complete(struct mrc_params *mrc_params) { u32 dco; ENTERFN(); dco = msg_port_read(MEM_CTLR, DCO); dco &= ~DCO_PMICTL; dco |= DCO_IC; msg_port_write(MEM_CTLR, DCO, dco); LEAVEFN(); } /* * This function will retrieve relevant timing data * * This data will be used on subsequent boots to speed up boot times * and is required for Suspend To RAM capabilities. */ void restore_timings(struct mrc_params *mrc_params) { uint8_t ch, rk, bl; const struct mrc_timings *mt = &mrc_params->timings; for (ch = 0; ch < NUM_CHANNELS; ch++) { for (rk = 0; rk < NUM_RANKS; rk++) { for (bl = 0; bl < NUM_BYTE_LANES; bl++) { set_rcvn(ch, rk, bl, mt->rcvn[ch][rk][bl]); set_rdqs(ch, rk, bl, mt->rdqs[ch][rk][bl]); set_wdqs(ch, rk, bl, mt->wdqs[ch][rk][bl]); set_wdq(ch, rk, bl, mt->wdq[ch][rk][bl]); if (rk == 0) { /* VREF (RANK0 only) */ set_vref(ch, bl, mt->vref[ch][bl]); } } set_wctl(ch, rk, mt->wctl[ch][rk]); } set_wcmd(ch, mt->wcmd[ch]); } } /* * Configure default settings normally set as part of read training * * Some defaults have to be set earlier as they may affect earlier * training steps. */ void default_timings(struct mrc_params *mrc_params) { uint8_t ch, rk, bl; for (ch = 0; ch < NUM_CHANNELS; ch++) { for (rk = 0; rk < NUM_RANKS; rk++) { for (bl = 0; bl < NUM_BYTE_LANES; bl++) { set_rdqs(ch, rk, bl, 24); if (rk == 0) { /* VREF (RANK0 only) */ set_vref(ch, bl, 32); } } } } } /* * This function will perform our RCVEN Calibration Algorithm. * We will only use the 2xCLK domain timings to perform RCVEN Calibration. * All byte lanes will be calibrated "simultaneously" per channel per rank. */ void rcvn_cal(struct mrc_params *mrc_params) { uint8_t ch; /* channel counter */ uint8_t rk; /* rank counter */ uint8_t bl; /* byte lane counter */ uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; #ifdef R2R_SHARING /* used to find placement for rank2rank sharing configs */ uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; #ifndef BACKUP_RCVN /* used to find placement for rank2rank sharing configs */ uint32_t num_ranks_enabled = 0; #endif #endif #ifdef BACKUP_RCVN #else uint32_t temp; /* absolute PI value to be programmed on the byte lane */ uint32_t delay[NUM_BYTE_LANES]; u32 dtr1, dtr1_save; #endif ENTERFN(); /* rcvn_cal starts */ mrc_post_code(0x05, 0x00); #ifndef BACKUP_RCVN /* need separate burst to sample DQS preamble */ dtr1 = msg_port_read(MEM_CTLR, DTR1); dtr1_save = dtr1; dtr1 |= DTR1_TCCD_12CLK; msg_port_write(MEM_CTLR, DTR1, dtr1); #endif #ifdef R2R_SHARING /* need to set "final_delay[][]" elements to "0" */ memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); #endif /* loop through each enabled channel */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* perform RCVEN Calibration on a per rank basis */ for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { /* * POST_CODE here indicates the current * channel and rank being calibrated */ mrc_post_code(0x05, 0x10 + ((ch << 4) | rk)); #ifdef BACKUP_RCVN /* et hard-coded timing values */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) set_rcvn(ch, rk, bl, ddr_rcvn[PLATFORM_ID]); #else /* enable FIFORST */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) { mrc_alt_write_mask(DDRPHY, B01PTRCTL1 + (bl >> 1) * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 0, 1 << 8); } /* initialize the starting delay to 128 PI (cas +1 CLK) */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { /* 1x CLK domain timing is cas-4 */ delay[bl] = (4 + 1) * FULL_CLK; set_rcvn(ch, rk, bl, delay[bl]); } /* now find the rising edge */ find_rising_edge(mrc_params, delay, ch, rk, true); /* Now increase delay by 32 PI (1/4 CLK) to place in center of high pulse */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { delay[bl] += QRTR_CLK; set_rcvn(ch, rk, bl, delay[bl]); } /* Now decrement delay by 128 PI (1 CLK) until we sample a "0" */ do { temp = sample_dqs(mrc_params, ch, rk, true); for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { if (temp & (1 << bl)) { if (delay[bl] >= FULL_CLK) { delay[bl] -= FULL_CLK; set_rcvn(ch, rk, bl, delay[bl]); } else { /* not enough delay */ training_message(ch, rk, bl); mrc_post_code(0xee, 0x50); } } } } while (temp & 0xff); #ifdef R2R_SHARING /* increment "num_ranks_enabled" */ num_ranks_enabled++; /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { delay[bl] += QRTR_CLK; /* add "delay[]" values to "final_delay[][]" for rolling average */ final_delay[ch][bl] += delay[bl]; /* set timing based on rolling average values */ set_rcvn(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); } #else /* Finally increment delay by 32 PI (1/4 CLK) to place in center of preamble */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { delay[bl] += QRTR_CLK; set_rcvn(ch, rk, bl, delay[bl]); } #endif /* disable FIFORST */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl += 2) { mrc_alt_write_mask(DDRPHY, B01PTRCTL1 + (bl >> 1) * DDRIODQ_BL_OFFSET + ch * DDRIODQ_CH_OFFSET, 1 << 8, 1 << 8); } #endif } } } } #ifndef BACKUP_RCVN /* restore original */ msg_port_write(MEM_CTLR, DTR1, dtr1_save); #endif LEAVEFN(); } /* * This function will perform the Write Levelling algorithm * (align WCLK and WDQS). * * This algorithm will act on each rank in each channel separately. */ void wr_level(struct mrc_params *mrc_params) { uint8_t ch; /* channel counter */ uint8_t rk; /* rank counter */ uint8_t bl; /* byte lane counter */ uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; #ifdef R2R_SHARING /* used to find placement for rank2rank sharing configs */ uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; #ifndef BACKUP_WDQS /* used to find placement for rank2rank sharing configs */ uint32_t num_ranks_enabled = 0; #endif #endif #ifdef BACKUP_WDQS #else /* determines stop condition for CRS_WR_LVL */ bool all_edges_found; /* absolute PI value to be programmed on the byte lane */ uint32_t delay[NUM_BYTE_LANES]; /* * static makes it so the data is loaded in the heap once by shadow(), * where non-static copies the data onto the stack every time this * function is called */ uint32_t address; /* address to be checked during COARSE_WR_LVL */ u32 dtr4, dtr4_save; #endif ENTERFN(); /* wr_level starts */ mrc_post_code(0x06, 0x00); #ifdef R2R_SHARING /* need to set "final_delay[][]" elements to "0" */ memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); #endif /* loop through each enabled channel */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { /* perform WRITE LEVELING algorithm on a per rank basis */ for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { /* * POST_CODE here indicates the current * rank and channel being calibrated */ mrc_post_code(0x06, 0x10 + ((ch << 4) | rk)); #ifdef BACKUP_WDQS for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { set_wdqs(ch, rk, bl, ddr_wdqs[PLATFORM_ID]); set_wdq(ch, rk, bl, ddr_wdqs[PLATFORM_ID] - QRTR_CLK); } #else /* * perform a single PRECHARGE_ALL command to * make DRAM state machine go to IDLE state */ dram_init_command(DCMD_PREA(rk)); /* * enable Write Levelling Mode * (EMRS1 w/ Write Levelling Mode Enable) */ dram_init_command(DCMD_MRS1(rk, 0x82)); /* * set ODT DRAM Full Time Termination * disable in MCU */ dtr4 = msg_port_read(MEM_CTLR, DTR4); dtr4_save = dtr4; dtr4 |= DTR4_ODTDIS; msg_port_write(MEM_CTLR, DTR4, dtr4); for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { /* * Enable Sandy Bridge Mode (WDQ Tri-State) & * Ensure 5 WDQS pulses during Write Leveling */ mrc_alt_write_mask(DDRPHY, DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch, 0x10000154, 0x100003fc); } /* Write Leveling Mode enabled in IO */ mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch, 1 << 16, 1 << 16); /* Initialize the starting delay to WCLK */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { /* * CLK0 --> RK0 * CLK1 --> RK1 */ delay[bl] = get_wclk(ch, rk); set_wdqs(ch, rk, bl, delay[bl]); } /* now find the rising edge */ find_rising_edge(mrc_params, delay, ch, rk, false); /* disable Write Levelling Mode */ mrc_alt_write_mask(DDRPHY, CCDDR3RESETCTL + DDRIOCCC_CH_OFFSET * ch, 0, 1 << 16); for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { /* Disable Sandy Bridge Mode & Ensure 4 WDQS pulses during normal operation */ mrc_alt_write_mask(DDRPHY, DQCTL + DDRIODQ_BL_OFFSET * bl + DDRIODQ_CH_OFFSET * ch, 0x00000154, 0x100003fc); } /* restore original DTR4 */ msg_port_write(MEM_CTLR, DTR4, dtr4_save); /* * restore original value * (Write Levelling Mode Disable) */ dram_init_command(DCMD_MRS1(rk, mrc_params->mrs1)); /* * perform a single PRECHARGE_ALL command to * make DRAM state machine go to IDLE state */ dram_init_command(DCMD_PREA(rk)); mrc_post_code(0x06, 0x30 + ((ch << 4) | rk)); /* * COARSE WRITE LEVEL: * check that we're on the correct clock edge */ /* hte reconfiguration request */ mrc_params->hte_setup = 1; /* start CRS_WR_LVL with WDQS = WDQS + 128 PI */ for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { delay[bl] = get_wdqs(ch, rk, bl) + FULL_CLK; set_wdqs(ch, rk, bl, delay[bl]); /* * program WDQ timings based on WDQS * (WDQ = WDQS - 32 PI) */ set_wdq(ch, rk, bl, (delay[bl] - QRTR_CLK)); } /* get an address in the targeted channel/rank */ address = get_addr(ch, rk); do { uint32_t coarse_result = 0x00; uint32_t coarse_result_mask = byte_lane_mask(mrc_params); /* assume pass */ all_edges_found = true; mrc_params->hte_setup = 1; coarse_result = check_rw_coarse(mrc_params, address); /* check for failures and margin the byte lane back 128 PI (1 CLK) */ for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { if (coarse_result & (coarse_result_mask << bl)) { all_edges_found = false; delay[bl] -= FULL_CLK; set_wdqs(ch, rk, bl, delay[bl]); /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */ set_wdq(ch, rk, bl, delay[bl] - QRTR_CLK); } } } while (!all_edges_found); #ifdef R2R_SHARING /* increment "num_ranks_enabled" */ num_ranks_enabled++; /* accumulate "final_delay[][]" values from "delay[]" values for rolling average */ for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { final_delay[ch][bl] += delay[bl]; set_wdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); /* program WDQ timings based on WDQS (WDQ = WDQS - 32 PI) */ set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled - QRTR_CLK); } #endif #endif } } } } LEAVEFN(); } void prog_page_ctrl(struct mrc_params *mrc_params) { u32 dpmc0; ENTERFN(); dpmc0 = msg_port_read(MEM_CTLR, DPMC0); dpmc0 &= ~DPMC0_PCLSTO_MASK; dpmc0 |= (4 << 16); dpmc0 |= DPMC0_PREAPWDEN; msg_port_write(MEM_CTLR, DPMC0, dpmc0); } /* * This function will perform the READ TRAINING Algorithm on all * channels/ranks/byte_lanes simultaneously to minimize execution time. * * The idea here is to train the VREF and RDQS (and eventually RDQ) values * to achieve maximum READ margins. The algorithm will first determine the * X coordinate (RDQS setting). This is done by collapsing the VREF eye * until we find a minimum required RDQS eye for VREF_MIN and VREF_MAX. * Then we take the averages of the RDQS eye at VREF_MIN and VREF_MAX, * then average those; this will be the final X coordinate. The algorithm * will then determine the Y coordinate (VREF setting). This is done by * collapsing the RDQS eye until we find a minimum required VREF eye for * RDQS_MIN and RDQS_MAX. Then we take the averages of the VREF eye at * RDQS_MIN and RDQS_MAX, then average those; this will be the final Y * coordinate. * * NOTE: this algorithm assumes the eye curves have a one-to-one relationship, * meaning for each X the curve has only one Y and vice-a-versa. */ void rd_train(struct mrc_params *mrc_params) { uint8_t ch; /* channel counter */ uint8_t rk; /* rank counter */ uint8_t bl; /* byte lane counter */ uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; #ifdef BACKUP_RDQS #else uint8_t side_x; /* tracks LEFT/RIGHT approach vectors */ uint8_t side_y; /* tracks BOTTOM/TOP approach vectors */ /* X coordinate data (passing RDQS values) for approach vectors */ uint8_t x_coordinate[2][2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; /* Y coordinate data (passing VREF values) for approach vectors */ uint8_t y_coordinate[2][2][NUM_CHANNELS][NUM_BYTE_LANES]; /* centered X (RDQS) */ uint8_t x_center[NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; /* centered Y (VREF) */ uint8_t y_center[NUM_CHANNELS][NUM_BYTE_LANES]; uint32_t address; /* target address for check_bls_ex() */ uint32_t result; /* result of check_bls_ex() */ uint32_t bl_mask; /* byte lane mask for result checking */ #ifdef R2R_SHARING /* used to find placement for rank2rank sharing configs */ uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; /* used to find placement for rank2rank sharing configs */ uint32_t num_ranks_enabled = 0; #endif #endif /* rd_train starts */ mrc_post_code(0x07, 0x00); ENTERFN(); #ifdef BACKUP_RDQS for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { set_rdqs(ch, rk, bl, ddr_rdqs[PLATFORM_ID]); } } } } } #else /* initialize x/y_coordinate arrays */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { /* x_coordinate */ x_coordinate[L][B][ch][rk][bl] = RDQS_MIN; x_coordinate[R][B][ch][rk][bl] = RDQS_MAX; x_coordinate[L][T][ch][rk][bl] = RDQS_MIN; x_coordinate[R][T][ch][rk][bl] = RDQS_MAX; /* y_coordinate */ y_coordinate[L][B][ch][bl] = VREF_MIN; y_coordinate[R][B][ch][bl] = VREF_MIN; y_coordinate[L][T][ch][bl] = VREF_MAX; y_coordinate[R][T][ch][bl] = VREF_MAX; } } } } } /* initialize other variables */ bl_mask = byte_lane_mask(mrc_params); address = get_addr(0, 0); #ifdef R2R_SHARING /* need to set "final_delay[][]" elements to "0" */ memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); #endif /* look for passing coordinates */ for (side_y = B; side_y <= T; side_y++) { for (side_x = L; side_x <= R; side_x++) { mrc_post_code(0x07, 0x10 + side_y * 2 + side_x); /* find passing values */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (0x1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (0x1 << rk)) { /* set x/y_coordinate search starting settings */ for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]); set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]); } /* get an address in the target channel/rank */ address = get_addr(ch, rk); /* request HTE reconfiguration */ mrc_params->hte_setup = 1; /* test the settings */ do { /* result[07:00] == failing byte lane (MAX 8) */ result = check_bls_ex(mrc_params, address); /* check for failures */ if (result & 0xff) { /* at least 1 byte lane failed */ for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { if (result & (bl_mask << bl)) { /* adjust the RDQS values accordingly */ if (side_x == L) x_coordinate[L][side_y][ch][rk][bl] += RDQS_STEP; else x_coordinate[R][side_y][ch][rk][bl] -= RDQS_STEP; /* check that we haven't closed the RDQS_EYE too much */ if ((x_coordinate[L][side_y][ch][rk][bl] > (RDQS_MAX - MIN_RDQS_EYE)) || (x_coordinate[R][side_y][ch][rk][bl] < (RDQS_MIN + MIN_RDQS_EYE)) || (x_coordinate[L][side_y][ch][rk][bl] == x_coordinate[R][side_y][ch][rk][bl])) { /* * not enough RDQS margin available at this VREF * update VREF values accordingly */ if (side_y == B) y_coordinate[side_x][B][ch][bl] += VREF_STEP; else y_coordinate[side_x][T][ch][bl] -= VREF_STEP; /* check that we haven't closed the VREF_EYE too much */ if ((y_coordinate[side_x][B][ch][bl] > (VREF_MAX - MIN_VREF_EYE)) || (y_coordinate[side_x][T][ch][bl] < (VREF_MIN + MIN_VREF_EYE)) || (y_coordinate[side_x][B][ch][bl] == y_coordinate[side_x][T][ch][bl])) { /* VREF_EYE collapsed below MIN_VREF_EYE */ training_message(ch, rk, bl); mrc_post_code(0xEE, 0x70 + side_y * 2 + side_x); } else { /* update the VREF setting */ set_vref(ch, bl, y_coordinate[side_x][side_y][ch][bl]); /* reset the X coordinate to begin the search at the new VREF */ x_coordinate[side_x][side_y][ch][rk][bl] = (side_x == L) ? RDQS_MIN : RDQS_MAX; } } /* update the RDQS setting */ set_rdqs(ch, rk, bl, x_coordinate[side_x][side_y][ch][rk][bl]); } } } } while (result & 0xff); } } } } } } mrc_post_code(0x07, 0x20); /* find final RDQS (X coordinate) & final VREF (Y coordinate) */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { uint32_t temp1; uint32_t temp2; /* x_coordinate */ DPF(D_INFO, "RDQS T/B eye rank%d lane%d : %d-%d %d-%d\n", rk, bl, x_coordinate[L][T][ch][rk][bl], x_coordinate[R][T][ch][rk][bl], x_coordinate[L][B][ch][rk][bl], x_coordinate[R][B][ch][rk][bl]); /* average the TOP side LEFT & RIGHT values */ temp1 = (x_coordinate[R][T][ch][rk][bl] + x_coordinate[L][T][ch][rk][bl]) / 2; /* average the BOTTOM side LEFT & RIGHT values */ temp2 = (x_coordinate[R][B][ch][rk][bl] + x_coordinate[L][B][ch][rk][bl]) / 2; /* average the above averages */ x_center[ch][rk][bl] = (uint8_t) ((temp1 + temp2) / 2); /* y_coordinate */ DPF(D_INFO, "VREF R/L eye lane%d : %d-%d %d-%d\n", bl, y_coordinate[R][B][ch][bl], y_coordinate[R][T][ch][bl], y_coordinate[L][B][ch][bl], y_coordinate[L][T][ch][bl]); /* average the RIGHT side TOP & BOTTOM values */ temp1 = (y_coordinate[R][T][ch][bl] + y_coordinate[R][B][ch][bl]) / 2; /* average the LEFT side TOP & BOTTOM values */ temp2 = (y_coordinate[L][T][ch][bl] + y_coordinate[L][B][ch][bl]) / 2; /* average the above averages */ y_center[ch][bl] = (uint8_t) ((temp1 + temp2) / 2); } } } } } #ifdef RX_EYE_CHECK /* perform an eye check */ for (side_y = B; side_y <= T; side_y++) { for (side_x = L; side_x <= R; side_x++) { mrc_post_code(0x07, 0x30 + side_y * 2 + side_x); /* update the settings for the eye check */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { if (side_x == L) set_rdqs(ch, rk, bl, x_center[ch][rk][bl] - (MIN_RDQS_EYE / 2)); else set_rdqs(ch, rk, bl, x_center[ch][rk][bl] + (MIN_RDQS_EYE / 2)); if (side_y == B) set_vref(ch, bl, y_center[ch][bl] - (MIN_VREF_EYE / 2)); else set_vref(ch, bl, y_center[ch][bl] + (MIN_VREF_EYE / 2)); } } } } } /* request HTE reconfiguration */ mrc_params->hte_setup = 1; /* check the eye */ if (check_bls_ex(mrc_params, address) & 0xff) { /* one or more byte lanes failed */ mrc_post_code(0xee, 0x74 + side_x * 2 + side_y); } } } #endif mrc_post_code(0x07, 0x40); /* set final placements */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { #ifdef R2R_SHARING /* increment "num_ranks_enabled" */ num_ranks_enabled++; #endif for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor); bl++) { /* x_coordinate */ #ifdef R2R_SHARING final_delay[ch][bl] += x_center[ch][rk][bl]; set_rdqs(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); #else set_rdqs(ch, rk, bl, x_center[ch][rk][bl]); #endif /* y_coordinate */ set_vref(ch, bl, y_center[ch][bl]); } } } } } #endif LEAVEFN(); } /* * This function will perform the WRITE TRAINING Algorithm on all * channels/ranks/byte_lanes simultaneously to minimize execution time. * * The idea here is to train the WDQ timings to achieve maximum WRITE margins. * The algorithm will start with WDQ at the current WDQ setting (tracks WDQS * in WR_LVL) +/- 32 PIs (+/- 1/4 CLK) and collapse the eye until all data * patterns pass. This is because WDQS will be aligned to WCLK by the * Write Leveling algorithm and WDQ will only ever have a 1/2 CLK window * of validity. */ void wr_train(struct mrc_params *mrc_params) { uint8_t ch; /* channel counter */ uint8_t rk; /* rank counter */ uint8_t bl; /* byte lane counter */ uint8_t bl_divisor = (mrc_params->channel_width == X16) ? 2 : 1; #ifdef BACKUP_WDQ #else uint8_t side; /* LEFT/RIGHT side indicator (0=L, 1=R) */ uint32_t temp; /* temporary DWORD */ /* 2 arrays, for L & R side passing delays */ uint32_t delay[2][NUM_CHANNELS][NUM_RANKS][NUM_BYTE_LANES]; uint32_t address; /* target address for check_bls_ex() */ uint32_t result; /* result of check_bls_ex() */ uint32_t bl_mask; /* byte lane mask for result checking */ #ifdef R2R_SHARING /* used to find placement for rank2rank sharing configs */ uint32_t final_delay[NUM_CHANNELS][NUM_BYTE_LANES]; /* used to find placement for rank2rank sharing configs */ uint32_t num_ranks_enabled = 0; #endif #endif /* wr_train starts */ mrc_post_code(0x08, 0x00); ENTERFN(); #ifdef BACKUP_WDQ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { set_wdq(ch, rk, bl, ddr_wdq[PLATFORM_ID]); } } } } } #else /* initialize "delay" */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { /* * want to start with * WDQ = (WDQS - QRTR_CLK) * +/- QRTR_CLK */ temp = get_wdqs(ch, rk, bl) - QRTR_CLK; delay[L][ch][rk][bl] = temp - QRTR_CLK; delay[R][ch][rk][bl] = temp + QRTR_CLK; } } } } } /* initialize other variables */ bl_mask = byte_lane_mask(mrc_params); address = get_addr(0, 0); #ifdef R2R_SHARING /* need to set "final_delay[][]" elements to "0" */ memset((void *)(final_delay), 0x00, (size_t)sizeof(final_delay)); #endif /* * start algorithm on the LEFT side and train each channel/bl * until no failures are observed, then repeat for the RIGHT side. */ for (side = L; side <= R; side++) { mrc_post_code(0x08, 0x10 + side); /* set starting values */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { set_wdq(ch, rk, bl, delay[side][ch][rk][bl]); } } } } } /* find passing values */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { /* get an address in the target channel/rank */ address = get_addr(ch, rk); /* request HTE reconfiguration */ mrc_params->hte_setup = 1; /* check the settings */ do { /* result[07:00] == failing byte lane (MAX 8) */ result = check_bls_ex(mrc_params, address); /* check for failures */ if (result & 0xff) { /* at least 1 byte lane failed */ for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { if (result & (bl_mask << bl)) { if (side == L) delay[L][ch][rk][bl] += WDQ_STEP; else delay[R][ch][rk][bl] -= WDQ_STEP; /* check for algorithm failure */ if (delay[L][ch][rk][bl] != delay[R][ch][rk][bl]) { /* * margin available * update delay setting */ set_wdq(ch, rk, bl, delay[side][ch][rk][bl]); } else { /* * no margin available * notify the user and halt */ training_message(ch, rk, bl); mrc_post_code(0xee, 0x80 + side); } } } } /* stop when all byte lanes pass */ } while (result & 0xff); } } } } } /* program WDQ to the middle of passing window */ for (ch = 0; ch < NUM_CHANNELS; ch++) { if (mrc_params->channel_enables & (1 << ch)) { for (rk = 0; rk < NUM_RANKS; rk++) { if (mrc_params->rank_enables & (1 << rk)) { #ifdef R2R_SHARING /* increment "num_ranks_enabled" */ num_ranks_enabled++; #endif for (bl = 0; bl < NUM_BYTE_LANES / bl_divisor; bl++) { DPF(D_INFO, "WDQ eye rank%d lane%d : %d-%d\n", rk, bl, delay[L][ch][rk][bl], delay[R][ch][rk][bl]); temp = (delay[R][ch][rk][bl] + delay[L][ch][rk][bl]) / 2; #ifdef R2R_SHARING final_delay[ch][bl] += temp; set_wdq(ch, rk, bl, final_delay[ch][bl] / num_ranks_enabled); #else set_wdq(ch, rk, bl, temp); #endif } } } } } #endif LEAVEFN(); } /* * This function will store relevant timing data * * This data will be used on subsequent boots to speed up boot times * and is required for Suspend To RAM capabilities. */ void store_timings(struct mrc_params *mrc_params) { uint8_t ch, rk, bl; struct mrc_timings *mt = &mrc_params->timings; for (ch = 0; ch < NUM_CHANNELS; ch++) { for (rk = 0; rk < NUM_RANKS; rk++) { for (bl = 0; bl < NUM_BYTE_LANES; bl++) { mt->rcvn[ch][rk][bl] = get_rcvn(ch, rk, bl); mt->rdqs[ch][rk][bl] = get_rdqs(ch, rk, bl); mt->wdqs[ch][rk][bl] = get_wdqs(ch, rk, bl); mt->wdq[ch][rk][bl] = get_wdq(ch, rk, bl); if (rk == 0) mt->vref[ch][bl] = get_vref(ch, bl); } mt->wctl[ch][rk] = get_wctl(ch, rk); } mt->wcmd[ch] = get_wcmd(ch); } /* need to save for a case of changing frequency after warm reset */ mt->ddr_speed = mrc_params->ddr_speed; } /* * The purpose of this function is to ensure the SEC comes out of reset * and IA initiates the SEC enabling Memory Scrambling. */ void enable_scrambling(struct mrc_params *mrc_params) { uint32_t lfsr = 0; uint8_t i; if (mrc_params->scrambling_enables == 0) return; ENTERFN(); /* 32 bit seed is always stored in BIOS NVM */ lfsr = mrc_params->timings.scrambler_seed; if (mrc_params->boot_mode == BM_COLD) { /* * factory value is 0 and in first boot, * a clock based seed is loaded. */ if (lfsr == 0) { /* * get seed from system clock * and make sure it is not all 1's */ lfsr = rdtsc() & 0x0fffffff; } else { /* * Need to replace scrambler * * get next 32bit LFSR 16 times which is the last * part of the previous scrambler vector */ for (i = 0; i < 16; i++) lfsr32(&lfsr); } /* save new seed */ mrc_params->timings.scrambler_seed = lfsr; } /* * In warm boot or S3 exit, we have the previous seed. * In cold boot, we have the last 32bit LFSR which is the new seed. */ lfsr32(&lfsr); /* shift to next value */ msg_port_write(MEM_CTLR, SCRMSEED, (lfsr & 0x0003ffff)); for (i = 0; i < 2; i++) msg_port_write(MEM_CTLR, SCRMLO + i, (lfsr & 0xaaaaaaaa)); LEAVEFN(); } /* * Configure MCU Power Management Control Register * and Scheduler Control Register */ void prog_ddr_control(struct mrc_params *mrc_params) { u32 dsch; u32 dpmc0; ENTERFN(); dsch = msg_port_read(MEM_CTLR, DSCH); dsch &= ~(DSCH_OOODIS | DSCH_OOOST3DIS | DSCH_NEWBYPDIS); msg_port_write(MEM_CTLR, DSCH, dsch); dpmc0 = msg_port_read(MEM_CTLR, DPMC0); dpmc0 &= ~DPMC0_DISPWRDN; dpmc0 |= (mrc_params->power_down_disable << 25); dpmc0 &= ~DPMC0_CLKGTDIS; dpmc0 &= ~DPMC0_PCLSTO_MASK; dpmc0 |= (4 << 16); dpmc0 |= DPMC0_PREAPWDEN; msg_port_write(MEM_CTLR, DPMC0, dpmc0); /* CMDTRIST = 2h - CMD/ADDR are tristated when no valid command */ mrc_write_mask(MEM_CTLR, DPMC1, 0x20, 0x30); LEAVEFN(); } /* * After training complete configure MCU Rank Population Register * specifying: ranks enabled, device width, density, address mode */ void prog_dra_drb(struct mrc_params *mrc_params) { u32 drp; u32 dco; u8 density = mrc_params->params.density; ENTERFN(); dco = msg_port_read(MEM_CTLR, DCO); dco &= ~DCO_IC; msg_port_write(MEM_CTLR, DCO, dco); drp = 0; if (mrc_params->rank_enables & 1) drp |= DRP_RKEN0; if (mrc_params->rank_enables & 2) drp |= DRP_RKEN1; if (mrc_params->dram_width == X16) { drp |= (1 << 4); drp |= (1 << 9); } /* * Density encoding in struct dram_params: 0=512Mb, 1=Gb, 2=2Gb, 3=4Gb * has to be mapped RANKDENSx encoding (0=1Gb) */ if (density == 0) density = 4; drp |= ((density - 1) << 6); drp |= ((density - 1) << 11); /* Address mode can be overwritten if ECC enabled */ drp |= (mrc_params->address_mode << 14); msg_port_write(MEM_CTLR, DRP, drp); dco &= ~DCO_PMICTL; dco |= DCO_IC; msg_port_write(MEM_CTLR, DCO, dco); LEAVEFN(); } /* Send DRAM wake command */ void perform_wake(struct mrc_params *mrc_params) { ENTERFN(); dram_wake_command(); LEAVEFN(); } /* * Configure refresh rate and short ZQ calibration interval * Activate dynamic self refresh */ void change_refresh_period(struct mrc_params *mrc_params) { u32 drfc; u32 dcal; u32 dpmc0; ENTERFN(); drfc = msg_port_read(MEM_CTLR, DRFC); drfc &= ~DRFC_TREFI_MASK; drfc |= (mrc_params->refresh_rate << 12); drfc |= DRFC_REFDBTCLR; msg_port_write(MEM_CTLR, DRFC, drfc); dcal = msg_port_read(MEM_CTLR, DCAL); dcal &= ~DCAL_ZQCINT_MASK; dcal |= (3 << 8); /* 63ms */ msg_port_write(MEM_CTLR, DCAL, dcal); dpmc0 = msg_port_read(MEM_CTLR, DPMC0); dpmc0 |= (DPMC0_DYNSREN | DPMC0_ENPHYCLKGATE); msg_port_write(MEM_CTLR, DPMC0, dpmc0); LEAVEFN(); } /* * Configure DDRPHY for Auto-Refresh, Periodic Compensations, * Dynamic Diff-Amp, ZQSPERIOD, Auto-Precharge, CKE Power-Down */ void set_auto_refresh(struct mrc_params *mrc_params) { uint32_t channel; uint32_t rank; uint32_t bl; uint32_t bl_divisor = 1; uint32_t temp; ENTERFN(); /* * Enable Auto-Refresh, Periodic Compensations, Dynamic Diff-Amp, * ZQSPERIOD, Auto-Precharge, CKE Power-Down */ for (channel = 0; channel < NUM_CHANNELS; channel++) { if (mrc_params->channel_enables & (1 << channel)) { /* Enable Periodic RCOMPS */ mrc_alt_write_mask(DDRPHY, CMPCTRL, 2, 2); /* Enable Dynamic DiffAmp & Set Read ODT Value */ switch (mrc_params->rd_odt_value) { case 0: temp = 0x3f; /* OFF */ break; default: temp = 0x00; /* Auto */ break; } for (bl = 0; bl < (NUM_BYTE_LANES / bl_divisor) / 2; bl++) { /* Override: DIFFAMP, ODT */ mrc_alt_write_mask(DDRPHY, B0OVRCTL + bl * DDRIODQ_BL_OFFSET + channel * DDRIODQ_CH_OFFSET, temp << 10, 0x003ffc00); /* Override: DIFFAMP, ODT */ mrc_alt_write_mask(DDRPHY, B1OVRCTL + bl * DDRIODQ_BL_OFFSET + channel * DDRIODQ_CH_OFFSET, temp << 10, 0x003ffc00); } /* Issue ZQCS command */ for (rank = 0; rank < NUM_RANKS; rank++) { if (mrc_params->rank_enables & (1 << rank)) dram_init_command(DCMD_ZQCS(rank)); } } } clear_pointers(); LEAVEFN(); } /* * Depending on configuration enables ECC support * * Available memory size is decreased, and updated with 0s * in order to clear error status. Address mode 2 forced. */ void ecc_enable(struct mrc_params *mrc_params) { u32 drp; u32 dsch; u32 ecc_ctrl; if (mrc_params->ecc_enables == 0) return; ENTERFN(); /* Configuration required in ECC mode */ drp = msg_port_read(MEM_CTLR, DRP); drp &= ~DRP_ADDRMAP_MASK; drp |= DRP_ADDRMAP_MAP1; drp |= DRP_PRI64BSPLITEN; msg_port_write(MEM_CTLR, DRP, drp); /* Disable new request bypass */ dsch = msg_port_read(MEM_CTLR, DSCH); dsch |= DSCH_NEWBYPDIS; msg_port_write(MEM_CTLR, DSCH, dsch); /* Enable ECC */ ecc_ctrl = (DECCCTRL_SBEEN | DECCCTRL_DBEEN | DECCCTRL_ENCBGEN); msg_port_write(MEM_CTLR, DECCCTRL, ecc_ctrl); /* Assume 8 bank memory, one bank is gone for ECC */ mrc_params->mem_size -= mrc_params->mem_size / 8; /* For S3 resume memory content has to be preserved */ if (mrc_params->boot_mode != BM_S3) { select_hte(); hte_mem_init(mrc_params, MRC_MEM_INIT); select_mem_mgr(); } LEAVEFN(); } /* * Execute memory test * if error detected it is indicated in mrc_params->status */ void memory_test(struct mrc_params *mrc_params) { uint32_t result = 0; ENTERFN(); select_hte(); result = hte_mem_init(mrc_params, MRC_MEM_TEST); select_mem_mgr(); DPF(D_INFO, "Memory test result %x\n", result); mrc_params->status = ((result == 0) ? MRC_SUCCESS : MRC_E_MEMTEST); LEAVEFN(); } /* Lock MCU registers at the end of initialization sequence */ void lock_registers(struct mrc_params *mrc_params) { u32 dco; ENTERFN(); dco = msg_port_read(MEM_CTLR, DCO); dco &= ~(DCO_PMICTL | DCO_PMIDIS); dco |= (DCO_DRPLOCK | DCO_CPGCLOCK); msg_port_write(MEM_CTLR, DCO, dco); LEAVEFN(); }