/*******************************************************************************
 * Copyright (c) 2021-2023 Renesas Electronics Corporation. All rights reserved.
 *
 *     RENESAS ELECTRONICS CONFIDENTIAL AND PROPRIETARY
 *
 * This software is provided as reference/sample code under the license
 * agreement between Renesas Electronics Corporation and licensee (the
 * "License Agreement") and shall be treated as specified in the License
 * Agreement.
 * These instructions, statements, and software are the confidential
 * information of Renesas Electronics Corporation. They must be used and
 * modified solely for the purpose for which it was furnished by Renesas
 * Electronics Corporation. All or part of these instructions, statements and
 * software must not be reproduced nor disclosed to any third party in any
 * form, unless permitted by the License Agreement.
 *
 * THIS SOFTWARE IS PROVIDED BY RENESAS ELEOCTRONICS CORPORATION "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
 * SATISFACTORY QUALITY, ACCURACY, TITLE AND NON-INFRINGEMENT ARE DISCLAIMED.
 * IN NO EVENT SHALL RENESAS ELECTRONICS CORPORATION BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, PUNITIVE, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
 * DAMAGE.
 ******************************************************************************/

#include <stdint.h>
#if defined(__RH850G3K__)
#include "mem_io.h"
#include "log.h"
#else
#include <mmio.h>
#include <debug.h>
#endif
#include "ddr_regdef.h"
#include "init_dram_tbl_s4.h"
#include "boot_init_dram_regdef.h"
#include "boot_init_dram.h"
#include "dram_sub_func.h"

#define DDR_BACKUPMODE
#define FATAL_MSG(x) NOTICE(x)

/*******************************************************************************
 *	variables
 ******************************************************************************/
/*
static uint32_t Prr_Product;
static uint32_t Prr_Cut;
*/
static uint32_t _cnf_BOARDTYPE;
static uint32_t brd_clk;
static uint32_t brd_clkdiv;
static uint32_t brd_clkdiva;
static uint32_t ddr_mbps;
static uint32_t ddr_mbpsdiv;
static uint32_t bus_mbps, bus_mbpsdiv;
static uint32_t ddr_tccd;
static const struct _boardcnf *Boardcnf;
static uint32_t ddr_phyvalid;
static uint32_t ddr_density[DRAM_CH_CNT][CS_CNT];
static uint32_t ch_have_this_cs[CS_CNT];
static uint32_t max_density;
static uint32_t ddr_mul;
static uint32_t ddr_mul_nf;
static uint32_t ddr_mul_low;
static uint32_t ddrtbl_load_num;

#define DDR_PHY_REGSET_MAX 143
#define DDR_PI_REGSET_MAX 223
static uint32_t _cnf_DDR_PHY_SLICE_REGSET[DDR_PHY_REGSET_MAX];
static uint32_t _cnf_DDR_PHY_ADR_V_REGSET[DDR_PHY_REGSET_MAX];
static uint32_t _cnf_DDR_PHY_ADR_G_REGSET[DDR_PHY_REGSET_MAX];
static uint32_t _cnf_DDR_PI_REGSET[DDR_PI_REGSET_MAX];
#ifdef DDR_BACKUPMODE
static uint32_t ddrBackup;
#endif

/*******************************************************************************
 *	Prototypes
 ******************************************************************************/
static uint32_t vch_nxt(uint32_t pos);
static void cpg_write_32(uint32_t a, uint32_t v);
static void pll3_control(uint32_t high);

static void send_dbcmd(uint32_t cmd);
static uint32_t reg_ddrphy_read(uint32_t phyno, uint32_t regadd);
static void reg_ddrphy_write(uint32_t phyno, uint32_t regadd, uint32_t regdata);
static void reg_ddrphy_write_a(uint32_t regadd, uint32_t regdata);
static void ddr_setval_s(uint32_t ch, uint32_t slice, uint32_t _regdef, uint32_t val);
static uint32_t ddr_getval_s(uint32_t ch, uint32_t slice, uint32_t _regdef);
static void ddr_setval(uint32_t ch, uint32_t regdef, uint32_t val);
static void ddr_setval_ach_s(uint32_t slice, uint32_t regdef, uint32_t val);
static void ddr_setval_ach(uint32_t regdef, uint32_t val);
static void ddr_setval_ach_as(uint32_t regdef, uint32_t val);
static uint32_t ddr_getval(uint32_t ch, uint32_t regdef);
static void _tblcopy(uint32_t *to, const uint32_t *from, uint32_t size);
static void ddrtbl_setval(uint32_t *tbl, uint32_t _regdef, uint32_t val);
static uint32_t ddrtbl_getval(uint32_t *tbl, uint32_t _regdef);
static uint16_t _f_scale(uint32_t f_ddr_mbps, uint32_t f_ddr_mbpsdiv, uint32_t ps, uint16_t cyc);
static void _f_scale_js2(uint32_t f_ddr_mbps, uint32_t f_ddr_mbpsdiv, uint16_t *f_js2);
static void ddrtbl_load(void);
static void ddrtbl_load_pi(void);
static void ddr_config(void);
static void dbsc_regset(void);
static void dbsc_regset_post(void);
static uint32_t dfi_init_start(void);
static void ddr_register_set(void);
static void wait_dbpdstat1(uint32_t status);
static uint32_t wait_freqchgreq(uint32_t req_assert);
static void set_freqchgack(uint32_t ack_assert);
static void set_dfifrequency(uint32_t freq);
static uint32_t pll3_freq(uint32_t fsel);
static uint32_t pi_training_go(void);
static void manual_frequency_change(void);
static uint32_t manual_training_wrlvl(void);
static uint32_t manual_training_rdgtlvl(void);
static uint32_t manual_training_rdlvl(void);
static uint32_t manual_training_wdqlvl(void);
static uint32_t ca_vref_training(void);
static uint32_t init_ddr(void);
static uint32_t boardcnf_get_brd_type(void);
static void dbsc_write_32(uintptr_t addr, uint32_t data);

/*******************************************************************************
 *	load board configuration
 ******************************************************************************/
#include "boot_init_dram_config.c"

/*******************************************************************************
 *	CA Vref Training setting
 ******************************************************************************/
#ifndef DDR_CAVREF_VAL
#define DDR_CAVREF_VAL		0x11
#endif

#ifndef DDR_CAVREF_DELTA
#define DDR_CAVREF_DELTA	3
#endif

/*******************************************************************************
 *	macro for channel selection loop
 ******************************************************************************/
static uint32_t vch_nxt(uint32_t pos)
{
	uint32_t posn;

	for (posn = pos; posn < DRAM_CH_CNT; posn++) {
		if (ddr_phyvalid & (1U << posn)) {
			break;
		}
	}
	return posn;
}

#define foreach_vch(ch) \
for (ch = vch_nxt(0); ch < DRAM_CH_CNT; ch = vch_nxt(ch + 1U))

#define foreach_ech(ch) \
for (ch = 0U; ch < DRAM_CH_CNT; ch++)

/*******************************************************************************
 *	Printing functions
 ******************************************************************************/
#define MSG_LF(...)

/*******************************************************************************
 *	clock settings, reset control
 ******************************************************************************/
static void cpg_write_32(uint32_t a, uint32_t v)
{
	mmio_write_32(CPG_CPGWPR, ~v);
	mmio_write_32(a, v);
}

static void pll3_control(uint32_t high)
{
	uint32_t dataDIV, dataMUL;
	uint32_t ssmode, dataNF;

	if (high) {
		/* High frequency mode */
		dataMUL = ddr_mul - 1U;
		dataDIV = 0x02U;
		dataNF  = ddr_mul_nf;
	} else {
		/* Low frequency mode (25MHz) */
		dataMUL = ddr_mul_low - 1U;	/* PLL3VCO = 1600MHz */
		dataDIV = 0x0CU;			/* div     = 64 */
		dataNF  = 0x00U;
	}

	ssmode = 0x04U;
	dataMUL = (dataMUL << 20) | (ssmode << 16);
	dataNF  = (dataNF  << 20);

	/* PLL3 multiplie set */
	if (((mmio_read_32(CPG_PLL3CR0) & 0x3FFFFF7FU) != dataMUL) || (mmio_read_32(CPG_PLL3CR1) != dataNF) ) {
		cpg_write_32(CPG_PLL3CR0, dataMUL);
		cpg_write_32(CPG_PLL3CR1, dataNF);
		cpg_write_32(CPG_PLL3CR0, mmio_read_32(CPG_PLL3CR0) | CPG_PLL3CR_KICK_BIT);
		while ((mmio_read_32(CPG_PLLECR) & CPG_PLLECR_PLL3ST_BIT) != CPG_PLLECR_PLL3ST_BIT);
	}
	/* PLL3 DIV set(Target value) */
	while ((mmio_read_32(CPG_FRQCRD) & CPG_FRQCRD_KICK_BIT));
	cpg_write_32(CPG_FRQCRD, dataDIV | (mmio_read_32(CPG_FRQCRD) & 0xFFFFFF80U));
	cpg_write_32(CPG_FRQCRD, mmio_read_32(CPG_FRQCRD) | CPG_FRQCRD_KICK_BIT);
	while ((mmio_read_32(CPG_FRQCRD) & CPG_FRQCRD_KICK_BIT));

}

/*******************************************************************************
 *	DDR memory register access
 ******************************************************************************/
static void send_dbcmd(uint32_t cmd)
{
	/* dummy read */
	mmio_read_32(DBSC_DBCMD);
	while ((mmio_read_32(DBSC_DBWAIT)) & 0x01U);

	dbsc_write_32((DBSC_DBCMD), cmd);
}

/*******************************************************************************
 *	DDRPHY register access (raw)
 ******************************************************************************/
static uint32_t reg_ddrphy_read(uint32_t phyno, uint32_t regadd)
{
	uint32_t val;

	val = 0U;
	mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00004000U);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != (regadd | 0x0000C000U));

	val = mmio_read_32(DBSC_DBPDRGA(phyno));

	mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000U);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd);

	mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000U);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd);

	val = mmio_read_32(DBSC_DBPDRGD(phyno));
	(void)val;

	return val;
}

static void reg_ddrphy_write(uint32_t phyno, uint32_t regadd, uint32_t regdata)
{
	mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd);

	mmio_write_32(DBSC_DBPDRGD(phyno), regdata);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != (regadd | 0x00008000U));

	mmio_write_32(DBSC_DBPDRGA(phyno), regadd | 0x00008000U);
	while (mmio_read_32(DBSC_DBPDRGA(phyno)) != regadd);

	mmio_write_32(DBSC_DBPDRGA(phyno), regadd);
}

static void reg_ddrphy_write_a(uint32_t regadd, uint32_t regdata)
{
	uint32_t ch;

	foreach_vch(ch) {
		reg_ddrphy_write(ch, regadd, regdata);
	}
}

/*******************************************************************************
 *	DDRPHY register access (field modify)
 ******************************************************************************/
static void ddr_setval_s(uint32_t ch, uint32_t slice, uint32_t _regdef, uint32_t val)
{
	uint32_t adr;
	uint32_t lsb;
	uint32_t len;
	uint32_t msk;
	uint32_t tmp;

	adr = DDR_REGDEF_ADR(_regdef) + 0x100U * slice;
	len = DDR_REGDEF_LEN(_regdef);
	lsb = DDR_REGDEF_LSB(_regdef);
	if (len == 0x20U) {
		msk = 0xffffffffU;
	} else {
		msk = ((1U << len) - 1U) << lsb;
	}

	tmp = reg_ddrphy_read(ch, adr);
	tmp = (tmp & (~msk)) | ((val << lsb) & msk);
	reg_ddrphy_write(ch, adr, tmp);
}

static uint32_t ddr_getval_s(uint32_t ch, uint32_t slice, uint32_t _regdef)
{
	uint32_t adr;
	uint32_t lsb;
	uint32_t len;
	uint32_t msk;
	uint32_t tmp;

	adr = DDR_REGDEF_ADR(_regdef) + 0x100U * slice;
	len = DDR_REGDEF_LEN(_regdef);
	lsb = DDR_REGDEF_LSB(_regdef);
	if (len == 0x20U) {
		msk = 0xffffffffU;
	} else {
		msk = ((1U << len) - 1U);
	}

	tmp = reg_ddrphy_read(ch, adr);
	tmp = (tmp >> lsb) & msk;

	return tmp;
}

static void ddr_setval(uint32_t ch, uint32_t regdef, uint32_t val)
{
	ddr_setval_s(ch, 0U, regdef, val);
}

static void ddr_setval_ach_s(uint32_t slice, uint32_t regdef, uint32_t val)
{
	uint32_t ch;

	foreach_vch(ch) {
		ddr_setval_s(ch, slice, regdef, val);
	}
}

static void ddr_setval_ach(uint32_t regdef, uint32_t val)
{
	ddr_setval_ach_s(0U, regdef, val);
}

static void ddr_setval_ach_as(uint32_t regdef, uint32_t val)
{
	uint32_t slice;

	for (slice = 0U; slice < SLICE_CNT; slice++) {
		ddr_setval_ach_s(slice, regdef, val);
	}
}

static uint32_t ddr_getval(uint32_t ch, uint32_t regdef)
{
	return ddr_getval_s(ch, 0U, regdef);
}

/*******************************************************************************
 *	DBSC register access
 ******************************************************************************/
static void dbsc_write_32(uintptr_t addr, uint32_t data)
{
	*((volatile uint32_t*)addr) = data;
}

/*******************************************************************************
 *	handling functions for setteing ddrphy value table
 ******************************************************************************/
static void _tblcopy(uint32_t *to, const uint32_t *from, uint32_t size)
{
	uint32_t i;

	for (i = 0U; i < size; i++) {
		to[i] = from[i];
	}
}

static void ddrtbl_setval(uint32_t *tbl, uint32_t _regdef, uint32_t val)
{
	uint32_t adr;
	uint32_t lsb;
	uint32_t len;
	uint32_t msk;
	uint32_t tmp;
	const uint32_t adrmsk = 0x000000ffU;

	adr = DDR_REGDEF_ADR(_regdef);
	len = DDR_REGDEF_LEN(_regdef);
	lsb = DDR_REGDEF_LSB(_regdef);
	if (len == 0x20U) {
		msk = 0xffffffffU;
	} else {
		msk = ((1U << len) - 1U) << lsb;
	}

	tmp = tbl[adr & adrmsk];
	tmp = (tmp & (~msk)) | ((val << lsb) & msk);
	tbl[adr & adrmsk] = tmp;
}

static uint32_t ddrtbl_getval(uint32_t *tbl, uint32_t _regdef)
{
	uint32_t adr;
	uint32_t lsb;
	uint32_t len;
	uint32_t msk;
	uint32_t tmp;
	const uint32_t adrmsk = 0x000000ffU;

	adr = DDR_REGDEF_ADR(_regdef);
	len = DDR_REGDEF_LEN(_regdef);
	lsb = DDR_REGDEF_LSB(_regdef);
	if (len == 0x20U) {
		msk = 0xffffffffU;
	} else {
		msk = ((1U << len) - 1U);
	}

	tmp = tbl[adr & adrmsk];
	tmp = (tmp >> lsb) & msk;

	return tmp;
}

/*******************************************************************************
 *	functions and parameters for timing setting
 ******************************************************************************/
struct _jedec_spec1 {
	uint16_t fx3;
	uint8_t RLwoDBI;
	uint8_t RLwDBI;
	uint8_t WL;
	uint8_t nWR;
	uint8_t nRTP;
	uint8_t ODTLon;
	uint8_t MR1;
	uint8_t MR2;
	uint16_t tRRD;
	uint16_t tFAW;
};

#define JS1_USABLEC_SPEC_LO 2
#define JS1_USABLEC_SPEC_HI 7
#define JS1_FREQ_TBL_NUM 8
#define JS1_MR1(f) (0x04 | ((f) << 4))
#define JS1_MR2(f) (0x00 | ((f) << 3) | (f))

const struct _jedec_spec1 js1[JS1_FREQ_TBL_NUM] = {
	{  800,  6,  6,  4,  6 , 8, 0, JS1_MR1(0), JS1_MR2(0)|0x40,10000, 40000 }, /*  533.333Mbps*/
	{ 1600, 10, 12,  8, 10 , 8, 0, JS1_MR1(1), JS1_MR2(1)|0x40,10000, 40000 }, /* 1066.666Mbps*/
	{ 2400, 14, 16, 12, 16 , 8, 6, JS1_MR1(2), JS1_MR2(2)|0x40,10000, 40000 }, /* 1600.000Mbps*/
	{ 3200, 20, 22, 10, 20 , 8, 4, JS1_MR1(3), JS1_MR2(3),     10000, 40000 }, /* 2133.333Mbps*/
	{ 4000, 24, 28, 12, 24 ,10, 4, JS1_MR1(4), JS1_MR2(4),     10000, 40000 }, /* 2666.666Mbps*/
	{ 4800, 28, 32, 14, 30 ,12, 6, JS1_MR1(5), JS1_MR2(5),     10000, 40000 }, /* 3200.000Mbps*/
	{ 5600, 32, 36, 16, 34 ,14, 6, JS1_MR1(6), JS1_MR2(6),     10000, 40000 }, /* 3733.333Mbps*/
	{ 6400, 36, 40, 18, 40 ,16, 8, JS1_MR1(7), JS1_MR2(7),      7500, 30000 }  /* 4266.666Mbps*/
};

struct _jedec_spec2 {
	uint16_t ps;
	uint16_t cyc;
};

#define JS2_tSR 0
#define JS2_tXP 1
#define JS2_tRTP 2
#define JS2_tRCD 3
#define JS2_tRPpb 4
#define JS2_tRPab 5
#define JS2_tRAS 6
#define JS2_tWR 7
#define JS2_tWTR 8
#define JS2_tRRD 9
#define JS2_tPPD 10
#define JS2_tFAW 11
#define JS2_tDQSCK 12
#define JS2_tCKEHCMD 13
#define JS2_tCKELCMD 14
#define JS2_tCKELPD 15
#define JS2_tMRR 16
#define JS2_tMRW 17
#define JS2_tMRD 18
#define JS2_tZQCALns 19
#define JS2_tZQLAT 20
#define JS2_tIEdly 21
#define JS2_tODTon_min 22
#define JS2_TBLCNT 23

#define JS2_tRCpb (JS2_TBLCNT)
#define JS2_tRCab (JS2_TBLCNT + 1)
#define JS2_tRFCab (JS2_TBLCNT + 2)
#define JS2_CNT (JS2_TBLCNT + 3)

#ifndef JS2_DERATE
#define JS2_DERATE 0
#endif
const struct _jedec_spec2 jedec_spec2[2][JS2_TBLCNT] = {
	{
/*tSR   */	{  15000,  3 },
/*tXP   */	{   7500,  3 },
/*tRTP  */	{   7500,  8 },
/*tRCD  */	{  18000,  4 },
/*tRPpb */	{  18000,  3 },
/*tRPab */	{  21000,  3 },
/*tRAS  */	{  42000,  3 },
/*tWR   */	{  18000,  4 },
/*tWTR  */	{  10000,  8 },
/*tRRD  */	{      0,  0 },
/*tPPD  */	{      0,  0 },
/*tFAW  */	{      0,  0 },
/*tDQSCK*/	{   3500,  0 },
/*tCKEHCMD*/	{   7500,  3 },
/*tCKELCMD*/	{   7500,  3 },
/*tCKELPD*/	{   7500,  3 },
/*tMRR*/	{      0,  8 },
/*tMRW*/	{  10000, 10 },
/*tMRD*/	{  14000, 10 },
/*tZQCALns*/	{   1000,  0 },
/*tZQLAT*/	{  30000, 10 },
/*tIEdly*/	{   6000,  0 },
/*tODTon_min*/	{   1500,  0 }
	},{
/*tSR   */	{  15000,  3 },
/*tXP   */	{   7500,  3 },
/*tRTP  */	{   7500,  8 },
/*tRCD  */	{  19875,  4 },
/*tRPpb */	{  19875,  3 },
/*tRPab */	{  22875,  3 },
/*tRAS  */	{  43875,  3 },
/*tWR   */	{  18000,  4 },
/*tWTR  */	{  10000,  8 },
/*tRRD  */	{   1875,  0 },
/*tPPD  */	{      0,  0 },
/*tFAW  */	{      0,  0 },
/*tDQSCK*/	{   3600,  0 },
/*tCKEHCMD*/	{   7500,  3 },
/*tCKELCMD*/	{   7500,  3 },
/*tCKELPD*/	{   7500,  3 },
/*tMRR*/	{      0,  8 },
/*tMRW*/	{  10000, 10 },
/*tMRD*/	{  14000, 10 },
/*tZQCALns*/	{   1000,  0 },
/*tZQLAT*/	{  30000, 10 },
/*tIEdly*/	{   6000,  0 },
/*tODTon_min*/	{   1500,  0 }
	}
};

const uint16_t jedec_spec2_tRFC_ab[5] = {
/*	4Gb, 6Gb, 8Gb,12Gb,16Gb (24Gb/32Gb non)	*/
	130, 180, 180, 280, 280
};

static uint32_t js1_ind;
static uint16_t js2[JS2_CNT];
static uint8_t RL;
static uint8_t WL;

static uint16_t _f_scale(uint32_t f_ddr_mbps, uint32_t f_ddr_mbpsdiv, uint32_t ps, uint16_t cyc)
{
	uint32_t tmp;
	uint32_t div;

	tmp = (((uint32_t)(ps) + 9U) / 10U) * f_ddr_mbps;
	div = tmp / (200000U * f_ddr_mbpsdiv);
	if (tmp != (div * 200000U * f_ddr_mbpsdiv)) {
		div = div + 1U;
	}

	if (div > cyc) {
		cyc = (uint16_t)div;
	}

	return cyc;
}

static void _f_scale_js2(uint32_t f_ddr_mbps, uint32_t f_ddr_mbpsdiv, uint16_t *f_js2)
{
	uint32_t i;

	for (i = 0U; i < JS2_TBLCNT; i++) {
		f_js2[i] = _f_scale(f_ddr_mbps, f_ddr_mbpsdiv,
			(uint32_t)jedec_spec2[JS2_DERATE][i].ps,
			jedec_spec2[JS2_DERATE][i].cyc);
	}

	f_js2[JS2_tRRD] = _f_scale(f_ddr_mbps, f_ddr_mbpsdiv, (uint32_t)(js1[js1_ind].tRRD + jedec_spec2[JS2_DERATE][JS2_tPPD].ps), 4U);
	f_js2[JS2_tFAW] = _f_scale(f_ddr_mbps, f_ddr_mbpsdiv, (uint32_t)(js1[js1_ind].tFAW + jedec_spec2[JS2_DERATE][JS2_tFAW].ps), 0U);
	f_js2[JS2_tZQCALns] = _f_scale(f_ddr_mbps, f_ddr_mbpsdiv, (uint32_t)(jedec_spec2[JS2_DERATE][JS2_tZQCALns].ps) * 1000U, 0U);
	f_js2[JS2_tRCpb] = f_js2[JS2_tRAS] + f_js2[JS2_tRPpb];
	f_js2[JS2_tRCab] = f_js2[JS2_tRAS] + f_js2[JS2_tRPab];
}

static const uint32_t _reg_PI_MR1_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR1_DATA_F0_0,
		_reg_PI_MR1_DATA_F0_1
	},
	{
		_reg_PI_MR1_DATA_F1_0,
		_reg_PI_MR1_DATA_F1_1
	},
	{
		_reg_PI_MR1_DATA_F2_0,
		_reg_PI_MR1_DATA_F2_1
	}
};

static const uint32_t _reg_PI_MR2_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR2_DATA_F0_0,
		_reg_PI_MR2_DATA_F0_1
	},
	{
		_reg_PI_MR2_DATA_F1_0,
		_reg_PI_MR2_DATA_F1_1
	},
	{
		_reg_PI_MR2_DATA_F2_0,
		_reg_PI_MR2_DATA_F2_1
	}
};

static const uint32_t _reg_PI_MR3_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR3_DATA_F0_0,
		_reg_PI_MR3_DATA_F0_1
	},
	{
		_reg_PI_MR3_DATA_F1_0,
		_reg_PI_MR3_DATA_F1_1
	},
	{
		_reg_PI_MR3_DATA_F2_0,
		_reg_PI_MR3_DATA_F2_1
	}
};

static const uint32_t _reg_PI_MR11_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR11_DATA_F0_0,
		_reg_PI_MR11_DATA_F0_1
	},
	{
		_reg_PI_MR11_DATA_F1_0,
		_reg_PI_MR11_DATA_F1_1
	},
	{
		_reg_PI_MR11_DATA_F2_0,
		_reg_PI_MR11_DATA_F2_1
	}
};

static const uint32_t _reg_PI_MR12_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR12_DATA_F0_0,
		_reg_PI_MR12_DATA_F0_1
	},
	{
		_reg_PI_MR12_DATA_F1_0,
		_reg_PI_MR12_DATA_F1_1
	},
	{
		_reg_PI_MR12_DATA_F2_0,
		_reg_PI_MR12_DATA_F2_1
	}
};

static const uint32_t _reg_PI_MR14_DATA_Fx_CSx[3][CS_CNT] = {
	{
		_reg_PI_MR14_DATA_F0_0,
		_reg_PI_MR14_DATA_F0_1
	},
	{
		_reg_PI_MR14_DATA_F1_0,
		_reg_PI_MR14_DATA_F1_1
	},
	{
		_reg_PI_MR14_DATA_F2_0,
		_reg_PI_MR14_DATA_F2_1
	}
};

/*******************************************************************************
 *	load table data into DDR registers
 ******************************************************************************/
static void ddrtbl_load(void)
{
	uint32_t i;
	uint32_t slice;
	uint32_t cs;
	uint32_t adr;
	uint32_t dataL;
	uint32_t tmp[2];

	const uint32_t DDR_PHY_SLICE_REGSET_OFS = DDR_PHY_SLICE_REGSET_OFS_S4;
	const uint32_t DDR_PHY_ADR_V_REGSET_OFS = DDR_PHY_ADR_V_REGSET_OFS_S4;
	const uint32_t DDR_PHY_ADR_G_REGSET_OFS = DDR_PHY_ADR_G_REGSET_OFS_S4;
	const uint32_t DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_S4;

	const uint32_t DDR_PHY_SLICE_REGSET_SIZE = DDR_PHY_SLICE_REGSET_SIZE_S4;

	const uint32_t DDR_PHY_SLICE_REGSET_NUM = DDR_PHY_SLICE_REGSET_NUM_S4;
	const uint32_t DDR_PHY_ADR_V_REGSET_NUM = DDR_PHY_ADR_V_REGSET_NUM_S4;
	const uint32_t DDR_PHY_ADR_G_REGSET_NUM = DDR_PHY_ADR_G_REGSET_NUM_S4;

	/***********************************************************************
	 * TIMING REGISTERS
	 ***********************************************************************/
	/* search jedec_spec1 index */
	for (i = JS1_USABLEC_SPEC_LO; i < JS1_FREQ_TBL_NUM - 1U; i++) {
		if (js1[i].fx3 * 2U * ddr_mbpsdiv >= ddr_mbps * 3U) {
			break;
		}
	}
	if (JS1_USABLEC_SPEC_HI < i) {
		js1_ind = JS1_USABLEC_SPEC_HI;
	} else {
		js1_ind = i;
	}

	RL = js1[js1_ind].RLwDBI;

	WL = js1[js1_ind].WL;

	/* calculate jedec_spec2 */
	_f_scale_js2(ddr_mbps, ddr_mbpsdiv, js2);

	/***********************************************************************
	 * PREPARE TBL
	 ***********************************************************************/
	_tblcopy(_cnf_DDR_PHY_SLICE_REGSET,
		DDR_PHY_SLICE_REGSET_S4, DDR_PHY_SLICE_REGSET_NUM_S4);
	_tblcopy(_cnf_DDR_PHY_ADR_V_REGSET,
		DDR_PHY_ADR_V_REGSET_S4, DDR_PHY_ADR_V_REGSET_NUM_S4);
	_tblcopy(_cnf_DDR_PHY_ADR_G_REGSET,
		DDR_PHY_ADR_G_REGSET_S4, DDR_PHY_ADR_G_REGSET_NUM_S4);
	_tblcopy(_cnf_DDR_PI_REGSET,
		DDR_PI_REGSET_S4, DDR_PI_REGSET_NUM_S4);

	/***********************************************************************
	 * Adjust PI paramters
	 ***********************************************************************/
	if (js2[JS2_tIEdly] > RL) {
		js2[JS2_tIEdly] = RL;
	}

	if (js2[JS2_tIEdly] >= 0x0fU) {
		dataL = 0x0fU;
	} else {
		dataL = js2[JS2_tIEdly];
	}

	ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_RDDATA_EN_DLY, dataL);
	ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_RDDATA_EN_TSEL_DLY, (dataL - 2U));
	ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_RDDATA_EN_OE_DLY, dataL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_RDLAT_ADJ_F1, RL - dataL - 1U);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_ADJ_F1, WL - 4U);

	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_F1, WL);

	tmp[0] = js1[js1_ind].MR1;
	tmp[1] = js1[js1_ind].MR2;

	for (cs = 0U; cs < CS_CNT; cs++) {
		for (i = 1U; i < 3U; i++) {
			ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR1_DATA_Fx_CSx[i][cs], tmp[0]);
			ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR2_DATA_Fx_CSx[i][cs], tmp[1]);
		}
	}

#ifdef DDR_BACKUPMODE
	if (ddrBackup == DRAM_BOOT_STATUS_WARM) {
		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_SET_DFI_INPUT_RST_PAD, 0x01U);
	}
#endif /* DDR_BACKUPMODE */

	/***********************************************************************
	 * Read Vref (SoC side) Training range
	 ***********************************************************************/
	dataL = (uint32_t)(Boardcnf->vref_r);
	if (dataL) {
		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_VREF_INITIAL_START_POINT, dataL & 0x00ffU);
		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_VREF_INITIAL_STOP_POINT, (dataL & 0xff00U) >> 8);
	}

	/***********************************************************************
	 * Write Vref (MR14) Training range
	 ***********************************************************************/
	dataL = (uint32_t)(Boardcnf->vref_w);
	if (dataL) {
		ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WDQLVL_VREF_DELTA_F0, (((dataL & 0xff00U) >> 8) - (dataL & 0x00ffU) + 1) / 2);
		ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_WDQLVL_VREF_DELTA_F1, (((dataL & 0xff00U) >> 8) - (dataL & 0x00ffU) + 1) / 2);
		for (cs = 0U; cs < CS_CNT; cs++) {
			for (i = 0U; i < 3U; i++) {
				ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR14_DATA_Fx_CSx[i][cs], (((dataL & 0xff00U) >> 8) + (dataL & 0x00ffU)) / 2);
			}
		}
	}

	/***********************************************************************
	 * CA Vref (MR12) Default configuration
	 ***********************************************************************/
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR12_DATA_F0_1, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR12_DATA_F0_0, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR12_DATA_F1_1, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_MR12_DATA_F1_0, DDR_CAVREF_VAL);

	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_CALVL_VREF_INITIAL_START_POINT_F0, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_CALVL_VREF_INITIAL_STOP_POINT_F0, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_CALVL_VREF_INITIAL_START_POINT_F1, DDR_CAVREF_VAL);
	ddrtbl_setval(_cnf_DDR_PI_REGSET, _reg_PI_CALVL_VREF_INITIAL_STOP_POINT_F1, DDR_CAVREF_VAL);

	/***********************************************************************
	 * Low Freq setting
	 ***********************************************************************/
	if (3 * ddr_mbps < 4 * 1600 * ddr_mbpsdiv) {
		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_PLL_CTRL, 0x1342);
	}

	if (ddr_mbps < 4 * 640 * ddr_mbpsdiv) {
		/* PCLK(10-100MHz) */
		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_PAD_DSLICE_IO_CFG, 0x05);
		ddrtbl_setval(_cnf_DDR_PHY_ADR_V_REGSET, _reg_PHY_PAD_ADR_IO_CFG_0, 0x05);
		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_PAD_ACS_IO_CFG, 0x05);

		/* CAL_CLK(10-20MHz) */
		ddrtbl_setval(_cnf_DDR_PHY_SLICE_REGSET, _reg_PHY_DATA_DC_CAL_CLK_SEL, 0x04);
		ddrtbl_setval(_cnf_DDR_PHY_ADR_V_REGSET, _reg_PHY_ADR_DC_CAL_CLK_SEL_0, 0x04);
		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_CLK_DC_CAL_CLK_SEL, 0x04);
		ddrtbl_setval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_CAL_CLK_SELECT_0, 0x05);
	}

	/***********************************************************************
	 * FREQ_SEL_MULTICAST & PER_CS_TRAINING_MULTICAST SET (for safety)
	 ***********************************************************************/
	ddr_setval_ach(_reg_PHY_FREQ_SEL_MULTICAST_EN, 0x01U);
	ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_MULTICAST_EN, 0x01U);

	/***********************************************************************
	 * SET DATA SLICE TABLE
	 ***********************************************************************/
	for (slice = 0U; slice < SLICE_CNT; slice++) {
		adr = DDR_PHY_SLICE_REGSET_OFS + DDR_PHY_SLICE_REGSET_SIZE * slice;
		for (i = 0U; i < DDR_PHY_SLICE_REGSET_NUM; i++) {
			reg_ddrphy_write_a(adr + i, _cnf_DDR_PHY_SLICE_REGSET[i]);
		}
	}

	/***********************************************************************
	 * SET ADR SLICE TABLE
	 ***********************************************************************/
	adr = DDR_PHY_ADR_V_REGSET_OFS;
	for (i = 0U; i < DDR_PHY_ADR_V_REGSET_NUM; i++) {
		reg_ddrphy_write_a(adr + i, _cnf_DDR_PHY_ADR_V_REGSET[i]);
	}

	/***********************************************************************
	 * SET ADRCTRL SLICE TABLE
	 ***********************************************************************/
	adr = DDR_PHY_ADR_G_REGSET_OFS;
	for (i = 0U; i < DDR_PHY_ADR_G_REGSET_NUM; i++) {
		reg_ddrphy_write_a(adr + i, _cnf_DDR_PHY_ADR_G_REGSET[i]);
	}

	/***********************************************************************
	 * SET PI REGISTERS
	 ***********************************************************************/
	adr = DDR_PI_REGSET_OFS;
	ddrtbl_load_num = 83U;
	for (i = 0U; i < ddrtbl_load_num; i++) {
		reg_ddrphy_write_a(adr + i, _cnf_DDR_PI_REGSET[i]);
	}
}

static void ddrtbl_load_pi(void)
{
	uint32_t ch, dataL;

	const uint32_t DDR_PI_REGSET_OFS = DDR_PI_REGSET_OFS_S4;
	const uint32_t DDR_PI_REGSET_NUM = DDR_PI_REGSET_NUM_S4;

	/***********************************************************************
	 * SET PI REGISTERS
	 ***********************************************************************/
	dataL = ddrtbl_load_num;
	while (dataL < DDR_PI_REGSET_NUM) {
		reg_ddrphy_write_a(DDR_PI_REGSET_OFS + dataL, _cnf_DDR_PI_REGSET[dataL]);
		++dataL;
	}

	foreach_vch(ch) {
		/* --- DATA_BYTE_SWAP --- */
		dataL = (uint32_t)(Boardcnf->ch[ch].dqs_swap);
		ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_EN, 0x01U);
		ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE0, dataL & 0x0fU);
		ddr_setval(ch, _reg_PI_DATA_BYTE_SWAP_SLICE1, (dataL >> 4) & 0x0fU);

		if (!(ch_have_this_cs[1] & (1U << ch))) {
			ddr_setval(ch, _reg_PI_CS_MAP, 0x01U);
		}
	}
}

/*******************************************************************************
 *	CONFIGURE DDR REGISTERS
 ******************************************************************************/
static void ddr_config(void)
{
	uint32_t ch, slice;
	uint32_t dataL;
	uint8_t high_byte[SLICE_CNT];

	const uint32_t _par_CALVL_DEVICE_MAP = 1U;

	foreach_vch(ch) {
	/***********************************************************************
	 * BOARD SETTINGS (DQ, DM, VREF_DRIVING)
	 ***********************************************************************/
		for (slice = 0U; slice < SLICE_CNT; slice++) {
			high_byte[slice] = ((uint32_t)(Boardcnf->ch[ch].dqs_swap) >> (4U * slice)) % 2;
			ddr_setval_s(ch, slice, _reg_PHY_DQ_DM_SWIZZLE0, Boardcnf->ch[ch].dq_swap[slice]);
			ddr_setval_s(ch, slice, _reg_PHY_DQ_DM_SWIZZLE1, (uint32_t)(Boardcnf->ch[ch].dm_swap[slice]));
			if (high_byte[slice]) {
				/* HIGHER 16 BYTE */
				ddr_setval_s(ch, slice, _reg_PHY_CALVL_VREF_DRIVING_SLICE, 0x00U);
			} else {
				/* LOWER 16 BYTE */
				ddr_setval_s(ch, slice, _reg_PHY_CALVL_VREF_DRIVING_SLICE, 0x01U);
			}
		}

	/***********************************************************************
	 * BOARD SETTINGS (CA, ADDR_SEL)
	 ***********************************************************************/
		dataL = (0x00ffffffU & (uint32_t)(Boardcnf->ch[ch].ca_swap));

		/* --- ADR_ADDR_SEL --- */
		ddr_setval(ch, _reg_PHY_ADR_ADDR_SEL_0, dataL);

		/* --- ADR_CALVL_SWIZZLE --- */
		if (high_byte[1]) {
			dataL |= 0x00888888U;
		}
		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE0_0, dataL);
		ddr_setval(ch, _reg_PHY_ADR_CALVL_SWIZZLE1_0, 0x00000000U);
		ddr_setval(ch, _reg_PHY_CALVL_DEVICE_MAP, _par_CALVL_DEVICE_MAP);

	/***********************************************************************
	 * mask CS_MAP if RANK1 is not found
	 ***********************************************************************/
		if (!(ch_have_this_cs[1] & (1U << ch))) {
			ddr_setval(ch, _reg_PHY_ADR_CALVL_RANK_CTRL_0, 0x00U);
			for (slice = 0U; slice < SLICE_CNT; slice++) {
				ddr_setval_s(ch, slice, _reg_PHY_PER_CS_TRAINING_EN, 0x00U);
			}
		}
	}
}

/*******************************************************************************
 *	DBSC register setting functions
 ******************************************************************************/
static void dbsc_regset_pre(void)
{
	uint32_t ch, cs;

	/***********************************************************************
	 * PRIMARY SETTINGS
	 ***********************************************************************/
	/* LPDDR4, BL=16, DFI interface */
	dbsc_write_32(DBSC_DBKIND, 0x0000000aU);
	dbsc_write_32(DBSC_DBKINDA, 0x0000000aU);
	dbsc_write_32(DBSC_DBBL, 0x00000002U);
	dbsc_write_32(DBSC_DBBLA, 0x00000002U);
	dbsc_write_32(DBSC_DBPHYCONF0, 0x00000001U);

	dbsc_write_32(DBSC_DBSYSCONF0, 0x00000001U);

	/* FREQRATIO=2 */
	dbsc_write_32(DBSC_DBSYSCONF1, 0x00000002U);
	dbsc_write_32(DBSC_DBSYSCONF1A, 0x00000002U);

	dbsc_write_32(DBSC_DBSYSCONF2, 0x00000001U);
	dbsc_write_32(DBSC_DBSYSCONF2A, 0x00000061U);

	foreach_ech(ch) {
		for (cs = 0U; cs < CS_CNT; cs++) {
			if (ddr_density[ch][cs] == 0xffU) {
				mmio_write_32(DBSC_DBMEMCONF(ch, cs), 0x00U);
				mmio_write_32(DBSC_DBMEMCONFA(ch, cs), 0x00U);
			} else {
				mmio_write_32(DBSC_DBMEMCONF(ch, cs), DBMEMCONF_REGD(ddr_density[ch][cs]));
				mmio_write_32(DBSC_DBMEMCONFA(ch, cs), DBMEMCONF_REGD(ddr_density[ch][cs]));
			}
		}
	}
}

static void dbsc_regset(void)
{
	uint32_t dataL;
	uint32_t tmp[4];

	/* RFC */
	js2[JS2_tRFCab] = _f_scale(ddr_mbps, ddr_mbpsdiv,
		1UL * jedec_spec2_tRFC_ab[max_density] * 1000, 0);

	/* DBTR0.CL  : RL */
	dbsc_write_32(DBSC_DBTR(0), RL);

	/* DBTR1.CWL : WL */
	dbsc_write_32(DBSC_DBTR(1), WL);

	/* DBTR2.AL  : 0 */
	dbsc_write_32(DBSC_DBTR(2), 0U);

	/* DBTR3.TRCD: tRCD */
	dbsc_write_32(DBSC_DBTR(3), js2[JS2_tRCD]);

	/* DBTR4.TRPA,TRP: tRPab,tRPpb */
	dbsc_write_32(DBSC_DBTR(4), (js2[JS2_tRPab] << 16) | js2[JS2_tRPpb]);

	/* DBTR5.TRC : use tRCpb */
	dbsc_write_32(DBSC_DBTR(5), js2[JS2_tRCpb]);

	/* DBTR6.TRAS : tRAS */
	dbsc_write_32(DBSC_DBTR(6), js2[JS2_tRAS]);

	/* DBTR7.TRRD : tRRD */
	dbsc_write_32(DBSC_DBTR(7), (js2[JS2_tRRD] << 16) | js2[JS2_tRRD]);

	/* DBTR8.TFAW : tFAW */
	dbsc_write_32(DBSC_DBTR(8), js2[JS2_tFAW]);

	/* DBTR9.TRDPR : nRTP */
	dbsc_write_32(DBSC_DBTR(9), js1[js1_ind].nRTP);

	/* DBTR10.TWR : nWR */
	dbsc_write_32(DBSC_DBTR(10), js1[js1_ind].nWR);

	/* DBTR11.TRDWR : RL +  BL / 2 + Rounddown(tRPST) + PHY_ODTLoff - ODTLon + tDQSCK - tODTon,min + PCB delay (out+in) + tPHY_ODToff */
	dbsc_write_32(DBSC_DBTR(11),
		RL + (16 / 2) + 1 + 2 - js1[js1_ind].ODTLon + js2[JS2_tDQSCK] - js2[JS2_tODTon_min] + _f_scale(ddr_mbps, ddr_mbpsdiv, (1500 + 500 + 800), 0));

	/* DBTR12.TWRRD : WL + 1 + BL/2 + tWTR */
	dataL = WL + 1 + (16 / 2) + js2[JS2_tWTR];
	dbsc_write_32(DBSC_DBTR(12), (dataL << 16) | dataL);

	/* DBTR13.TRFCAB : tRFCab */
	dbsc_write_32(DBSC_DBTR(13),
		(js2[JS2_tRFCab]));

	/* DBTR14.TCKEHDLL,tCKEH : tCKEHCMD,tCKEHCMD */
	dbsc_write_32(DBSC_DBTR(14),
		((js2[JS2_tCKEHCMD] + 3U) << 16) | (js2[JS2_tCKEHCMD] + 3U));

	/* DBTR15.TCKESR,TCKEL : tSR,tCKELPD */
	dbsc_write_32(DBSC_DBTR(15),
		(js2[JS2_tSR] << 16) | (js2[JS2_tCKELPD]));

	/* DBTR16 */
	tmp[0] = WL;
	tmp[1] = WL - 4U;
	tmp[2] = RL + 33U + 2U;
	tmp[3] = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_RDLAT_ADJ_F1) - 1U;
	dbsc_write_32(DBSC_DBTR(16),
		(tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);

	/* DBTR24 */
	/* WRCSLAT */
	tmp[0] = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_WRLAT_ADJ_F1) - 5U;
	/* WRCSGAP = 6 + 1 */
	tmp[1] = 7U;
	/* RDCSLAT */
	tmp[2] = RL - 6U;
	/* RDCSGAP */
	tmp[3] = 4U + 2U;
	dbsc_write_32(DBSC_DBTR(24),
		(tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);

	/* DBTR17.TMODRD,TMOD,TRDMR: tMRR,tMRD,(0)*/
	dbsc_write_32(DBSC_DBTR(17), (js2[JS2_tMRR] << 24) | (js2[JS2_tMRD] << 16));

	/* DBTR18.RODTL, RODTA, WODTL, WODTA : do not use in LPDDR4 */
	dbsc_write_32(DBSC_DBTR(18), 0U);

	/* DBTR19.TZQCL, TZQCS : do not use in LPDDR4 */
	dbsc_write_32(DBSC_DBTR(19), 0U);

	/* DBTR20.TXSDLL, TXS : tRFCab+tCKEHCMD */
	dataL = js2[JS2_tRFCab] + js2[JS2_tCKEHCMD] + 3U;
	dbsc_write_32(DBSC_DBTR(20), (dataL << 16) | dataL);

	/* DBTR21.TCCD */
	dbsc_write_32(DBSC_DBTR(21), (ddr_tccd << 16) | ddr_tccd);

	/* DBTR22.ZQLAT : */
	dbsc_write_32(DBSC_DBTR(22), (js2[JS2_tZQCALns] << 16) | js2[JS2_tZQLAT]);

	/* DBTR23.RRSPC */
	dbsc_write_32(DBSC_DBTR(23), 0x00000003U);

	/* DBTR25 : do not use in LPDDR4 */
	dbsc_write_32(DBSC_DBTR(25), 0U);

	/* DBRNK : */
	/* DBSC_DBRNK2 rkrr */
	dbsc_write_32(DBSC_DBRNK(2), 0x000000CC);

	/* DBSC_DBRNK3 rkrw */
	dbsc_write_32(DBSC_DBRNK(3), 0x00000066);

	/* DBSC_DBRNK4 rkwr */
	dbsc_write_32(DBSC_DBRNK(4), 0x00000066);

	/* DBSC_DBRNK5 rkww */
	dbsc_write_32(DBSC_DBRNK(5), 0x000000CC);

	/***********************************************************************
	 * timing registers for Scheduler
	 ***********************************************************************/
	/* SCFCTST0 */
	/* SCFCTST0 ACT-ACT*/
	tmp[3] = 1UL * js2[JS2_tRCpb] * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST0 RDA-ACT*/
	tmp[2] = 1UL * ((16U / 2U) + js2[JS2_tRTP] - 8U + js2[JS2_tRPpb]) * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST0 WRA-ACT*/
	tmp[1] = 1UL * (WL + 1U + (16U / 2U) + js1[js1_ind].nWR) * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST0 PRE-ACT*/
	tmp[0] = 1UL * js2[JS2_tRPpb] * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	dbsc_write_32(DBSC_SCFCTST0, (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);

	/* SCFCTST1 */
	/* SCFCTST1 RD-WR*/
	tmp[3] = 1UL * (mmio_read_32(DBSC_DBTR(11)) & 0xffU) * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST1 WR-RD*/
	tmp[2] = 1UL * (mmio_read_32(DBSC_DBTR(12)) & 0xffU) * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST1 ACT-RD/WR*/
	tmp[1] = 1UL * js2[JS2_tRCD] * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	/* SCFCTST1 ASYNCOFS*/
	tmp[0] = 12U;
	dbsc_write_32(DBSC_SCFCTST1, (tmp[3] << 24) | (tmp[2] << 16) | (tmp[1] << 8) | tmp[0]);

	/* DBSCHRW1 */
	/* DBSCHRW1 SCTRFCAB*/
	tmp[0] = 1UL * js2[JS2_tRFCab] * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv;
	dataL =(((mmio_read_32(DBSC_DBTR(16)) & 0x00FF0000U) >> 16)
		+ (mmio_read_32(DBSC_DBTR(22)) & 0x0000FFFFU)
		+ (0x28U * 2U)) * bus_mbps * ddr_mbpsdiv / ddr_mbps / bus_mbpsdiv + 7U;
	if (tmp[0] < dataL) {
		tmp[0] = dataL;
	}
	dbsc_write_32(DBSC_DBSCHRW1, tmp[0]
		+ ((mmio_read_32(DBSC_DBTR(22)) & 0x0000FFFF)
		* bus_mbps * ddr_mbpsdiv + (ddr_mbps - 1U)) / ddr_mbps / bus_mbpsdiv);

	/***********************************************************************
	 * QOS and CAM
	 ***********************************************************************/
	dbsc_write_32(DBSC_DBBCAMDIS, 0x00000001U);
}

static void dbsc_regset_post(void)
{
	uint32_t dataL;

#if RCAR_REWT_TRAINING != 0
	uint32_t ctrl_clk;
	uint32_t clk_count;
	uint32_t phymster_req_interval;
#endif /* RCAR_REWT_TRAINING */

	/*set DBI */
	dbsc_write_32(DBSC_DBDBICNT, 0x00000003U);

	/*set REFCYCLE */
	dataL = DBSC_REFINT * ddr_mbps / 2000U / ddr_mbpsdiv;
	dbsc_write_32(DBSC_DBRFCNF1, 0x00080000U | (dataL & 0x0000ffffU));
	dbsc_write_32(DBSC_DBRFCNF2, 0x00010000U | DBSC_REFINTS);

#if RCAR_REWT_TRAINING != 0
	/* Periodic-WriteDQ Training seeting */
	dbsc_write_32(DBSC_DBDFIPMSTRCNF, 0x00000000U);

	ddr_setval_ach(_reg_PI_WDQLVL_EN_F1, 0x03U);
	ddr_setval_ach(_reg_PI_WDQLVL_VREF_EN, 0x00U);
	ddr_setval_ach(_reg_PI_WDQLVL_PERIODIC, 0x01U);

	/* DFI_PHYMSTR_ACK , WTmode = b'01 */
	dbsc_write_32(DBSC_DBDFIPMSTRCNF, 0x00000011U);
#endif /* RCAR_REWT_TRAINING */

	/* PCLK */
	if (ddr_mbps < 4 * 640 * ddr_mbpsdiv) {
		ddr_setval_ach(_reg_PHY_PAD_CAL_IO_CFG_0, 0x05U);
	} else {
		ddr_setval_ach(_reg_PHY_PAD_CAL_IO_CFG_0, 0x06U);
	}

	/* periodic SoC zqcal enable */
	ddr_setval_ach(_reg_PHY_CAL_MODE_0, ddrtbl_getval(_cnf_DDR_PHY_ADR_G_REGSET, _reg_PHY_CAL_MODE_0) | 0x02U);

	/* periodic dram zqcal enable */
	dbsc_write_32(DBSC_DBCALCNF, 0x01000010U);

	/* periodic phy ctrl update enable */
	dbsc_write_32(DBSC_DBDFICUPDCNF, 0x504C0001U);

#ifdef DDR_BACKUPMODE
	if (ddrBackup == DRAM_BOOT_STATUS_WARM) {
		/* SRX */
		send_dbcmd(0x0A840001U);
	}
#endif /* DDR_BACKUPMODE */
	/* set Auto Refresh */
	dbsc_write_32(DBSC_DBRFEN, 0x00000001U);

#if RCAR_REWT_TRAINING != 0
	/* Periodic WriteDQ Training */
	clk_count = 1024U - ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_LONG_COUNT_MASK) * 32U;
	ctrl_clk = ddr_mbps / ddr_mbpsdiv / 4U;
	dataL = clk_count * (1000U * 1000U * 1000U / (1000U * ctrl_clk));

	phymster_req_interval = REWT_TRAINING_INTERVAL - 3000;

	ddr_setval_ach(_reg_PI_WDQLVL_INTERVAL, phymster_req_interval * 1000U * 100U / (dataL / 10U) );
#endif /* RCAR_REWT_TRAINING */

	/* dram access enable */
	dbsc_write_32(DBSC_DBACEN, 0x00000001U);

	/* MR13: vrcg(normal mode) */
	send_dbcmd(0x0e840dc0U);

	MSG_LF("dbsc_regset_post(done)\n");
}

/*******************************************************************************
 *	DFI_INIT_START
 ******************************************************************************/
static uint32_t dfi_init_start(void)
{
	uint32_t ch;
	uint32_t phytrainingok;
	uint32_t retry;
	uint32_t dataL;
	const uint32_t RETRY_MAX = 0x10000U;

	/***********************************************************************
	 * init start
	 ***********************************************************************/
	pll3_control(0U);	/* Low frequency mode */
	/* dbdficnt0:
	 * dfi_dram_clk_disable=1
	 * dfi_frequency = 0
	 * freq_ratio = 01 (2:1)
	 * init_start =0
	 */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F10U);
	}

	/* dbdficnt0:
	 * dfi_dram_clk_disable=1
	 * dfi_frequency = 0
	 * freq_ratio = 01 (2:1)
	 * init_start =1
	 */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBDFICNT(ch), 0x00000F11U);
	}

	wait_dbpdstat1(0x03U);

	/***********************************************************************
	 * dll rst
	 ***********************************************************************/
	/* dll_rst negate */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBPDCNT3(ch), 0x0000CF01U);
	}

	/***********************************************************************
	 * wait init_complete
	 ***********************************************************************/
	phytrainingok = 0U;
	retry = 0U;
	while (retry++ < RETRY_MAX) {
		foreach_vch(ch) {
			dataL = mmio_read_32(DBSC_DBDFISTAT(ch));
			if (dataL & 0x00000001U) {
				phytrainingok |= (1U << ch);
			}
		}
		if (phytrainingok == ddr_phyvalid) {
			break;
		}
	}

	/***********************************************************************
	 * all ch ok?
	 ***********************************************************************/
	if ((phytrainingok & ddr_phyvalid) != ddr_phyvalid) {
		return (phytrainingok);
	}
	/* dbdficnt0:
	 * dfi_dram_clk_disable=0
	 * dfi_frequency = 0
	 * freq_ratio = 01 (2:1)
	 * init_start =0
	 */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBDFICNT(ch), 0x00000010U);
	}

	return (phytrainingok);
}

/*******************************************************************************
 *	DDR mode register setting
 ******************************************************************************/
static void ddr_register_set(void)
{
	int32_t fspwp;
	uint32_t tmp;

	for (fspwp = 0U; fspwp < 2U; fspwp++) {
		/* MR13: fspop,fspwp */
		send_dbcmd(0x0e840d08U | ((2U - fspwp) << 6));

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR1_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840100U | tmp);

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR2_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840200U | tmp);

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR3_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840300U | tmp);

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR11_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840b00U | tmp);

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR12_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840c00U | tmp);

		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR14_DATA_Fx_CSx[fspwp][0]);
		send_dbcmd(0x0e840e00U | tmp);

		/* MR22 */
		tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR22_DATA_F0_0);
		send_dbcmd(0x0e801600U | tmp);

		if (ch_have_this_cs[1]) {
			tmp = ddrtbl_getval(_cnf_DDR_PI_REGSET, _reg_PI_MR22_DATA_F0_1);
			send_dbcmd(0x0e811600U | tmp);
		}

		/* ZQCAL start */
		send_dbcmd(0x0d84004fU);

		/* ZQLAT */
		send_dbcmd(0x0d840051U);
	}

	/* MR13, fspwp */
	send_dbcmd(0x0e840d08U);
}

/*******************************************************************************
 *	Training handshake functions
 ******************************************************************************/
static void wait_dbpdstat1(uint32_t status)
{
	uint32_t i, ch, dataL;

	for (i = 0U; i < 2U; i++) {
		do {
			dataL = status;
			foreach_vch(ch) {
				dataL &= mmio_read_32(DBSC_DBPDSTAT1(ch));
			}
		} while (dataL != status);
	}
}

static uint32_t wait_freqchgreq(uint32_t req_assert)
{
	uint32_t dataL;
	uint32_t count;
	uint32_t ch;

	count = 0xFFFFFFU;

	if (req_assert) {
		do {
			dataL = 1U;
			foreach_vch(ch) {
				dataL &= mmio_read_32(DBSC_DBPDSTAT(ch));
			}
			count = count - 1U;
		} while (((dataL & 0x01U) != 0x01U) & (count != 0U));
	} else {
		do {
			dataL = 0U;
			foreach_vch(ch) {
				dataL |= mmio_read_32(DBSC_DBPDSTAT(ch));
			}
			count = count - 1U;
		} while (((dataL & 0x01U) != 0x00U) & (count != 0U));
	}

	return (count == 0U);
}

static void set_freqchgack(uint32_t ackassert)
{
	uint32_t ch;
	uint32_t dataL;

	if (ackassert) {
		dataL = 0x0000CF01U;
	} else {
		dataL = 0x00000000U;
	}

	foreach_vch(ch) {
		mmio_write_32(DBSC_DBPDCNT2(ch), dataL);
	}
}

static void set_dfifrequency(uint32_t freq)
{
	uint32_t ch;

	foreach_vch(ch) {
		mmio_clrsetbits_32(DBSC_DBDFICNT(ch), 0x1fU << 24, (freq << 24));
	}
}

static uint32_t pll3_freq(uint32_t fsel)
{
	uint32_t timeout;

	wait_freqchgreq(1);

	if (fsel == 0U) {
		pll3_control(0);	/* Low frequency mode */
	} else {
		pll3_control(1);	/* High frequency mode */
	}

	set_dfifrequency(fsel);
	set_freqchgack(1);
	
	timeout = wait_freqchgreq(0);
	set_freqchgack(0);

	wait_dbpdstat1(0x03U);

	if (timeout) {
		FATAL_MSG("BL2: Time out[2]\n");
		return (1);
	}
	return (0);
}

/*******************************************************************************
 *	training by pi
 ******************************************************************************/
static uint32_t pi_training_go(void)
{
	uint32_t flag;
	uint32_t dataL;
	uint32_t retry;
	const uint32_t RETRY_MAX = 4096 * 16;
	uint32_t ch;

	uint32_t mst_ch;
	uint32_t cur_frq;
	uint32_t complete;
	uint32_t frqchg_req;

	/***********************************************************************
	 * pi_start
	 ***********************************************************************/
	foreach_vch(ch) {
		while((ddr_getval(ch, _reg_PI_INT_STATUS)) != 0U);
	}

	foreach_vch(ch) {
		mmio_write_32(DBSC_DBPDRGA(ch), DDR_REGDEF_ADR(_reg_PI_START));
		mmio_write_32(DBSC_DBPDRGD(ch), 0x00000B01U);
	}

	foreach_vch(ch) {
		while((mmio_read_32(DBSC_DBPDRGA(ch)) & 0x00008000U) == 0U);
		mmio_write_32(DBSC_DBPDRGA(ch), mmio_read_32(DBSC_DBPDRGA(ch)));
	}

	/* set dfi_phymstr_ack = 1 */
	dbsc_write_32(DBSC_DBDFIPMSTRCNF, 0x00000001U);

	/***********************************************************************
	 * wait pi_int_status[0]
	 ***********************************************************************/
	flag = 0U;
	complete = 0U;
	cur_frq = 0U;
	retry = RETRY_MAX;

	do {
		for (frqchg_req = 0U, ch = 0U; ch < DRAM_CH_CNT; ch++) {
			if (((~complete) & ddr_phyvalid) & (1U << ch)) {
				if (mmio_read_32(DBSC_DBPDSTAT(ch)) & 0x01U) {
					frqchg_req = 1U;
					mst_ch = ch;
					break;
				}
			}
		}

		if (frqchg_req) {
			cur_frq = (0x0300U & mmio_read_32(DBSC_DBPDSTAT(mst_ch))) >> 8;
			flag = pll3_freq(cur_frq);
			if (flag) {
				break;
			}
		} else {
			foreach_vch(ch) {
				if (complete & (1U << ch)) {
					continue;
				}
				mmio_write_32(DBSC_DBPDRGA(ch), DDR_REGDEF_ADR(_reg_PI_INT_STATUS) | 0x00004000U);
				if (0x00008000U & mmio_read_32(DBSC_DBPDRGA(ch))) {
					mmio_write_32(DBSC_DBPDRGA(ch), DDR_REGDEF_ADR(_reg_PI_INT_STATUS) | 0x00008000U);
					dataL = mmio_read_32(DBSC_DBPDRGD(ch));
					mmio_write_32(DBSC_DBPDRGA(ch), DDR_REGDEF_ADR(_reg_PI_INT_STATUS) | 0x00000000U);
					if (dataL & 0x01U) {
						complete |= (1U << ch);
					}
				}
			}
			if (complete == ddr_phyvalid) {
				break;
			}
		}
	} while (--retry);

	foreach_vch(ch) {
		/* dummy read */
		dataL = ddr_getval(ch, _reg_PI_START);
		dataL = ddr_getval(ch, _reg_PI_INT_STATUS);
		ddr_setval(ch, _reg_PI_INT_ACK, dataL);
	}

	return complete;
}

static void manual_frequency_change(void)
{
	uint32_t ch;

	/* FSP-OP:1 FSP-WR:1 VRCG:1 */
	send_dbcmd(0x0e840dc8U);

	/* PDE */
	send_dbcmd(0x08840000U);

	/* init start ass */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBDFICNT(ch), 0x01000011U); 
	}

	/* wait init comp neg */
	foreach_vch(ch) {
		while((mmio_read_32(DBSC_DBDFISTAT(ch)) & 0x00000001U) != 0U);
	}

	pll3_control(1);

	/* init start neg */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBDFICNT(ch), 0x01000010U);
	}

	/* wait init comp ass */
	foreach_vch(ch) {
		while((mmio_read_32(DBSC_DBDFISTAT(ch)) & 0x00000001U) != 1U);
	}

	/* PDX */
	send_dbcmd(0x08840001U);

	/* set MR13 for FSP */
	ddr_setval_ach(_reg_PI_MR13_DATA_0, 0xc8U);
	ddr_setval_ach(_reg_PI_MR13_DATA_1, 0xc8U);
}

static uint32_t manual_training_wrlvl(void)
{
	uint32_t dataL;
	uint32_t ch, cs, rank;
	uint32_t wr_training_ng;

	if (ch_have_this_cs[1] != 0) {
		rank = 2;
	} else {
		rank = 1;
	}

	/* WR LVL */
	wr_training_ng = 0x00U;

	for(cs = 0; rank > cs; cs++) {
		ddr_setval_ach(_reg_PI_WRLVL_CS, cs);
		ddr_setval_ach(_reg_PI_WRLVL_REQ, 0x1U);

		foreach_vch(ch) {
			while(1) {
				dataL = ddr_getval(ch, _reg_PI_INT_STATUS);
				if(dataL & 0x00001000U) break;
			}
		}
		foreach_vch(ch) {
			if (ddr_getval(ch, _reg_PHY_WRLVL_STATUS_OBS) & ( 1U << 12 )) {	/* bit12 wrlvl_error */
				wr_training_ng |= 1U << ch;
			}
		}
		ddr_setval_ach(_reg_PI_INT_ACK, 0x3FFFFFU);
	}
	return ~wr_training_ng;
}

static uint32_t manual_training_rdgtlvl(void)
{
	uint32_t dataL;
	uint32_t ch, cs, rank;
	uint32_t gt_training_ng;

	if (ch_have_this_cs[1] != 0) {
		rank = 2;
	} else {
		rank = 1;
	}

	/* RD GATE LVL */
	gt_training_ng = 0x00U;

	for(cs = 0; rank > cs; cs++) {
		ddr_setval_ach(_reg_PI_RDLVL_CS, cs);
		ddr_setval_ach(_reg_PI_RDLVL_GATE_REQ, 0x1U);

		foreach_vch(ch) {
			while(1) {
				dataL = ddr_getval(ch, _reg_PI_INT_STATUS);
				if(dataL & 0x00001000U) break;
			}
		}
		foreach_vch(ch) {
			if (ddr_getval(ch, _reg_PHY_GTLVL_STATUS_OBS) & ( 3U << 6 )) {	/* bit6 Minimum delay setup error, bit7 Maximum delay setup error */
				gt_training_ng |= 1U << ch;
			}
		}
		ddr_setval_ach(_reg_PI_INT_ACK, 0x3FFFFFU);
	}
	return ~gt_training_ng;
}

static uint32_t manual_training_rdlvl(void)
{
	uint32_t dataL;
	uint32_t ch, cs, rank;
	uint32_t rd_training_ng;

	if (ch_have_this_cs[1] != 0) {
		rank = 2;
	} else {
		rank = 1;
	}

	/* RD LVL */
	rd_training_ng = 0x00U;

	for(cs = 0; rank > cs; cs++) {
		ddr_setval_ach(_reg_PI_RDLVL_CS, cs);
		ddr_setval_ach(_reg_PI_RDLVL_REQ, 0x1U);

		foreach_vch(ch) {
			while(1) {
				dataL = ddr_getval(ch, _reg_PI_INT_STATUS);
				if(dataL & 0x00001000U) break;
			}
		}
		foreach_vch(ch) {
			if (ddr_getval(ch, _reg_PI_INT_STATUS) & ( 1U << 1 )) {	/* bit1 PI_RDLVL_ERROR_BIT */
				rd_training_ng |= 1U << ch;
			}
		}
		ddr_setval_ach(_reg_PI_INT_ACK, 0x3FFFFFU);
	}
	return ~rd_training_ng;
}

static uint32_t manual_training_wdqlvl(void)
{
	uint32_t dataL;
	uint32_t ch, cs, rank;
	uint32_t wdq_training_ng;

	if (ch_have_this_cs[1] != 0) {
		rank = 2;
	} else {
		rank = 1;
	}

	/* WDQ LVL */
	wdq_training_ng = 0x00U;

	for(cs = 0; rank > cs; cs++) {
		ddr_setval_ach(_reg_PI_WDQLVL_CS, cs);
		ddr_setval_ach(_reg_PI_WDQLVL_REQ, 0x1U);

		foreach_vch(ch) {
			while(1) {
				dataL = ddr_getval(ch, _reg_PI_INT_STATUS);
				if(dataL & 0x00001000U) break;
			}
		}
		foreach_vch(ch) {
			if (ddr_getval(ch, _reg_PI_INT_STATUS) & ( 1U << 5 )) {	/* bit5 PI_WDQLVL_ERROR_BIT */
				wdq_training_ng |= 1U << ch;
			}
		}
		ddr_setval_ach(_reg_PI_INT_ACK, 0x3FFFFFU);
	}
	return ~wdq_training_ng;
}

/***********************************************************************
 * CA Vref Training
 ***********************************************************************/
static uint32_t ca_vref_training(void)
{
	uint32_t dvw, dvw_tmp[DRAM_CH_CNT][CS_CNT];
	uint32_t cavref, cavref_set_min[DRAM_CH_CNT][CS_CNT], cavref_set_max[DRAM_CH_CNT][CS_CNT];
	uint32_t ch, cs, rank;
	uint32_t i, dataL, tmp;
	uint32_t vref_start, vref_end;
	uint32_t ca_training_ng;

	foreach_ech(ch) {
		for (cs = 0; CS_CNT > cs; cs++) {
			dvw_tmp[ch][cs] = 0x0000U;
			cavref_set_min[ch][cs] = 0x0000U;
			cavref_set_max[ch][cs] = 0x0000U;
		}
	}

	/***********************************************************************
	 * CA Vref (MR12) Training range
	 ***********************************************************************/
	dataL = (uint32_t)(Boardcnf->vref_ca);
	if (dataL) {
		vref_start = dataL & 0x00ffU;
		vref_end = (dataL & 0xff00U) >> 8;
	} else {
		vref_start = 0x000eU;
		vref_end = 0x0014U;
	}

	/***********************************************************************
	 *  CA training setting
	 ***********************************************************************/
	ddr_setval_ach(_reg_PI_MR13_DATA_0, 0xc8U);
	ddr_setval_ach(_reg_PI_MR13_DATA_1, 0xc8U);

	ddr_setval_ach(_reg_PI_CA_TRAIN_VREF_EN, 0x0001U);

	if (ch_have_this_cs[1] != 0) {
		rank = 2;
	} else {
		rank = 1;
	}

	/***********************************************************************
	 * Search Best CA VREF
	 ***********************************************************************/
	for(cs =0; rank > cs; cs++) {
		for (cavref = vref_start; cavref <= vref_end; cavref += DDR_CAVREF_DELTA) {
			ddr_setval_ach(_reg_PI_CALVL_CS, cs);
			if (cs) {
				ddr_setval_ach(_reg_PI_MR12_DATA_F1_1, cavref);
			} else {
				ddr_setval_ach(_reg_PI_MR12_DATA_F1_0, cavref);
			}
			ddr_setval_ach(_reg_PI_CALVL_REQ, 0x0001U);

			dataL = 0x00U;
			while(dataL == 0x00U) {
				dataL = 0x01U << 19;
				foreach_vch(ch) {
					dataL &= (ddr_getval(ch, _reg_PI_INT_STATUS));
				}
			}

			foreach_vch(ch) {
				if((0x00000010U & ddr_getval(ch, _reg_PI_INT_STATUS)) == 0x00000000U) {
					dvw = 0x0000ffffU;
					for (i = 0; i < 6; i++){
						ddr_setval(ch, _reg_PHY_ADR_CALVL_OBS_SELECT_0, i);
						dataL = ddr_getval(ch, _reg_PHY_ADR_CALVL_CH0_OBS0_0);
						tmp = (dataL & 0x000007ffU) - ((dataL >> 16) & 0x000007ffU);
						if(dvw > tmp) {
							dvw = tmp;
						}
					}

					if(dvw_tmp[ch][cs] < dvw) {
						dvw_tmp[ch][cs] = dvw;
						cavref_set_min[ch][cs] = cavref;
						cavref_set_max[ch][cs] = cavref;
					} else if (dvw_tmp[ch][cs] == dvw) {
						cavref_set_max[ch][cs] = cavref;
					}
				}
				ddr_setval(ch, _reg_PI_INT_ACK, ddr_getval(ch, _reg_PI_INT_STATUS));
			}
		}
	}

	/***********************************************************************
	 * Use the Best CA VREF to do CA training
	 ***********************************************************************/
	if (rank == 2) {
		ddr_setval_ach(_reg_PHY_ADR_CALVL_RANK_CTRL_0, 0x03U);
	}

	foreach_vch(ch) {
		ddr_setval(ch, _reg_PI_MR12_DATA_F1_0, (cavref_set_min[ch][0] + cavref_set_max[ch][0]) / 2);
		ddr_setval(ch, _reg_PI_MR12_DATA_F1_1, (cavref_set_min[ch][rank - 1] + cavref_set_max[ch][rank - 1]) / 2);
	}

	ca_training_ng = 0x00U;
	for(cs = 0; rank > cs; cs++) {
		ddr_setval_ach(_reg_PI_CALVL_CS, cs);
		ddr_setval_ach(_reg_PI_CALVL_REQ, 0x0001U);

		dataL = 0x00U;
		while(dataL == 0x00U) {
			dataL = 0x01U << 19;
			foreach_vch(ch) {
				dataL &= (ddr_getval(ch, _reg_PI_INT_STATUS));
			}
		}

		foreach_vch(ch) {
			if (ddr_getval(ch, _reg_PHY_CSLVL_OBS1) & 0xF0000000U) {
				ca_training_ng |= 1U << ch;
			}
			ddr_setval(ch, _reg_PI_INT_ACK, ddr_getval(ch, _reg_PI_INT_STATUS));
		}
	}

	ddr_setval_ach(_reg_SC_PHY_MANUAL_UPDATE, 0x01U);
	ddr_setval_ach(_reg_PHY_ADRCTL_MANUAL_UPDATE, 0x01U);

	return ~ca_training_ng;
}

/*******************************************************************************
 * DSKEWCALLOCK status check
 ******************************************************************************/
static uint32_t dskewcallock_chk(void)
{
	uint32_t ch;
	uint32_t dataL;
	uint32_t dskewcallock_ok;

	ddr_setval_ach(_reg_SC_PHY_PLL_SPO_CAL_SNAP_OBS, 0x01U);

	dskewcallock_ok = 0U;
	foreach_ech(ch) {
		dataL =  0x01U & (ddr_getval(ch, _reg_PHY_PLL_SPO_CAL_OBS_0) >> 16);
		dataL &= 0x01U & (ddr_getval(ch, _reg_PHY_PLL_SPO_CAL_OBS_1) >> 16);
		dskewcallock_ok |= dataL << ch;
	}

	return dskewcallock_ok;
}

/*******************************************************************************
 *	Initialize ddr
 ******************************************************************************/
static uint32_t init_ddr(void)
{
	uint32_t phytrainingok;
	uint32_t ch;
	uint32_t err;

	MSG_LF("init_ddr:0\n");
#ifdef DDR_BACKUPMODE
	dram_get_boot_status(&ddrBackup);
#endif

	/***********************************************************************
	 * PLL3 initialization setting
	 ***********************************************************************/
	/* Reset Status Monitor clear */
	cpg_write_32(CPG_FSRCHKCLRR4, 0x00000200U);
	/* Reset Status Monitor set */
	cpg_write_32(CPG_FSRCHKSETR4, 0x00000200U);
	/* ddrphy soft reset assert */
	cpg_write_32(CPG_SRST4, mmio_read_32(CPG_SRST4) | 0x00000200U);
	/* wait reset FB */
	while((mmio_read_32(CPG_FSRCHKRA4) & 0x00000200U) != 0U);
	/* Reset Status Monitor clear */
	cpg_write_32(CPG_FSRCHKCLRR4, 0x00000200U);

	pll3_control(1);	/* Initialize PLL3 setting */

	/* ddrphy soft reset negate */
	cpg_write_32(CPG_SRSTCLR4, 0x00000200U);
	while((mmio_read_32(CPG_SRST4) & 0x00000200U) != 0U);

	/***********************************************************************
	 * unlock phy
	 ***********************************************************************/
	/* Unlock DDRPHY register */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBPDLK(ch), 0x0000A55AU);
	}

	/***********************************************************************
	 * dbsc register pre-setting
	 ***********************************************************************/
	dbsc_regset_pre();

	/***********************************************************************
	 * load ddrphy registers
	 ***********************************************************************/
	ddrtbl_load();

	/***********************************************************************
	 * configure ddrphy registers
	 ***********************************************************************/
	ddr_config();

	/***********************************************************************
	 * ddr backupmode end
	 ***********************************************************************/
#ifdef DDR_BACKUPMODE
	if (ddrBackup) {
		NOTICE("BL2: [WARM_BOOT]\n");
	} else {
		NOTICE("BL2: [COLD_BOOT]\n");
	}

	err = dram_update_boot_status(ddrBackup);
	if (err) {
		NOTICE("BL2: [BOOT_STATUS_UPDATE_ERROR]\n");
		return INITDRAM_ERR_I;
	}
#endif
	MSG_LF("init_ddr:1\n");

	/***********************************************************************
	 * dfi_init_start (start ddrphy)
	 ***********************************************************************/
	phytrainingok = dfi_init_start();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:1\n");
		return (phytrainingok);
	}

	MSG_LF("init_ddr:2\n");

	/***********************************************************************
	 * load pi registers
	 ***********************************************************************/
	ddrtbl_load_pi();

	/***********************************************************************
	 * dbsc register set
	 ***********************************************************************/
	dbsc_regset();
	MSG_LF("init_ddr:3\n");

	/***********************************************************************
	 * setup DDR mode registers
	 ***********************************************************************/
	/* Dummy PDE */
	send_dbcmd(0x08840000U);

	/* PDX */
	send_dbcmd(0x08840001U);

	/* MRS */
	ddr_register_set();

	MSG_LF("init_ddr:4\n");

	/***********************************************************************
	 * exec pi_training
	 ***********************************************************************/
	phytrainingok &= pi_training_go();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:2\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * exec frequency change
	 ***********************************************************************/
	manual_frequency_change();

	/***********************************************************************
	 * CA Vref Training
	 ***********************************************************************/
	phytrainingok &= ca_vref_training();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:3\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * exec wr_training
	 ***********************************************************************/
	phytrainingok &= manual_training_wrlvl();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:4\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * exec rdgt_training
	 ***********************************************************************/
	phytrainingok &= manual_training_rdgtlvl();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:5\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * exec rd_training
	 ***********************************************************************/
	phytrainingok &= manual_training_rdlvl();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:6\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * exec wdq_training
	 ***********************************************************************/
	phytrainingok &= manual_training_wdqlvl();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:7\n");
		return (phytrainingok);
	}

	MSG_LF("init_ddr:5\n");

	/***********************************************************************
	 * FREQ_SEL_MULTICAST & PER_CS_TRAINING_MULTICAST SET (disable)
	 ***********************************************************************/
	ddr_setval_ach_as(_reg_PHY_PER_CS_TRAINING_MULTICAST_EN, 0x00U);
	ddr_setval_ach(_reg_PHY_FREQ_SEL_MULTICAST_EN, 0x00U);

	MSG_LF("init_ddr:6\n");

	/***********************************************************************
	 * DSKEWCALLOCK status check
	 ***********************************************************************/
	phytrainingok &= dskewcallock_chk();

	if (ddr_phyvalid != phytrainingok) {
		MSG_LF("init_ddr_error:8\n");
		return (phytrainingok);
	}

	/***********************************************************************
	 * training complete, setup dbsc
	 ***********************************************************************/
	dbsc_regset_post();
	MSG_LF("init_ddr:7\n");

	/***********************************************************************
	 * lock phy
	 ***********************************************************************/
	/* Lock DDRPHY register */
	foreach_vch(ch) {
		mmio_write_32(DBSC_DBPDLK(ch), 0x00000000U);
	}

	return phytrainingok;
}

/*******************************************************************************
 *	DDR Initialize entry
 ******************************************************************************/
int32_t InitDram(void)
{
	uint32_t ch, cs;
	uint32_t dataL;
	uint32_t failcount;

	dbsc_write_32(DBSC_DBSYSCNT0, 0x00001234U);
	dbsc_write_32(DBSC_DBSYSCNT0A, 0x00001234U);

	/***********************************************************************
	 * Judge product and cut
	 ***********************************************************************/
/*
	Prr_Product = mmio_read_32(PRR) & PRR_PRODUCT_MASK;
	Prr_Cut = mmio_read_32(PRR) & PRR_CUT_MASK;
*/
	/***********************************************************************
	 * Judge board type
	 ***********************************************************************/
	_cnf_BOARDTYPE = boardcnf_get_brd_type();
	if (_cnf_BOARDTYPE >= BOARDNUM) {
		FATAL_MSG("BL2: DDR:Unknown Board\n");
		return 0xffU;
	}
	Boardcnf = (const struct _boardcnf *)&boardcnfs[_cnf_BOARDTYPE];

	ddr_phyvalid = (uint32_t)(Boardcnf->phyvalid);

	max_density = 0U;

	for (cs = 0U; cs < CS_CNT; cs++) {
		ch_have_this_cs[cs] = 0U;
	}

	foreach_ech(ch) {
		for (cs = 0U; cs < CS_CNT; cs++) {
			ddr_density[ch][cs] = 0xffU;
		}
	}

	foreach_vch(ch) {
		for (cs = 0U; cs < CS_CNT; cs++) {
			dataL = (uint32_t)(Boardcnf->ch[ch].ddr_density[cs]);
			ddr_density[ch][cs] = dataL;
			if (dataL == 0xffU) {
				continue;
			}
			if (dataL > max_density) {
				max_density = dataL;
			}
			ch_have_this_cs[cs] |= (1U << ch);
		}
	}

	/***********************************************************************
	 * Judge board clock frequency (in MHz)
	 ***********************************************************************/
	boardcnf_get_brd_clk(_cnf_BOARDTYPE, &brd_clk, &brd_clkdiv);
	if ((0x01U << 14) & mmio_read_32(RST_MODEMR0)) {
		brd_clkdiva = 1U;
	} else {
		brd_clkdiva = 0U;
	}

	/***********************************************************************
	 * Judge ddr operating frequency clock(in Mbps)
	 ***********************************************************************/
	boardcnf_get_ddr_mbps(_cnf_BOARDTYPE, &ddr_mbps, &ddr_mbpsdiv);

	ddr_mul = CLK_DIV(ddr_mbps, ddr_mbpsdiv, brd_clk, brd_clkdiv * (brd_clkdiva + 1U));
	ddr_mul_nf = (16U * ddr_mbps * (brd_clkdiv * (brd_clkdiva + 1U))) / (ddr_mbpsdiv * brd_clk) - (16U * ddr_mul);
	ddr_mul_low = CLK_DIV(1600U, 1U, brd_clk, brd_clkdiv * (brd_clkdiva + 1U));
	if((ddr_mul_low * brd_clk / (brd_clkdiv * (brd_clkdiva + 1U))) != 1600U) {
		ddr_mul_low += 1;
	}

	dataL = 0x03U & (mmio_read_32(RST_MODEMR0) >> 13);
	bus_mbps = 0U;
	bus_mbpsdiv = 0U;

	switch (dataL) {
	case 0U:
		bus_mbps = brd_clk * 0x32U;
		bus_mbpsdiv = brd_clkdiv;
		break;
	case 1U:
		bus_mbps = brd_clk * 0x28U;
		bus_mbpsdiv = brd_clkdiv;
		break;
	case 2U:
		/* Not supported */
		bus_mbps = brd_clk * 0x40U;
		bus_mbpsdiv = brd_clkdiv * 2U;
		break;
	case 3U:
		bus_mbps = brd_clk * 0x28U;
		bus_mbpsdiv = brd_clkdiv * 2U;
		break;
	default:
		bus_mbps = brd_clk * 0x28U;
		bus_mbpsdiv = brd_clkdiv * 2U;
		break;
	}

	/***********************************************************************
	 * Adjust tccd
	 ***********************************************************************/
	ddr_tccd = 8U;

	NOTICE("BL2: DDR%d(%s)", ddr_mbps / ddr_mbpsdiv, RCAR_DDR_VERSION);

	MSG_LF("Start\n");

	/***********************************************************************
	 * initialize DDR
	 ***********************************************************************/
	dataL = init_ddr();
	if (dataL == ddr_phyvalid) {
		failcount = 0U;
	} else {
		failcount = 1U;
	}

	NOTICE("..%d\n", failcount);

	dbsc_write_32(DBSC_DBSYSCNT0, 0x00000000U);
	dbsc_write_32(DBSC_DBSYSCNT0A, 0x00000000U);

	if (failcount == 0U) {
		return INITDRAM_OK;
	} else {
		return INITDRAM_NG;
	}
}

/*******************************************************************************
 * END
 ******************************************************************************/
