/*******************************************************************************

NAME			$RCSfile: ssNTB.c,v $
SUMMARY			Board Stress for PLX Non-Transparent Bridge
VERSION			$Revision: 1.16 $
UPDATE DATE		$Date: 2010/06/29 11:59:01 $
PROGRAMMER		$Author: alan $

		Copyright 2009 LSI Corporation. All Rights Reserved.

DESCRIPTION:

REFERENCE:

*******************************************************************************/
#include <linux/version.h>
#include <linux/kthread.h>
#include <linux/wait.h>
#include <linux/pci.h>
#include <linux/vmalloc.h>
#include <linux/async_tx.h>
#include <linux/delay.h>
#include <linux/spinlock.h>
#include <linux/dmaengine.h>
#include <drivers/dma/ioat/dma.h>
#include <drivers/dma/ioat/dma_v2.h>
#include <drivers/dma/ioat/registers.h>
#include <drivers/dma/ioat/hw.h>
#include <linux/kobject.h>

#if defined(CHASSIS_WSAS)
#include <linux/sched.h>
#endif

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/errno.h>
#include <asm/atomic.h>
#include <asm/processor.h>
#include <asm/uaccess.h>
#include <linux/file.h>
#include <linux/time.h>

//#include <commPattern.h>
//#include <dmaLibP.h>
//#include <dmaStruct.h>
#include <commRpt.h>
#include <commMEC.h>
//#include <memLib.h>
#include <commTypes.h>
#include <commDebug.h>
//#include <dmaLib.h>

#include "plxDiag.h"

#if defined(CHASSIS_WSAS)
#include "wemblySAS.h"
#endif

//typedef unsigned long phys_addr_t;

/* FIXME: CB3 dma compare is not ready */
#define NO_DMA_COMPARE
#define TOTALSLEEP

#define SSPRINTK(a...)
//#define SSPRINTK(a, args...)	if(a) printk(args)

//extern void plxMemShow(uint32_t addrUpper, uint32_t addrLower, int size);

//PUBLIC BOOLEAN dmaMemCompare(UINT64 addrPhySrc, UINT64 addrPhyDst, UINT32 len);
extern void plxIpcExit(void);
extern int plxPeerIpcExit(void);
//#ifdef PLX_NT_BACK_TO_BACK
extern void plxNTBSetTransAddress(unsigned long long address);
//#endif
extern int raw_pci_write(unsigned int domain, unsigned int bus,	unsigned int devfn, int reg, int len, u32 val);

/******************************************************
 *      Testing Memory Layout of two controllers
 *
 *              _____________               _____________      
 *             |                           |           
 *             | Kernel Memory             | Kernel Memory
 *             |_____________              |_____________
 *             |                           |
 *             |            (Local BAR2/3) |
 * LocalAddr ->|____________    TransAddr->|______________
 *             |                           |
 *             | Local Memory              | Remote Memory     
 *             |_____________              |______________
 *             |                           |
 *             |                           |
 *
 ******************************************************/

struct ssNTB_TEST_CB {
	struct task_struct *task;
	phys_addr_t	localAddr;	/* Local start physical address */
	phys_addr_t	remoteAddr;	/* Translation peer physical address */
	uint32_t	maxSize;	/* Testing Max range */
	uint32_t	rangeSize;	/* Testing range */
	uint32_t	blockSize;	/* IO unit */
	uint32_t	iterations;
	uint32_t	flags;		/* option flags */
	uint32_t	pattern;	/* pattern assigned */
	uint32_t	ErrCnt;	/* Error Count */
	uint32_t	client;	/* client may be PCU or DMA */
};

#define STOP_ON_ERROR		BIT(16)
#define Data_Verification	BIT(17)
#define SS_PERFPERIOD	20
#define SS_SLEEPTIME	1	// mini seconds

extern int ssCPU_Cores;
#define DMA_ChCnt	8
static struct ssNTB_TEST_CB ssNTBCPU_Cb[4];
static struct ssNTB_TEST_CB ssNTBDMA_Cb[DMA_ChCnt];
static struct ssNTB_TEST_CB ssDMA_Cb[DMA_ChCnt];

static __iomem void *ssNTB_MMIO = NULL;
static PLX_PCIE_SWT ssNTB_SWT;

uint32_t ssNTB_PatternCnt=0;
uint32_t ssNTB_Patterns[16];
uint32_t ssNTB_NoVerify=0;
//#define	BIT(i)	(1<<(i))

//uint32_t ssNTBCPU_TestMap=0;
//uint32_t ssNTBDMA_TestMap=0;
//uint32_t ssDMA_TestMap=0;
//#define	ssNTB_TestMapAdd(a, n)	{sslock_NTB_RunMap(); a |= BIT(n); ssunlock_NTB_RunMap();}
//#define	ssNTBCPU_TestMapDel(a, n) {sslock_ParamRW(); sslock_CommonNTBCPU(); a &= ~BIT(n);ssunlock_CommonNTBCPU(); ssunlock_ParamRW();}
//#define	ssNTBCPU_TestMapDel(n)	{sslock_CommonNTBCPU(); ssNTBCPU_TestMap &= ~BIT(n); ssunlock_CommonNTBCPU();}
//#define	ssNTBDMA_TestMapDel(n)	{sslock_CommonNTBDMA(); ssNTBDMA_TestMap &= ~BIT(n); ssunlock_CommonNTBDMA();}
//#define	ssDMA_TestMapDel(n)	{sslock_CommonDMA();    ssDMA_TestMap    &= ~BIT(n); ssunlock_CommonDMA();   }
//#define	ssNTBCPU_TestMapAdd(n)	{sslock_CommonNTBCPU(); ssNTBCPU_TestMap |= BIT(n); ssunlock_CommonNTBCPU();}
//#define	ssNTBDMA_TestMapAdd(n)	{sslock_CommonNTBDMA(); ssNTBDMA_TestMap |= BIT(n); ssunlock_CommonNTBDMA();}
//#define	ssDMA_TestMapAdd(n)	{sslock_CommonDMA();    ssDMA_TestMap    |= BIT(n); ssunlock_CommonDMA();   }

uint32_t ssNTB_VirtWinSize, ssNTB_LinkWinSize;
uint64_t ssNTB_VirtMapBase, ssNTB_LinkMapBase;
uint64_t ssDMA_MapBase=0;

static int	alreadyInit=0;
extern int plxEepromProgramSettings(const char *string);
extern struct cpuinfo_x86	boot_cpu_data;
extern unsigned long get_seconds(void);
extern void sslock_CommonNTBCPU(void);
extern void ssunlock_CommonNTBCPU(void);
extern void sslock_CommonNTBDMA(void);
extern void ssunlock_CommonNTBDMA(void);
extern void sslock_CommonDMA(void);
extern void ssunlock_CommonDMA(void);
extern void sslock_ParamRW(void);
extern void ssunlock_ParamRW(void);

#define SSPLX_NT_MAP_BASE       0x800000000ULL /* 32 GB */
//#define SSPLX_NT_WINDOW_SIZE     0x20000000ULL  /*  512 MB */
#define SSPLX_NT_WINDOW_SIZE     0x10000000ULL  /*  256 MB */  /* Modified by Luap 20091216 */
//#define PLX_NT_TRANS_ADDR      0x20000000ULL  /*  512 MB */
//#define PLX_NT_TRANS_ADDR      0x60000000ULL  /* 1536 MB */ /* Modified by Luap 20091216 */
#define SSPLX_NT_TRANS_ADDR      0x48000000ULL  /* 1152 MB */ /* Modified by Alan Wang 20091216 */
#define SSPLX_NT_BUFFER_SIZE     0x10000000ULL  /*  256 MB */

#define MAPAreaSize	0x10000000ULL  /* 256 MB */
uint32_t	ssNTB_MaxRange=0;
uint32_t	ssDMA_MaxRange=0;
	
uint32_t	ssNTBCPU_TestErrCnt=0;
uint32_t	ssNTBDMA_TestErrCnt=0;
uint32_t	ssDMA_TestErrCnt=0;
char	*nccname="/tmp/NTB/ssNTBCPU_CR.log";
char	*ncpname="/tmp/NTB/ssNTBCPU_PR.log";
char	*ndcname="/tmp/NTB/ssNTBDMA_CR.log";
char	*ndpname="/tmp/NTB/ssNTBDMA_PR.log";
char	*mdcname="/tmp/MEM/ssMemDMA_CR.log";
char	*mdpname="/tmp/MEM/ssMemDMA_PR.log";
#define ssNTB_Output(fn, args...) {	\
	char	buf[128];		\
	sprintf(buf, args);		\
	ssNTB_WrFile(fn, buf);		\
	}
//	sslock_NTB_RunMap();	
//	ssunlock_NTB_RunMap();	

#define ssNTBCPU_CROutput(args...)	ssNTB_Output(nccname, args)
#define ssNTBCPU_PROutput(args...)	ssNTB_Output(ncpname, args)
#define ssNTBDMA_CROutput(args...)	ssNTB_Output(ndcname, args)
#define ssNTBDMA_PROutput(args...)	ssNTB_Output(ndpname, args)
#define ssDMA_CROutput(args...)		ssNTB_Output(mdcname, args)
#define ssDMA_PROutput(args...)		ssNTB_Output(mdpname, args)

#define PLX_VIRT_MMIO_WRITE(reg, value) \
{ \
	PLX_PCIE_SWT *swt = &ssNTB_SWT; \
	switch (swt->deviceID) \
	{ \
		case 0x8648: \
			writel(value, ssNTB_MMIO + (64*1024) + reg); \
			break; \
		case 0x8649: \
			writel(value, ssNTB_MMIO + (248*1024) + reg); \
			break; \
		default: \
			printk("Unsupported PLX device %04x\n", swt->deviceID); \
			break; \
	} \
}

#define PLX_VIRT_MMIO_READ(reg) \
({ \
	PLX_PCIE_SWT *swt = &ssNTB_SWT; \
	uint32_t ret; \
	switch (swt->deviceID) \
	{ \
		case 0x8648: \
			ret = readl(ssNTB_MMIO + (64*1024) + reg); \
			break; \
		case 0x8649: \
			ret = readl(ssNTB_MMIO + (248*1024) + reg); \
			break; \
		default: \
			printk("Unsupported PLX device %04x\n", swt->deviceID); \
			ret = 0xffffffff; \
			break; \
	} \
	ret; \
})

#define PLX_LINK_MMIO_WRITE(reg, value) \
{ \
	PLX_PCIE_SWT *swt = &ssNTB_SWT; \
	switch (swt->deviceID) \
	{ \
		case 0x8648: \
			writel(value, ssNTB_MMIO + (68*1024) + reg); \
			break; \
		case 0x8649: \
			writel(value, ssNTB_MMIO + (252*1024) + reg); \
			break; \
		default: \
			printk("Unsupported PLX device %04x\n", swt->deviceID); \
			break; \
	} \
}

#define PLX_LINK_MMIO_READ(reg) \
({ \
	PLX_PCIE_SWT *swt = &ssNTB_SWT; \
	uint32_t ret; \
	switch (swt->deviceID) \
	{ \
		case 0x8648: \
			ret = readl(ssNTB_MMIO + (68*1024) + reg); \
			break; \
		case 0x8649: \
			ret = readl(ssNTB_MMIO + (252*1024) + reg); \
			break; \
		default: \
			printk("Unsupported PLX device %04x\n", swt->deviceID); \
 			ret = 0xffffffff; \
			break; \
	} \
	ret; \
})
#if 0
static int ssNTB_ReadPatterns(void)
{
	struct file *filp;
	int r;
	mm_segment_t oldfs;

	oldfs=get_fs();

	set_fs(KERNEL_DS);

	filp=filp_open("/tmp/ssPatterns", O_RDONLY, 0);
	if(IS_ERR(filp))
        {
		printk("File Open Error:%s\n",filename);
		return 1;
        }
	filp->f_pos=0;
	
	r=filp->f_op->read(filp,buf,strlen(buf),&filp->f_pos);
	ssNTB_Patterns[i]=
	//printk("\nWrite :%xh bytes Offset=%xh\n",r,(unsigned)filp->f_pos);

	filp_close(filp,NULL);

	set_fs(oldfs);

	return 0;
}
#endif
static int ssNTB_WrFile(char *filename, char *buf)
{
	//char buf[]="This is a kernel-mode file operation testing program";
	struct file *filp;
	int r;
	mm_segment_t oldfs;

	oldfs=get_fs();

	set_fs(KERNEL_DS);

	filp=filp_open(filename,O_CREAT|O_RDWR|O_APPEND,0777);
	if(IS_ERR(filp))
        {
		printk("File Open Error:%s\n",filename);
		return 1;
        }

	//printk("Offset:%xh,buf=%s\n",(unsigned)filp->f_pos,buf);

	r=filp->f_op->write(filp,buf,strlen(buf),&filp->f_pos);

	//printk("\nWrite :%xh bytes Offset=%xh\n",r,(unsigned)filp->f_pos);

	filp_close(filp,NULL);

	set_fs(oldfs);

	return 0;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:     plxScanDevice
* SUMMARY:  Scan PLX device on the system
*
* SCOPE:    Private
*
* DESCRIPTION:
*	Find out all plx bridges on the system
*	get bridge hw configurations for diagnostics
*
* RETURNS:
*
* NOTES:
*
*/
#define SSNTB_PCIELCAP	0x74
#define SSNTB_PCIELSTS	0x78

static int ssNTB_ScanDevice(PLX_PCIE_SWT *swt)
{
	struct pci_dev *pdev = NULL;
	uint8_t portNum;
	uint32_t dwVal;
	uint16_t deviceID;
//	uint16_t wVal;sysStress tools
	int devFound = 0;

	while ((pdev = pci_get_device(0x10b5, PCI_ANY_ID, pdev)))
	{

		pci_read_config_word(pdev, PCI_DEVICE_ID, &deviceID);
		switch (deviceID)
		{
		case 0x8648:
			break;
		case 0x8649:
			break;
		default:
			continue;
		}

		pci_read_config_dword(pdev, SSNTB_PCIELCAP, &dwVal);
		portNum = dwVal >> 24;

		if (pdev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
			pdev->pcie_type == PCI_EXP_TYPE_ENDPOINT)
		{
			/* This maybe NT virtual or NT link devices.
			 * If we had device put in port slot then
			 * this is NT link device from peer controller.
			 * Otherwise put in port slot.
			 */
			if (swt->port[portNum].flag & PLX_PORT_USED)
				continue;
		}

		pci_read_config_dword(pdev, SSNTB_PCIELSTS, &dwVal);

		swt->port[portNum].pdev = pdev;
		swt->port[portNum].flag |= PLX_PORT_USED;
		swt->port[portNum].portNum = portNum;
		swt->port[portNum].laneWidth = (dwVal & 0x3f00000) >>20;
		swt->port[portNum].linkSpeed = (dwVal &   0xf0000) >>16;

		if (portNum == 0)
		{
			pci_read_config_word(pdev, PCI_DEVICE_ID, &swt->deviceID);
		}

		SSPRINTK(1, "PLX Port[%d], Landwidth x%d, LinkSpeed %sGb\n", portNum,
				swt->port[portNum].laneWidth, (swt->port[portNum].linkSpeed==1)?"2.5":"5");

		devFound++;
	}

	if (devFound == 0)
		return (-ENODEV);

	return (0);
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTB_Setup
* SUMMARY: Setup NTB virtual and link interface registers
*
* SCOPE:   Public
*
* DESCRIPTION:
*	To enable NTB, we need to setup registers of NT virtual (link) device
*	1. BAR Setup Register
*	2. NT Bridging Specific Regisgters - Requester ID Translation Lookup Table Entry
*	3. BAR Translation Register
* RETURNS:
*
* NOTES:
*
*/

int ssNTB_SetUp(void)
{
	PLX_PCIE_SWT *swt = &ssNTB_SWT;
	struct pci_dev *pdevUp; // , *pdevP0;
	uint32_t	bl, bu, mapSize; //pciCmdSts, , pn;
	uint32_t	reg; //, prefetch_limit, prefetch_limitH, mapBaseH;
	uint64_t	memBase=0; //
	int err;
	unsigned long long barWindow;

	memset(swt, 0, sizeof(PLX_PCIE_SWT));

	err = ssNTB_ScanDevice(swt);
	if (err)
	{
		commReportError(MEC_PCIE_SWITCH, PCIE_SWITCH_ITEM_CFG,
				PCIE_SWITCH_CODE_CFG, PCIE_SWITCH_MSG_ESCAN,
				0, 0, 0, DG_FORMAT_NONE);
		return (err);
	}

	/* mmio base at upstream port */
	switch (swt->deviceID)
	{
		case 0x8648:
			pdevUp = swt->port[4].pdev;
			mapSize = 128 * 1024;
			break;
		case 0x8649:
			pdevUp = swt->port[20].pdev;
			mapSize = 256 * 1024;
			break;
		default:
			printk("Unsupported PLX device %04x\n", swt->deviceID);
			return (-EINVAL);
	}
	pci_read_config_dword(pdevUp, 0x10, &bl);

	if (bl & 0x4)
	{
#ifdef CONFIG_PHYS_ADDR_T_64BIT
	/* 64-bit address */
	/* BAR1 */
		pci_read_config_dword(pdevUp, 0x14, &bu);
		memBase = bu;
		memBase = memBase << 32;
#else
		BUG();
#endif
	}

	memBase += (bl & 0xfff00000);

	/* map entire mmio size */
	ssNTB_MMIO = ioremap(memBase, mapSize);
	if (ssNTB_MMIO == NULL)
	{
		printk("ioremap failed for MMIO base\n");
		return -ENOMEM;
	}
	
	pci_read_config_dword(pdevUp, 0xa30, &reg);/* upstream Switch Debug */

	/*****************************************************
	 *               Virtual Interface
	 *****************************************************/
	barWindow = PLX_VIRT_MMIO_READ(0xd8);
	barWindow = (barWindow << 32) + PLX_VIRT_MMIO_READ(0xd4);
	ssNTB_VirtWinSize = ~(barWindow & ~0xfl)+1;
	
	/* Virt-side Address */
	ssNTB_VirtMapBase = PLX_VIRT_MMIO_READ(0xc40);
	ssNTB_VirtMapBase = (ssNTB_VirtMapBase << 32) + PLX_VIRT_MMIO_READ(0xc3c);
	
	barWindow = PLX_LINK_MMIO_READ(0xec);
	barWindow = (barWindow << 32) + PLX_LINK_MMIO_READ(0xe8);
	ssNTB_LinkWinSize = ~(barWindow & ~0xfl)+1;

	/* link-side Address */
	ssNTB_LinkMapBase = PLX_LINK_MMIO_READ(0xc40);
	ssNTB_LinkMapBase = (ssNTB_LinkMapBase<<32) + PLX_LINK_MMIO_READ(0xc3c);

	printk("[%s] PLX NTB Virt. Base=%lld(MB), Link Base=%lld(MB), Map Size=%d(MB)\n", __FUNCTION__,
		ssNTB_VirtMapBase/(1024*1024), ssNTB_LinkMapBase/(1024*1024), ssNTB_LinkWinSize/(1024*1024));

	return (0);
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTB_Setup
* SUMMARY: Setup NTB virtual and link interface registers
*
* SCOPE:   Public
*
* DESCRIPTION:
*	To enable NTB, we need to setup registers of NT virtual (link) device
*	1. BAR Setup Register
*	2. NT Bridging Specific Regisgters - Requester ID Translation Lookup Table Entry
*	3. BAR Translation Register
* RETURNS:
*
* NOTES:
*
*/

int ssNTB_ShowSetUp(void)
{
	PLX_PCIE_SWT *swt = &ssNTB_SWT;
	struct pci_dev *pdevUp; // , *pdevP0;
	uint32_t	pciCmdSts, bl, bu, mapSize, pn;
	uint32_t	reg; //, prefetch_limit, prefetch_limitH, mapBaseH;
	uint64_t	BAR_3_2, memBase=0; //
	int i,err;
	unsigned long long barWindow;
//	unsigned long long PREFETCH_LIMIT;

	memset(swt, 0, sizeof(PLX_PCIE_SWT));

	err = ssNTB_ScanDevice(swt);
	if (err)
	{
		commReportError(MEC_PCIE_SWITCH, PCIE_SWITCH_ITEM_CFG,
				PCIE_SWITCH_CODE_CFG, PCIE_SWITCH_MSG_ESCAN,
				0, 0, 0, DG_FORMAT_NONE);
		return (err);
	}

	/* mmio base at upstream port */
	switch (swt->deviceID)
	{
		case 0x8648:
			pdevUp = swt->port[4].pdev;
			mapSize = 128 * 1024;
			break;
		case 0x8649:
			pdevUp = swt->port[20].pdev;
			mapSize = 256 * 1024;
			break;
		default:
			printk("Unsupported PLX device %04x\n", swt->deviceID);
			return (-EINVAL);
	}
	pci_read_config_dword(pdevUp, 0x10, &bl);

	if (bl & 0x4)
	{
#ifdef CONFIG_PHYS_ADDR_T_64BIT
	/* 64-bit address */
	/* BAR1 */
		pci_read_config_dword(pdevUp, 0x14, &bu);
		memBase = bu;
		memBase = memBase << 32;
#else
		BUG();
#endif
	}

	memBase += (bl & 0xfff00000);

	/* map entire mmio size
	ssNTB_MMIO = ioremap(memBase, mapSize);
	if (ssNTB_MMIO == NULL)
	{
		printk("ioremap failed for MMIO base\n");
		return -ENOMEM;
	} */
	printk("PLX memBase %llx, mapSize %x, map MMIO %p\n", memBase, mapSize, ssNTB_MMIO);

	pci_read_config_dword(pdevUp, 0xa30, &reg);/* upstream Switch Debug */
	printk("Upstream Switch Debug (0xA30): 0x%08x\n", reg);

	/*****************************************************
	 *               Virtual Interface
	 *****************************************************/
	printk("NTB Virtual Interface:-------------------------------------------\n");
	/* BAR2/3 Setup Register, 64Bit, Prefetch */
	reg=PLX_VIRT_MMIO_READ(0xd0); /* Config BAR Setup */
	printk("Config BAR Setup(0xd0): 0x%08x\n", reg);

	/* BAR2/3 Register */
	BAR_3_2 = PLX_VIRT_MMIO_READ(0x1c);
	BAR_3_2 = (BAR_3_2 << 32) + PLX_VIRT_MMIO_READ(0x18);
	printk("BAR_3/BAR_2(0x1c)(0x18)  : 0x%llx\n", BAR_3_2);

	barWindow=PLX_VIRT_MMIO_READ(0xd8);
	barWindow=(barWindow << 32) + PLX_VIRT_MMIO_READ(0xd4);
	printk("BAR 2/3 Window(0xd8)(0xd4): 0x%llx\t", barWindow);
	ssNTB_VirtWinSize=~(barWindow & ~0xfl)+1;
	printk("Address Range Size:0x%08x\n", ssNTB_VirtWinSize);

	/* Link-side Address */
	ssNTB_VirtMapBase = PLX_VIRT_MMIO_READ(0xc40);
	ssNTB_VirtMapBase = (ssNTB_VirtMapBase << 32) + PLX_VIRT_MMIO_READ(0xc3c);
	printk("Virtual Trans Addr(0xc40)(0xc3c): 0x%016llx\n", ssNTB_VirtMapBase);

	reg=PLX_VIRT_MMIO_READ(0x764); /* Ingress Chip Control */
	printk("Chip Ctrl(0x764): 0x%08x\n", reg);
	/* Requester ID */
	for(pn=0xd94, i=0; pn<0xdb0+4; pn+=4, i++){
		uint32_t bus, dev, ns, fun;
		reg=PLX_VIRT_MMIO_READ(pn);
		printk("Register:0x%08x;\t", reg);
		bus=(reg&0x0000ff00)>>8; dev=(reg&0xf1)>>3;fun=reg & 0x07;ns=reg & BIT(30);
		if(reg & BIT(31))
			printk("entry %d: %02x:%02x.%x, %s\t", i, bus, dev, fun, ns?"NS":" S");
		printk("\n");
	}
	pciCmdSts = PLX_VIRT_MMIO_READ(0x04);
//	PLX_VIRT_MMIO_WRITE(0x04, pciCmdSts | 0x6);
	printk("pciCmdSts(0x04): 0x%08x\n", pciCmdSts);
	pciCmdSts = PLX_VIRT_MMIO_READ(0x70);
	printk("Virtual Dev Sts and Ctrl(0x70): 0x%08x\n", pciCmdSts);

	/*****************************************************
	 *               Link Interface
	 *****************************************************/
	printk("NTB Link Interface:-------------------------------------------\n");
	/* BAR2/3 Setup Register, 64Bit, Prefetch */
	reg=PLX_LINK_MMIO_READ(0xe4); /* Config BAR Setup */
	printk("Link Config BAR Setup(0xe4): 0x%08x\n", reg);

	/* BAR2/3 Register */
	BAR_3_2 = PLX_LINK_MMIO_READ(0x1c);
	BAR_3_2 = (BAR_3_2<< 32) + PLX_LINK_MMIO_READ(0x18);
	printk("Link BAR_3/BAR_2(0x1c)(0x18): 0x%llx\n", BAR_3_2);

	barWindow = PLX_LINK_MMIO_READ(0xec);
	barWindow = (barWindow << 32) + PLX_LINK_MMIO_READ(0xe8);
	printk("Link BAR 2/3 Window(0xec)(0xe8): 0x%llx\t", barWindow);
	ssNTB_LinkWinSize=~(barWindow & ~0xfl)+1;
	printk("Link Address Range Size:0x%08x\n", ssNTB_LinkWinSize);

	/* BAR2/3 Address Translation Register */
	ssNTB_LinkMapBase = PLX_LINK_MMIO_READ(0xc40);
	ssNTB_LinkMapBase = (ssNTB_LinkMapBase<<32) + PLX_LINK_MMIO_READ(0xc3c);
	printk("Link Trans Addr(0xc40)(0xc3c): 0x%llx\n", ssNTB_LinkMapBase);

	reg=PLX_LINK_MMIO_READ(0x764); /* Ingress Chip Control */
	printk("Chip Ctrl(0x764): 0x%08x\n", reg);
	/* Requester ID */
	for(pn=0xdb4, i=0; pn<0xdf0+4; pn+=4, i+=2){
		uint16_t wVal, bus, dev, ns;
		reg=PLX_LINK_MMIO_READ(pn);
		printk("Register:0x%08x;\t", reg);
		wVal=reg&0xFFFF; bus=wVal>>8; dev=(wVal&0xf1)>>3;ns=wVal & BIT(1);
		if(wVal&BIT(0))
			printk("entry %d: %02x:%02x.  %s\t", i, bus, dev, ns?"NS":"S");
		wVal=reg>>16;    bus=wVal>>8; dev=(wVal&0xf1)>>3;ns=wVal & BIT(1);
		if(wVal&BIT(0))
			printk("entry %d: %02x:%02x.%s\t", i+1, bus, dev, ns?"NS":"S");
		printk("\n");
	}

	pciCmdSts = PLX_LINK_MMIO_READ(0x04);
//	PLX_LINK_MMIO_WRITE(0x04, pciCmdSts | 0x6);
	printk("pciCmdSts(0x04): 0x%08x\n", pciCmdSts);
	pciCmdSts = PLX_LINK_MMIO_READ(0x70);
	printk("Link Dev Sts and Ctrl(0x70): 0x%08x\n", pciCmdSts);

	printk("[%s] NTB Virt(Base=%lld(MB),Size=%d(MB)), Link(Base=%lld(MB)), size=%d(MB))\n", __FUNCTION__,
		ssNTB_VirtMapBase/(1024*1024), ssNTB_VirtWinSize/(1024*1024),
		ssNTB_LinkMapBase/(1024*1024), ssNTB_LinkWinSize/(1024*1024));

	return (0);
}

void
ssNTB_SetUpStream(void)
{
	struct pci_dev *pdevUp;
	unsigned long long MAP_BASE = SSPLX_NT_MAP_BASE;
//	unsigned long long BAR_WINDOW = ~(SSPLX_NT_WINDOW_SIZE - 1);
	unsigned long long PREFETCH_LIMIT = MAP_BASE + SSPLX_NT_WINDOW_SIZE - 1;
	
	pdevUp=ssNTB_SWT.port[20].pdev;

	/* Setup upstream port prefetch window */
	pci_write_config_dword(pdevUp, 0x24,
			(((uint32_t) PREFETCH_LIMIT) & 0xfff00000) |
			((uint32_t) (MAP_BASE >> 16) & 0x0000fff0) | 0x00010001);
	pci_write_config_dword(pdevUp, 0x28, ((uint32_t) (MAP_BASE >> 32)));
	pci_write_config_dword(pdevUp, 0x2c, ((uint32_t) (PREFETCH_LIMIT >> 32)));

	/* PCIe root */
	raw_pci_write(0, 0, 0x18, 0x24, 4,
			(((uint32_t) PREFETCH_LIMIT) & 0xfff00000) |
			((uint32_t) (MAP_BASE >> 16) & 0x0000fff0) | 0x00010001);
	raw_pci_write(0, 0, 0x18, 0x28, 4, ((uint32_t) (MAP_BASE >> 32)));
	raw_pci_write(0, 0, 0x18, 0x2c, 4, ((uint32_t) (PREFETCH_LIMIT >> 32)));

// Virtual I/F
	PLX_VIRT_MMIO_WRITE(0x70,  0x0000008F); /* Device Status and Control */
// Link    I/F
	PLX_LINK_MMIO_WRITE(0x70,  0x0000008F); /* Device Status and Control */
	
#define CENTRAL_SET
#ifdef CENTRAL_SET
	PLX_VIRT_MMIO_WRITE(0xd94, 0x80000000); /* 00:00:0 --- PCI-e Root port 0 */
	PLX_VIRT_MMIO_WRITE(0xd98, 0x80000018); /* 00:03:0 --- PCI-e Root port 1 */
	PLX_VIRT_MMIO_WRITE(0xd9c, 0x80000050); /* 00:0a:0 --- CB DMA 0 */
	PLX_VIRT_MMIO_WRITE(0xda0, 0x80000051); /* 00:0a:1 --- CB DMA 1 */
	PLX_VIRT_MMIO_WRITE(0xda4, 0x80000052); /* 00:0a:2 --- CB DMA 2 */
	PLX_VIRT_MMIO_WRITE(0xda8, 0x80000053); /* 00:0a:3 --- CB DMA 3 */
	PLX_VIRT_MMIO_WRITE(0xdac, 0x80000054); /* 00:0a:4 --- CB DMA 4 */
	PLX_VIRT_MMIO_WRITE(0xdb0, 0x80000055); /* 00:0a:5 --- CB DMA 5 */
#endif

}

void ssNTBmemcpy(
	void	*target,
	void	*source,
	uint32_t	BUFFER_SIZE)
{
	uint32_t i;
	uint64_t *t=(uint64_t *)target;
	uint64_t *s=(uint64_t *)source;

	for(i=BUFFER_SIZE/sizeof(uint64_t);i; i--)
		*t++=*s++;
}

int ssNTBmemcmp(
	void	*target,
	void	*source,
	uint32_t	BUFFER_SIZE)
{
	uint32_t i;
	uint64_t *t=(uint64_t *)target;
	uint64_t *s=(uint64_t *)source;

//	if(ssNTB_NoVerify)
//		return 0;

	for(i=BUFFER_SIZE/sizeof(uint64_t);i; i--)
		if(*t++ != *s++)
			return 1;
	return 0;
}

#if 0
int chkMapResult(void *addr, uint32_t size, char *str)
{
	uint32_t *p;
	uint32_t i;

	p = (uint32_t *)addr;
	for(i=0; i<size/sizeof(uint32_t); i++) {
		p[i]=0xdeadbeef;
	}
	for(i=0; i<size/sizeof(uint32_t); i+=1024*16) {
		if(p[i]!=0xdeadbeef){
			printk("=NTB=>%s 0x%p ioremap failed(i=%d, 0x%08x)\n", str, p, i, p[i]);
//			ssNTB_TestErrCnt++;
			return 0;
		}
		printk("%s OK i=%d, loc=%p, data=0x%x\r", str, i, &p[i], p[i]);
	}
	return 1;
}
#endif

void *ssNTB_VirAddrLocal=NULL;
void *ssNTB_VirAddrRemote=NULL;

int ssNTB_Init(void)
{
	/*
	 * Find out NT Device and setup memory windows for this test
	 */
	if(alreadyInit)
		return 1;
	
	plxIpcExit();
	plxPeerIpcExit();

/*	if(plxEepromProgramSettings("NTB") !=0) {
		printk("[%s] NTB EEPROM set up failure!!!\n",__FUNCTION__);
		return 0;
	}
*/	
	if(ssNTB_SetUp()) {
		printk("[%s] NTB set up failure!!!\n",__FUNCTION__);
//		alreadyInit=1;
		return 0;
	}

	ssNTB_MaxRange=16*1024*1024; // ssNTB_VirtWinSize/(ssCPU_Cores+DMA_ChCnt)/(1024*1024)*(1024*1024);

	ssDMA_MapBase=0;
	ssDMA_MaxRange=MAPAreaSize/DMA_ChCnt/(1024*1024)*(1024*1024);

	ssNTB_PatternCnt=0;
//	ssNTB_ReadPatterns();

	alreadyInit=1;
	return 0;
}

int ssNTB_OpbyCPU(void)
{
	int cpu;
	struct ssNTB_TEST_CB *cb;

	ssNTBCPU_CROutput("\n[%s]===================================================================\n", __FUNCTION__);
	ssNTB_SetUpStream();
#ifndef CENTRAL_SET
	PLX_VIRT_MMIO_WRITE(0xd94, 0x80000000); /* 00:00:0 --- PCI-e Root port 0 */
	PLX_VIRT_MMIO_WRITE(0xd98, 0x80000018); /* 00:03:0 --- PCI-e Root port 1 */
	PLX_VIRT_MMIO_WRITE(0xd9c, 0x800000b0); /* 00:16:0 --- Link */
	PLX_VIRT_MMIO_WRITE(0xda0, 0x800000b1); /* 00:16:1 --- Routing and Protocol */
	PLX_VIRT_MMIO_WRITE(0xda4, 0x80000400); /* 04:00:0 --- fc channel 0 */
	PLX_VIRT_MMIO_WRITE(0xda8, 0x80000401); /* 04:00:1 --- fc channel 1 */
	PLX_VIRT_MMIO_WRITE(0xdac, 0x80000402); /* 04:00:2 --- fc channel 2 */
	PLX_VIRT_MMIO_WRITE(0xdb0, 0x80000403); /* 04:00:3 --- fc channel 3 */
#endif
//	ssNTBCPU_TestMap=0;
	ssNTBCPU_TestErrCnt=0;
	
	for(cpu=0; cpu<ssCPU_Cores; cpu++) {
		cb = &ssNTBCPU_Cb[cpu];
		cb->task = NULL;
		cb->maxSize = ssNTB_MaxRange; // MAPAreaSize/(ssCPU_Cores+DMA_ChCnt);
		cb->localAddr  = (phys_addr_t)ssNTB_LinkMapBase + (cb->maxSize)*(cpu+DMA_ChCnt);
		cb->remoteAddr = (phys_addr_t)ssNTB_VirtMapBase + (cb->maxSize)*(cpu+DMA_ChCnt);
		ssNTBCPU_CROutput("[%s] NTB CPU(%d) Local@0x%010llx, Remote@0x%010llx, Map Size=%d(MB)\n", __FUNCTION__,
				cpu, cb->localAddr, cb->remoteAddr, cb->maxSize/(1024*1024));
		cb->ErrCnt=0;
	}

	return 0;
}

int ssNTB_OpbyDMA(void)
{
	int dma;
	struct ssNTB_TEST_CB *cb;

	ssNTBDMA_CROutput("\n[%s]===================================================================\n", __FUNCTION__);
	ssNTB_SetUpStream();
#ifndef CENTRAL_SET
	PLX_VIRT_MMIO_WRITE(0xd94, 0x80000000); /* 00:00:0 --- PCI-e Root port 0 */
	PLX_VIRT_MMIO_WRITE(0xd98, 0x80000018); /* 00:03:0 --- PCI-e Root port 1 */
	PLX_VIRT_MMIO_WRITE(0xd9c, 0x80000050); /* 00:0a:0 --- CB DMA 0 */
	PLX_VIRT_MMIO_WRITE(0xda0, 0x80000051); /* 00:0a:1 --- CB DMA 1 */
	PLX_VIRT_MMIO_WRITE(0xda4, 0x80000052); /* 00:0a:2 --- CB DMA 2 */
	PLX_VIRT_MMIO_WRITE(0xda8, 0x80000053); /* 00:0a:3 --- CB DMA 3 */
	PLX_VIRT_MMIO_WRITE(0xdac, 0x80000054); /* 00:0a:4 --- CB DMA 4 */
	PLX_VIRT_MMIO_WRITE(0xdb0, 0x80000055); /* 00:0a:5 --- CB DMA 5 */
#endif
//	ssNTBDMA_TestMap=0;
	ssNTBDMA_TestErrCnt=0;
	
	for(dma=0; dma<DMA_ChCnt; dma++) {
		cb = &ssNTBDMA_Cb[dma];
		cb->task = NULL;
		cb->maxSize = ssNTB_MaxRange; // MAPAreaSize/(ssCPU_Cores+DMA_ChCnt);
		cb->localAddr  = (phys_addr_t)ssNTB_LinkMapBase + (cb->maxSize)*dma;
		cb->remoteAddr = (phys_addr_t)ssNTB_VirtMapBase + (cb->maxSize)*dma;
		ssNTBDMA_CROutput("[%s] NTB DMA(%d) Local@0x%010llx, Remote@0x%010llx, Map Size=%d(MB)\n", __FUNCTION__,
				dma, cb->localAddr, cb->remoteAddr, cb->maxSize/(1024*1024));
		cb->ErrCnt=0;
	}

	return 0;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:     ssNTBCPURun
* SUMMARY:  Generate I/O traffic on NT Bridge
*
* SCOPE:    Private
*
* DESCRIPTION:
*	This test 1) set patterns to local controller's memory. 2) dma local memory
*	to peer controller's memory. 3)dma back peer memory to local and compare with
*	pattern memory.
* RETURNS:
*
* NOTES:
*
*/

//       time_t time(time_t *t);
static int
ssNTBCPU_Run(void *data)
{
	struct ssNTB_TEST_CB *cb = data;
	int i, iter=0, cpu, ret=0;
	time_t		tt_start, tt_end;
	uint64_t	ttotalMB=0;
	uint64_t	ttotalSleep=0;
	time_t		t_start, t_end;
	uint64_t	totalMB=0;
	uint64_t	totalSleep=0;
	uint32_t rem, period;
	uint32_t BUFFER_SIZE = cb->blockSize;
	uint32_t *virAddrPat = NULL;
	uint32_t *phyAddrBack;
	void *virAddrBack = NULL;
	void *virAddrLocal = NULL;
	void *virAddrRemote = NULL;
	void *virAddrLocalOrig = (void *)cb->localAddr;
	void *virAddrRemoteOrig = (void *)cb->remoteAddr;

	cpu=cb->client;
	virAddrLocalOrig = (void *)ioremap(cb->localAddr, cb->maxSize);
	if (virAddrLocalOrig == NULL)
	{
		ssNTBCPU_CROutput("[%s](%d) 4. Local. 0x%llx ioremap failed\n", __FUNCTION__, cpu, cb->localAddr);
		cb->ErrCnt++;
		ret=1; goto ssNTBCPU_Run_end_1;
	}

	virAddrRemoteOrig = (void *)ioremap(cb->remoteAddr, cb->maxSize);
	if (virAddrRemoteOrig == NULL)
	{
		ssNTBCPU_CROutput("[%s](%d) 4. Remote. 0x%llx ioremap failed\n", __FUNCTION__, cpu, cb->remoteAddr);
		cb->ErrCnt++;
		ret=1; goto ssNTBCPU_Run_end_1;
	}

/*	if(!chkMapResult(virAddrRemoteOrig, cb->maxSize, "REMOTE")) {
		ret=1; goto ssNTB_Run_end_1;
	}
*/
	virAddrPat = (uint32_t *)virAddrLocalOrig;
	if(cb->pattern!=0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=cb->pattern;

	virAddrBack= virAddrLocalOrig+(cb->rangeSize - BUFFER_SIZE);
	phyAddrBack= (uint32_t *)(cb->localAddr+(cb->rangeSize - BUFFER_SIZE));

	set_user_nice(current, 19);
	
// -------------------------------------------------------------------------------
	tt_start=t_start=get_seconds();
	ttotalSleep=0;
	totalSleep=0;
	iter=0;

ssNTBCPU_Run_iteration_start:
/*	if(kthread_should_stop()){
		ret=0; goto ssNTBCPU_Run_end;
	}
*/	if(cb->pattern==0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=ssNTB_Patterns[iter%8];
	
	virAddrLocal  = virAddrLocalOrig;
	virAddrRemote = virAddrRemoteOrig;
	for (i=0; virAddrLocal < virAddrBack; i++) {

		ssNTBmemcpy(virAddrRemote, virAddrLocal, BUFFER_SIZE);
		ttotalMB += BUFFER_SIZE;
		totalMB  += BUFFER_SIZE;
		msleep(SS_SLEEPTIME);
		ttotalSleep+=SS_SLEEPTIME;
		totalSleep+=SS_SLEEPTIME;

		virAddrLocal += BUFFER_SIZE;
		ssNTBmemcpy(virAddrLocal, virAddrRemote, BUFFER_SIZE);
		ttotalMB += BUFFER_SIZE;
		totalMB  += BUFFER_SIZE;
		msleep(SS_SLEEPTIME);
		ttotalSleep+=SS_SLEEPTIME;
		totalSleep+=SS_SLEEPTIME;

		if (kthread_should_stop()/* || ssNTBmemcmp(virAddrPat, virAddrLocal, BUFFER_SIZE)*/){
			virAddrBack = virAddrLocal;
			break;
		}

		virAddrRemote += BUFFER_SIZE;
	}

	if(cb->flags & Data_Verification)
	if (ssNTBmemcmp(virAddrPat, virAddrBack, BUFFER_SIZE))
	{
		uint32_t *p;
		uint32_t cnt, j;
		/* TODO */
		sslock_CommonNTBCPU();
		printk("[%s](%d)(%d) compare error(at IO=%d), block 0x%p\n",
			       __FUNCTION__, cpu, iter, i, virAddrBack);
		ssNTBCPU_CROutput("[%s](%d)(%d) compare error(at IO=%d), block 0x%p\n",
			       __FUNCTION__, cpu, iter, i, virAddrBack);
		ssNTBCPU_CROutput("[%s](%d) Location\t\tExpect\t\tFound\t\tDevice\n", __FUNCTION__, cpu);
		p=(uint32_t *)virAddrBack;
		cnt=0;
		for(j=0; j<BUFFER_SIZE/sizeof(uint32_t); j++){
			if(virAddrPat[j]==p[j])
				continue;
			if(++cnt>10) break;
			ssNTBCPU_CROutput("[%s](%d) %p\t0x%08x\t0x%08x\tNTB+CPU(%d)\n", __FUNCTION__, cpu,
				       &phyAddrBack[j], virAddrPat[j], p[j], cb->client);
		}
//		commReportError(MEC_PCIE_SWITCH,
//					DMA_DIAG_ITEM_CODE_BOARD_STRESS_COPY,
//					DMA_DIAG_ERROR_CODE_TEST_INCOMPLETE,
//					DMA_DIAG_ERROR_MSG_TEST_INCOMPLETE,
//					0,
//					0,
//					0,
//					DG_FORMAT_NONE);
		ssunlock_CommonNTBCPU();
		cb->ErrCnt++;
		if(cb->flags & STOP_ON_ERROR) {
			ret=1; goto ssNTBCPU_Run_end;
		}
	}
//	ttotalMB += BUFFER_SIZE*i*2;
//	totalMB  += BUFFER_SIZE*i*2;
	t_end=get_seconds(); period=t_end - t_start-(totalSleep/1000);
	iter++;
	if(period > SS_PERFPERIOD) {
		totalMB /= (1024*1024);
		rem=totalMB-totalMB/period*period;
		ssNTBCPU_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), period throughput %lld.%d(MBps)\n", __FUNCTION__, cpu,
			iter, period, totalMB, totalMB/period, rem*100/period);
		totalMB=0;
		totalSleep=0;
		t_start=get_seconds();
	}
//	printk("[%s](%d) Iteration %d PASS\n", __FUNCTION__, cpu, iter);

	if((cb->iterations > 0 && iter < cb->iterations) || cb->iterations == 0) {
//		msleep(SS_SLEEPTIME);
		if (! kthread_should_stop())
			goto ssNTBCPU_Run_iteration_start;
//		else
//printk("%s: kthread_should_stop is TRUE, Map=0x%04x\n", __FUNCTION__, ssNTBCPU_TestMap);
	}
// -------------------------------------------------------------------------------
//printk("%s: Iter=%d, cb->iterations=%d\n", __FUNCTION__, iter, cb->iterations);
	ret=0;
	
ssNTBCPU_Run_end:

	tt_end=get_seconds(); period=tt_end - tt_start;
	ttotalMB /= (1024*1024);
	if(period) {
		uint32_t	adjPeriod=(uint32_t)(period-(ttotalSleep/1000));
		rem=ttotalMB-ttotalMB/period*period;
		ssNTBCPU_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), throughput %lld.%d(MBps)\n", __FUNCTION__, cpu,
			iter, adjPeriod, ttotalMB, ttotalMB/adjPeriod, rem*100/adjPeriod);
	} else
		ssNTBCPU_PROutput("[%s](%d) %d iterations, time elapse %d(sec) < 1, IO Data %lld(MB)\n", __FUNCTION__, cpu,
			iter, period, ttotalMB);

ssNTBCPU_Run_end_1:
	if(ret)
		ssNTBCPU_TestErrCnt++;
	ssNTBCPU_CROutput("[%s](%d) %d iterations with error count=%d\n", __FUNCTION__, cpu, iter, cb->ErrCnt+ssNTBCPU_TestErrCnt);

	if(virAddrLocalOrig)
		iounmap(virAddrLocalOrig);
	if(virAddrRemoteOrig)
		iounmap(virAddrRemoteOrig);
//	ssNTBCPU_TestMapDel(cb->client);
//	ssNTB_TestMapAdd(cb->client);
	cb->task = NULL;
	return ret;
}

/**
 * ssNTB_AsyncMemcpyBuf2Buf - offloaded copy between virtual addresses
 * @chan: DMA channel to offload copy to
 * @dest: destination address (virtual)
 * @src: source address (virtual)
 * @len: length
 *
 * Both @dest and @src must be mappable to a bus address according to the
 * DMA mapping API rules for streaming mappings.
 * Both @dest and @src must stay memory resident (kernel memory or locked
 * user space pages).
 */
dma_cookie_t
ssNTB_AsyncMemcpyBuf2Buf(
	struct dma_chan	*chan,
	void *dest,
	void *src,
	size_t len)
{
	struct dma_device *dev = chan->device;
	struct dma_async_tx_descriptor *tx;
	dma_addr_t dma_dest, dma_src;
	dma_cookie_t cookie;
//	int cpu;

	dma_src =(dma_addr_t)src;
	dma_dest=(dma_addr_t)dest;
//	dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
//	dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
//printk("--->src addr: %llx, des addr: %llx\n", dma_src, dma_dest);
	tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len,
					 DMA_CTRL_ACK);

	if (!tx) {
		struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);
		
//		dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE);
//		dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE);
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30)
		spin_lock_bh(&ioat->ring_lock);
#else
		spin_lock_bh(&ioat->prep_lock);
#endif

		return -ENOMEM;
	}

	tx->callback = NULL;
	cookie = tx->tx_submit(tx);
/*
	cpu = get_cpu();
	per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
	per_cpu_ptr(chan->local, cpu)->memcpy_count++;
	put_cpu();
*/
	return cookie;
}


enum dma_status ssDMA_SyncWait(struct dma_chan *chan, dma_cookie_t cookie)
{
	enum dma_status status;
//	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
	unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
	struct ioat_chan_common*	pCBDMAChan;
	struct ioatdma_device*		pCBDMADevice;
	// int chanId;
	struct ioat2_dma_chan *ioat = to_ioat2_chan(chan);

	ASSERT(chan);
	pCBDMAChan = to_chan_common(chan);
	ASSERT(pCBDMAChan);
	pCBDMADevice = pCBDMAChan->device;
	ASSERT(pCBDMADevice);

	// chanId= (pCBDMADevice->pdev->device - CBDMA_DEVICE_0) % CBDMA_CHAN_NUM;

	dma_async_issue_pending(chan);
	do {
		status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
		if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
			struct ioatdma_device *device = ioat->base.device;
			
#if LINUX_VERSION_CODE < KERNEL_VERSION(2,6,30)
			device->cleanup_tasklet((unsigned long) ioat);
#else
			device->cleanup_fn((unsigned long) ioat);
#endif

			//printk(KERN_ERR "[%s](%d) dma_sync_wait_timeout!\n", __FUNCTION__, chanId);
			printk(KERN_ERR "[%s] (%s) dma_sync_wait_timeout!\n", __FUNCTION__, dma_chan_name(chan));

//			ssDMA_CROutput("[%s](%d) dma_sync_wait_timeout!\n", __FUNCTION__, chanId);
			return DMA_ERROR;
		}
//		msleep(1);
	} while (status == DMA_IN_PROGRESS);

	return status;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:     ssNTBDMARun
* SUMMARY:  Generate I/O traffic on NT Bridge
*
* SCOPE:    Private
*
* DESCRIPTION:
*	This test 1) set patterns to local controller's memory. 2) dma local memory
*	to peer controller's memory. 3)dma back peer memory to local and compare with
*	pattern memory.
* RETURNS:
*
* NOTES:
*
*/

// extern struct dma_chan* dmaGetDMAChan(dma_cap_mask_t mask, UINT32 idxChan);
// extern VOID dmaFreeDMAChan(UINT32 idxChan);

uint32_t ssNTB_Sleep=2;
//       time_t time(time_t *t);


static int
ssNTBDMA_Run(void *data)
{
	struct ssNTB_TEST_CB *cb = data;
	int i, iter=0, dma, ret=0;
	time_t		tt_start, tt_end;
	uint64_t	ttotalMB=0;
	time_t		t_start, t_end;
	uint64_t	totalMB=0;
#ifdef TOTALSLEEP
	uint64_t	ttotalSleep=0;
	uint64_t	totalSleep=0;
#endif
	uint32_t rem, period;
	uint32_t BUFFER_SIZE = cb->blockSize;
	uint32_t *virAddrPat = NULL;
	uint32_t *phyAddrBack = NULL;
	void *virAddrBack = NULL;
	void *virAddrLocal = NULL;
	void *virAddrRemote = NULL;
	void *phyAddrLocal = NULL;
	void *phyAddrRemote = NULL;
	void *virAddrLocalOrig = (void *)cb->localAddr;
	void *virAddrRemoteOrig = (void *)cb->remoteAddr;

	dma_cap_mask_t 	mask;
	struct dma_chan	*dmaChan;

#if defined(CHASSIS_WSAS)
#ifdef SINGLE_CORE
    struct cpumask affmask;
    affmask.bits[0] = 0x08;
    sched_setaffinity(0,(const struct cpumask *)&affmask);
#endif
#endif

	dma_cap_zero(mask);
	dma_cap_set(DMA_MEMCPY, mask);

	dma=cb->client;
	//dmaChan = dmaGetDMAChan(mask, dma);
	dmaChan = dma_request_channel(mask, NULL, NULL);


	if(dmaChan == NULL){
		ssNTBDMA_CROutput("[%s](%d) DMA channel not found!!!\n", __FUNCTION__, dma);
		cb->ErrCnt++;
		ret=1; goto ssNTBDMA_Run_end_1;
	}
	virAddrLocalOrig = (void *)ioremap(cb->localAddr, cb->maxSize);
	if (virAddrLocalOrig == NULL)
	{
		ssNTBDMA_CROutput("[%s](%d) 4. Local. 0x%llx ioremap failed\n", __FUNCTION__, dma, cb->localAddr);
		cb->ErrCnt++;
		ret=1; goto ssNTBDMA_Run_end_1;
	}
	SSPRINTK(1, "[%s](%d) local virtual map: 0x%016llx ==> 0x%p\n", __FUNCTION__, dma, cb->localAddr, virAddrLocalOrig);

//	cb->remoteAddr=cb->localAddr-(128*1024*1024);
//printk("Local/Remote Location: %llx/%llx\n", cb->localAddr, cb->remoteAddr);
	virAddrRemoteOrig = (void *)ioremap(cb->remoteAddr, cb->maxSize);
	if (virAddrRemoteOrig == NULL)
	{
		ssNTBDMA_CROutput("[%s](%d) 4. Remote. 0x%llx ioremap failed\n", __FUNCTION__, dma , cb->remoteAddr);
		cb->ErrCnt++;
		ret=1; goto ssNTBDMA_Run_end_1;
	}
	SSPRINTK(1, "[%s](%d) remote virtual map: 0x%016llx ==> 0x%p\n", __FUNCTION__, dma, cb->remoteAddr, virAddrRemoteOrig);

/*	if(!chkMapResult(virAddrRemoteOrig, cb->maxSize, "REMOTE")) {
		ret=1; goto ssNTBDMA_Run_end_1;
	}
*/
	virAddrPat = (uint32_t *)virAddrLocalOrig;
	if(cb->pattern!=0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=cb->pattern;

	virAddrBack= virAddrLocalOrig+(cb->rangeSize - BUFFER_SIZE);
	phyAddrBack= (uint32_t *)(cb->localAddr+(cb->rangeSize - BUFFER_SIZE));

// -------------------------------------------------------------------------------
	tt_start=t_start=get_seconds();
#ifdef TOTALSLEEP
	totalSleep=0;
	ttotalSleep=0;
#endif
	iter=0;
//printk("here, here!!\n");

ssNTBDMA_Run_iteration_start:
/*	if(kthread_should_stop()){
		ret=0; goto ssNTBDMA_Run_end;
	}
*/	if(cb->pattern==0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=ssNTB_Patterns[iter%8];

	virAddrLocal  = virAddrLocalOrig;
	phyAddrLocal  = (void *)cb->localAddr;
	virAddrRemote = virAddrRemoteOrig;
	phyAddrRemote = (void *)cb->remoteAddr;
	for (i=0; virAddrLocal < virAddrBack; i++) {
		dma_cookie_t	cookie;
		enum dma_status sts;
#if 0
{		
		uint32_t j, *p;
		p=(uint32_t *)virAddrRemote;
		for(j=0; j<BUFFER_SIZE/sizeof(uint32_t); j++)
			p[j]=0xdeadbeef;
}
#endif
//		ssNTBmemcpy(virAddrRemote, virAddrLocal, BUFFER_SIZE);
//		cookie = dma_async_memcpy_buf_to_buf(dmaChan, virAddrRemote, virAddrLocal, BUFFER_SIZE);
		cookie = ssNTB_AsyncMemcpyBuf2Buf(dmaChan, phyAddrRemote, phyAddrLocal, BUFFER_SIZE);
		if(cookie <0) {
			ssNTBDMA_CROutput("[%s](%d)(%d)(%d)(L->R) Cookie Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrRemote, phyAddrLocal);
//					__FUNCTION__, dma, i, virAddrRemote, virAddrLocal);
			ret=1; goto ssNTBDMA_Run_end;
		}
		
//		sts=dma_sync_wait(dmaChan, cookie);
		sts=ssDMA_SyncWait(dmaChan, cookie); 
		if(sts == DMA_ERROR){
			ssNTBDMA_CROutput("[%s](%d)(%d)(%d)(L->R) Xfer Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrLocal, phyAddrRemote);
//					__FUNCTION__, dma, i, virAddrLocal, virAddrRemote);
			ret=1; goto ssNTBDMA_Run_end;
		}
		ttotalMB += BUFFER_SIZE;
		totalMB  += BUFFER_SIZE;

		virAddrLocal += BUFFER_SIZE;
		phyAddrLocal += BUFFER_SIZE;
//		ssNTBmemcpy(virAddrLocal, virAddrRemote, BUFFER_SIZE);
//		cookie = dma_async_memcpy_buf_to_buf(dmaChan, virAddrLocal, virAddrRemote, BUFFER_SIZE);
		cookie = ssNTB_AsyncMemcpyBuf2Buf(dmaChan, phyAddrLocal, phyAddrRemote, BUFFER_SIZE);
		if(cookie <0) {
			ssNTBDMA_CROutput("[%s](%d)(%d)(%d)(R->L) Cookie Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrLocal, phyAddrRemote);
			ret=1; goto ssNTBDMA_Run_end;
		}
		
//		sts=dma_sync_wait(dmaChan, cookie);
		sts=ssDMA_SyncWait(dmaChan, cookie);
		if(sts == DMA_ERROR){
			ssNTBDMA_CROutput("[%s](%d)(%d)(%d)(R->L) Xfer Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrRemote, phyAddrLocal);
			ret=1; goto ssNTBDMA_Run_end;
		}
		ttotalMB += BUFFER_SIZE;
		totalMB  += BUFFER_SIZE;

		if (kthread_should_stop()/* || ssNTBmemcmp(virAddrPat, virAddrLocal, BUFFER_SIZE)*/){
			virAddrBack = virAddrLocal;
			break;
		}

		virAddrRemote += BUFFER_SIZE;
		phyAddrRemote += BUFFER_SIZE;
#ifdef TOTALSLEEP
		msleep(SS_SLEEPTIME);
		totalSleep+=SS_SLEEPTIME;
		ttotalSleep+=SS_SLEEPTIME;
#endif
	}

	if(cb->flags & Data_Verification)
	if (ssNTBmemcmp(virAddrPat, virAddrBack, BUFFER_SIZE))
	{
		uint32_t *p;
		uint32_t cnt, j;
		/* TODO */
		sslock_CommonNTBDMA();
		printk("[%s](%d)(%d) compare error(at IO=%d), block 0x%p, size 0x%08x\n",
					__FUNCTION__, dma, iter, i, virAddrBack, BUFFER_SIZE);
		ssNTBDMA_CROutput("[%s](%d)(%d) compare error(at IO=%d), block 0x%p, size 0x%08x\n",
					__FUNCTION__, dma, iter, i, virAddrBack, BUFFER_SIZE);
		ssNTBDMA_CROutput("[%s](%d) Location\t\tExpect\t\tFound\t\tDevice\n", __FUNCTION__, dma);
		p=(uint32_t *)virAddrBack;
		cnt=0;
		for(j=0; j<BUFFER_SIZE/sizeof(uint32_t); j++){
			if(virAddrPat[j]==p[j])
				continue;
			if(++cnt>10) break;
			ssNTBDMA_CROutput("[%s](%d) %p\t0x%08x\t0x%08x\tNTB+DMA(%d)\n", __FUNCTION__, dma,
				       &phyAddrBack[j], virAddrPat[j], p[j], cb->client);
		}
//		commReportError(MEC_PCIE_SWITCH,
//					DMA_DIAG_ITEM_CODE_BOARD_STRESS_COPY,
//					DMA_DIAG_ERROR_CODE_TEST_INCOMPLETE,
//					DMA_DIAG_ERROR_MSG_TEST_INCOMPLETE,
//					0,
//					0,
//					0,
//					DG_FORMAT_NONE);
		ssunlock_CommonNTBDMA();
		cb->ErrCnt++;
		if(cb->flags & STOP_ON_ERROR) {
			ret=1; goto ssNTBDMA_Run_end;
		}
	}
//printk("there, there!!\n");
	iter++;
	t_end=get_seconds(); period=t_end - t_start;
	if(period > SS_PERFPERIOD) {
#ifdef TOTALSLEEP
		uint32_t adjPeriod=period-(totalSleep/1000);
#else
		uint32_t adjPeriod=period;
#endif
		totalMB /= (1024*1024);
		rem=totalMB-totalMB/adjPeriod*adjPeriod;
		ssNTBDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), period throughput %lld.%d(MBps)\n", __FUNCTION__, dma,
			iter, adjPeriod, totalMB, totalMB/adjPeriod, rem*100/adjPeriod);
		totalMB=0;
#ifdef TOTALSLEEP
		totalSleep=0;
#endif
		t_start=get_seconds();
	}
//	printk("[%s](%d) Iteration %d PASS\n", __FUNCTION__, dma, iter);

	if((cb->iterations > 0 && iter < cb->iterations) || cb->iterations == 0) {
//		msleep(SS_SLEEPTIME);
		if(!kthread_should_stop())
			goto ssNTBDMA_Run_iteration_start;
	}
// -------------------------------------------------------------------------------
	ret=0;

ssNTBDMA_Run_end:

	tt_end=get_seconds(); period=tt_end - tt_start;
	ttotalMB /= (1024*1024);
	if(period) {
#ifdef TOTALSLEEP
		uint32_t adjPeriod=period-(ttotalSleep/1000);
#else
		uint32_t adjPeriod=period;
#endif
		
		rem=ttotalMB-ttotalMB/adjPeriod*adjPeriod;
		ssNTBDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), throughput %lld.%d(MBps)\n", __FUNCTION__, dma,
			iter, adjPeriod, ttotalMB, ttotalMB/adjPeriod, rem*100/adjPeriod);
	} else
		ssNTBDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec) < 1, IO Data %lld(MB)\n", __FUNCTION__, dma,
			iter, period, ttotalMB);

ssNTBDMA_Run_end_1:
//printk("where, where!!\n");
	if(ret)
		ssNTBDMA_TestErrCnt++;
	ssNTBDMA_CROutput("[%s](%d) %d iterations with error count=%d\n", __FUNCTION__, dma, iter, cb->ErrCnt+ssNTBDMA_TestErrCnt);

	if(virAddrLocalOrig)
		iounmap(virAddrLocalOrig);
	if(virAddrRemoteOrig)
		iounmap(virAddrRemoteOrig);
//	ssNTBDMA_TestMapDel(cb->client);
//	ssNTB_TestMapAdd(cb->client);
	//dmaFreeDMAChan(cb->client);
//	dma_release_channel(cb->client);

	if(dmaChan!=NULL) {
		dma_release_channel(dmaChan);
	}
	cb->task = NULL;
	return ret;
}

void ssMem_ShowData(uint32_t argAddr)
{
	phys_addr_t	phyAddr=(phys_addr_t)argAddr;
	uint32_t	*virtAddr;
	uint32_t	i;
	
#define	SHOW_COUNT 64
	virtAddr = (void *)ioremap(phyAddr, SHOW_COUNT*sizeof(uint32_t));
	printk("\t   %8x %8x %8x %8x %8x %8x %8x %8x\n", 0,1,2,3,4,5,6,7);
	for(i=0; i<SHOW_COUNT; i++){
		if(i%8==0) printk("%8lx | ", (uint32_t)phyAddr+i*sizeof(uint32_t));
		printk("%08x", virtAddr[i]);
		if(i%8==7) printk("\n"); else printk(" ");
	}
	iounmap(virtAddr);
}

int ssMem_OpbyDMA(void)
{
	int dma;
	struct ssNTB_TEST_CB *cb;

//	ssDMA_MapBase=1152*1024*1024;
//	ssDMA_MaxRange=MAPAreaSize/DMA_ChCnt;

//	ssDMA_TestMap=0;
	ssDMA_TestErrCnt=0;
	
	for(dma=0; dma<DMA_ChCnt; dma++) {
		cb=&ssDMA_Cb[dma];
		cb->task = NULL;
//		cb->maxSize = ssDMA_MaxRange;
//		cb->localAddr  = (phys_addr_t)ssDMA_MapBase + (cb->maxSize)*dma;
		cb->remoteAddr = 0;
//		ssDMA_CROutput("[%s] Memory DMA(%d) Local@0x%016llx, Remote@0x%016llx, IO Size=%d(MB)\n", __FUNCTION__,
//				dma, cb->localAddr, cb->remoteAddr, cb->maxSize/(1024*1024));
		cb->ErrCnt=0;
	}

	return 0;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:     ssDMA_Run
* SUMMARY:  Generate DMA copy on RAM
*
* SCOPE:    Private
*
* DESCRIPTION:
*	This test 1) set patterns to first allocated controller's memory block. 2) dma copy from src block
*	to next block. 3)dma copy 1st block to 2nd block 4) until last block in test area has copied
*	5) compare the last block to the source block.
* RETURNS:
*
* NOTES:
*
*/
static int
ssDMA_Run(void *data)
{
	struct ssNTB_TEST_CB *cb = data;
	int i, iter=0, dma, ret=0;
	time_t		tt_start, tt_end;
	uint64_t	ttotalMB=0;
	uint64_t	ttotalSleep=0;
	time_t		t_start, t_end;
	uint64_t	totalMB=0;
	uint64_t	totalSleep=0;
	uint32_t rem, period;
	uint32_t BUFFER_SIZE = cb->blockSize;
	uint32_t *virAddrPat = NULL;
	uint32_t *phyAddrBack;
	void *virAddrBack = NULL;
	void *virAddrSrc = NULL;
	void *virAddrTgt = NULL;
	void *phyAddrSrc = NULL;
	void *phyAddrTgt = NULL;
	void *virAddrSrcOrig = (void *)cb->localAddr;
//	void *virAddrTgtOrig = (void *)cb->remoteAddr;

	dma_cap_mask_t 	mask;
	struct dma_chan	*dmaChan;

	dma_cap_zero(mask);
	dma_cap_set(DMA_MEMCPY, mask);

	dma=cb->client;
//	mutex_lock(&dma_list_mutex);
	//dmaChan = dmaGetDMAChan(mask, dma);
	dmaChan = dma_request_channel(mask, NULL, NULL);
//	mutex_unlock(&dma_list_mutex);
	if(dmaChan == NULL){
		ssDMA_CROutput("[%s](%d) DMA channel not found!!!\n", __FUNCTION__, dma);
		cb->ErrCnt++;
		ret=1; goto ssDMA_Run_end_1;
	}
	virAddrSrcOrig = (void *)ioremap(cb->localAddr, cb->maxSize);
	if (virAddrSrcOrig == NULL)
	{
		ssDMA_CROutput("[%s](%d) Local. 0x%llx ioremap failed\n", __FUNCTION__, dma, cb->localAddr);
		cb->ErrCnt++;
		ret=1; goto ssDMA_Run_end_1;
	}

	virAddrPat = (uint32_t *)virAddrSrcOrig;
	if(cb->pattern!=0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=cb->pattern;

	virAddrBack= virAddrSrcOrig+(cb->rangeSize - BUFFER_SIZE);
	phyAddrBack= (uint32_t *)(cb->localAddr+(cb->rangeSize - BUFFER_SIZE));

// -------------------------------------------------------------------------------
	tt_start=t_start=get_seconds();
	ttotalSleep=0;
	totalSleep=0;
	iter=0;
//printk("(%d)(%d)here, here!!\n", dma, iter);

ssDMA_Run_iteration_start:
/*	if(kthread_should_stop()){
		ret=0; goto ssDMA_Run_end;
	}
*/	if(cb->pattern==0xdeadbeef)
	for(i=0; i<BUFFER_SIZE/sizeof(uint32_t); i++)
		virAddrPat[i]=ssNTB_Patterns[iter%8];

	phyAddrSrc = (void *)cb->localAddr;
	virAddrSrc = virAddrSrcOrig;
	
	phyAddrTgt = (void *)phyAddrSrc;
	virAddrTgt = virAddrSrc;
	for (i=0; virAddrSrc < virAddrBack; i++) {
		dma_cookie_t	cookie;
		enum dma_status sts;

		phyAddrTgt += BUFFER_SIZE;
		virAddrTgt += BUFFER_SIZE;

#if 0
{
		uint32_t j, *p;
		p=(uint32_t *)virAddrRemote;
		for(j=0; j<BUFFER_SIZE/sizeof(uint32_t); j++)
			p[j]=0xdeadbeef;
}
#endif
//		ssNTBmemcpy(virAddrRemote, virAddrLocal, BUFFER_SIZE);
//		cookie = dma_async_memcpy_buf_to_buf(dmaChan, virAddrRemote, virAddrLocal, BUFFER_SIZE);
		cookie = ssNTB_AsyncMemcpyBuf2Buf(dmaChan, phyAddrTgt, phyAddrSrc, BUFFER_SIZE);
		if(cookie <0) {
			ssDMA_CROutput("[%s](%d)(%d)(%d) Cookie Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrSrc, phyAddrTgt);
			ret=1; goto ssDMA_Run_end;
		}

//		sts=dma_sync_wait(dmaChan, cookie);
		sts=ssDMA_SyncWait(dmaChan, cookie);
		if(sts == DMA_ERROR){
			ssDMA_CROutput("[%s](%d)(%d)(%d) Xfer Fail from 0x%p to 0x%p!!!\n",
					__FUNCTION__, dma, iter, i, phyAddrSrc, phyAddrTgt);
			ret=1; goto ssDMA_Run_end;
		}
		ttotalMB += BUFFER_SIZE;
		totalMB  += BUFFER_SIZE;

		phyAddrSrc = phyAddrTgt;
		virAddrSrc = virAddrTgt;

		if (kthread_should_stop()/* || ssNTBmemcmp(virAddrPat, virAddrLocal, BUFFER_SIZE) */){
			virAddrBack = virAddrSrc;
			break;
		}
		msleep(SS_SLEEPTIME);
		ttotalSleep+=SS_SLEEPTIME;
		totalSleep+=SS_SLEEPTIME;
	}

	if(cb->flags & Data_Verification)
	if (ssNTBmemcmp(virAddrPat, virAddrBack, BUFFER_SIZE))
	{
		uint32_t *p;
		uint32_t cnt, j;
		/* TODO */
		sslock_CommonDMA();
		ssDMA_CROutput("[%s](%d) compare error(at IO=%d), block 0x%p, size 0x%08x\n",
					__FUNCTION__, dma, i, virAddrBack, BUFFER_SIZE);
		printk("[%s](%d) compare error(at IO=%d), block 0x%p, size 0x%08x\n",
					__FUNCTION__, dma, i, virAddrBack, BUFFER_SIZE);
		ssDMA_CROutput("[%s](%d) Location\t\tExpect\t\tFound\t\tDevice\n", __FUNCTION__, dma);
		p=(uint32_t *)virAddrBack;
		cnt=0;
		for(j=0; j<BUFFER_SIZE/sizeof(uint32_t); j++){
			if(virAddrPat[j]==p[j])
				continue;
			if(++cnt>10) break;
			ssDMA_CROutput("[%s](%d) %p\t0x%08x\t0x%08x\tMEM+DMA(%d)\n", __FUNCTION__, dma,
				       &phyAddrBack[j], virAddrPat[j], p[j], cb->client);
		}
//		commReportError(MEC_PCIE_SWITCH,
//					DMA_DIAG_ITEM_CODE_BOARD_STRESS_COPY,
//					DMA_DIAG_ERROR_CODE_TEST_INCOMPLETE,
//					DMA_DIAG_ERROR_MSG_TEST_INCOMPLETE,
//					0,
//					0,
//					0,
//					DG_FORMAT_NONE);
		ssunlock_CommonDMA();
		cb->ErrCnt++;
		if(cb->flags & STOP_ON_ERROR) {
			ret=1; goto ssDMA_Run_end;
		}
	}
	iter++;
	t_end=get_seconds(); period=t_end - t_start;
	if(period > SS_PERFPERIOD) {
		uint32_t adjPeriod=period-(totalSleep/1000);
		
		totalMB /= (1024*1024);
		rem=totalMB-totalMB/adjPeriod*adjPeriod;
		ssDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), period throughput %lld.%d(MBps)\n", __FUNCTION__, dma,
			iter, adjPeriod, totalMB, totalMB/adjPeriod, rem*100/adjPeriod);
		totalMB=0;
		totalSleep=0;
		t_start=get_seconds();
	}
//printk("[%s](%d) Iteration %d PASS\n", __FUNCTION__, dma, iter);

	if((cb->iterations > 0 && iter < cb->iterations) || cb->iterations == 0) {
//		msleep(SS_SLEEPTIME);
		if(! kthread_should_stop())
			goto ssDMA_Run_iteration_start;
//		else
//printk("%s: kthread_should_stop is TRUE, Map=0x%04x\n", __FUNCTION__, ssDMA_TestMap);
	}
// -------------------------------------------------------------------------------
//printk("%s: Iter=%d, cb->iterations=%d\n", __FUNCTION__, iter, cb->iterations);
	ret=0;

ssDMA_Run_end:

	tt_end=get_seconds(); period=tt_end - tt_start;
	ttotalMB /= (1024*1024);
	if(period){
		uint32_t adjPeriod=period-(ttotalSleep/1000);
		
		rem=ttotalMB-ttotalMB/adjPeriod*adjPeriod;
		ssDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec), IO Data %lld(MB), throughput %lld.%d(MBps)\n", __FUNCTION__, dma,
			iter, adjPeriod, ttotalMB, ttotalMB/adjPeriod, rem*100/adjPeriod);
	} else {
		ssDMA_PROutput("[%s](%d) %d iterations, time elapse %d(sec) < 1, IO Data %lld(MB)\n", __FUNCTION__, dma,
			iter, period, ttotalMB);
	}

ssDMA_Run_end_1:
//printk("(%d)(%d)where, where!!\n", dma, iter);
	if(ret)
		ssDMA_TestErrCnt++;
	ssDMA_CROutput("[%s](%d) %d iterations with error count=%d\n", __FUNCTION__, dma, iter, ssDMA_TestErrCnt+cb->ErrCnt);

	if(virAddrSrcOrig)
		iounmap(virAddrSrcOrig);
//	ssDMA_TestMapDel(cb->client);
//	ssNTB_TestMapAdd(cb->client);
//	mutex_lock(&dma_list_mutex);
//	dmaFreeDMAChan(cb->client);
//	dma_release_channel(cb->client);
	if(dmaChan!=NULL) {
		dma_release_channel(dmaChan);
	}
//	mutex_unlock(&dma_list_mutex);
	cb->task = NULL;
	return ret;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTB_SetPattern
* SUMMARY: Entry point to start stress test on NTB
*
* SCOPE:   Public
*
* DESCRIPTION:
*
* RETURNS:
*
* NOTES:
*
*/

void ssNTB_SetPattern(
	uint32_t	p0,
	uint32_t	p1,
	uint32_t	p2,
	uint32_t	p3)
{
	ssNTB_Patterns[ssNTB_PatternCnt++]=p0;
	ssNTB_Patterns[ssNTB_PatternCnt++]=p1;
	ssNTB_Patterns[ssNTB_PatternCnt++]=p2;
	ssNTB_Patterns[ssNTB_PatternCnt++]=p3;
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTBCPU_Start
* SUMMARY: Entry point to start stress test on NTB
*
* SCOPE:   Public
*
* DESCRIPTION:
*
* RETURNS:
*
* NOTES:
*
*/
int ssNTBCPU_Start(
	uint32_t	cpu,
//	phys_addr_t localAddrMB,
//	phys_addr_t remoteAddrMB,	because of param shortage <=5
	uint32_t	rangeSizeMB,
	uint32_t	blockSizeKB,
	uint32_t	iterations,
	uint32_t	pattern)
{
	struct ssNTB_TEST_CB *cb=&ssNTBCPU_Cb[cpu];

	cb->client    = cpu;
	cb->rangeSize = rangeSizeMB * MB;
	cb->blockSize = (blockSizeKB & 0xFFFF) * KB;
	cb->flags     = blockSizeKB & 0xFFFF0000;
	cb->iterations= iterations;
	cb->pattern   = pattern;
	cb->ErrCnt    = 0;

//	ssNTBCPU_TestMapAdd(cpu);
//	ssNTB_CROutput("[%s] NTB CPU (%d) stress test\n", __FUNCTION__, cpu);
//	ssNTB_CROutput("[%s] Range: %d(MB), Block Size: %d(KB)\n", __FUNCTION__, rangeSizeMB, blockSizeKB & 0xFFFF);
//	ssNTB_CROutput("[%s] Iterations: %d(times), Pattern: '0x%08x'\n", __FUNCTION__, iterations, pattern);

	/* Clean up PLX IPC resource before this test */

//	ssNTB_Init();

	cb->task = kthread_create(ssNTBCPU_Run, cb, "ssNTBCPUTask(%d)", cpu);
	if (IS_ERR(cb->task))
	{
		printk("Create PLX NTB CPU Task Failed on CPU(%d)\n", cpu);
		/* TODO */
		cb->ErrCnt++;
		return (-EFAULT);
	}

	kthread_bind(cb->task, cpu);

	wake_up_process(cb->task);

	return (0);
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTBDMA_Start
* SUMMARY: Entry point to start stress test on NTB
* 
* SCOPE:   Public 
*
* DESCRIPTION:
*
* RETURNS:
*
* NOTES:
*
*/
int ssNTBDMA_Start1(
	uint32_t	dma,
//	phys_addr_t localAddrMB,
//	phys_addr_t remoteAddrMB,	because of param shortage <=5
	uint32_t	rangeSizeMB,
	uint32_t	blockSizeKB,
	uint32_t	iterations,
	uint32_t	pattern)
{
	struct ssNTB_TEST_CB *cb=&ssNTBDMA_Cb[dma];

	cb->client    = dma;
	cb->rangeSize = rangeSizeMB * MB;
	cb->blockSize = (blockSizeKB & 0xFFFF) * KB;
	cb->flags     = blockSizeKB & 0xFFFF0000;
	cb->iterations= iterations;
	cb->pattern   = pattern;
	cb->ErrCnt    = 0;

//	ssNTBDMA_TestMapAdd(dma);
	/* Clean up PLX IPC resource before this test */

//	ssNTB_Init();

	cb->task = kthread_run(ssNTBDMA_Run, cb, "ssNTBDMATask(%d)", cb->client);
	if (IS_ERR(cb->task))
	{
		printk("Create PLX NTB Task Failed on DMA(%d)\n", cb->client);
		/* TODO */
		cb->ErrCnt++;
		return (-EFAULT);
	}

	return (0);
}

/*******************************************************************************
* PROCEDURE
*
* NAME:    ssNTBDMA_Start
* SUMMARY: Entry point to start stress test on NTB
*
* SCOPE:   Public
*
* DESCRIPTION:
*
* RETURNS:
*
* NOTES:
*
*/
int ssDMA_Start(
	uint32_t	dma,
//	phys_addr_t localAddrMB,
//	phys_addr_t remoteAddrMB,	because of param shortage <=5
	uint32_t	rangeSizeMB,
	uint32_t	blockSizeKB,
	uint32_t	iterations,
	uint32_t	pattern)
{
	struct ssNTB_TEST_CB *cb=&ssDMA_Cb[dma & 0xF];

	cb->client	= dma & 0xF;
	cb->maxSize	= ssDMA_MaxRange * MB;
	cb->localAddr	= (phys_addr_t)ssDMA_MapBase * MB + (cb->maxSize)*((dma&0xF0)>>4);
	cb->rangeSize	= rangeSizeMB * MB;
	cb->blockSize	= (blockSizeKB & 0xFFFF) * KB;
	cb->flags	= blockSizeKB & 0xFFFF0000;
	cb->iterations	= iterations;
	cb->pattern	= pattern;
	cb->ErrCnt	= 0;

//	ssDMA_TestMapAdd(cb->client);

	ssDMA_CROutput("DMA %d, IO Address 0x%014llx, Range 0x%x\n",
	       cb->client, cb->localAddr, cb->rangeSize);
	if(cb->localAddr >12*1024*1024*1024l)
		return (-EFAULT);
//	ssNTB_TestMapDel(dma);
	/* Clean up PLX IPC resource before this test */

	cb->task = kthread_run(ssDMA_Run, cb, "ssDMATask(%d)", cb->client);
	if (IS_ERR(cb->task))
	{
		printk("Create DMA Task Failed on DMA(%d)\n", cb->client);
		/* TODO */
		cb->ErrCnt++;
		return (-EFAULT);
	}

	return (0);
}

#define	ssNTB_CROutput1(args...) {	\
	int r;\
	sprintf(buf, args);		\
	r=filp->f_op->write(filp,buf,strlen(buf),&filp->f_pos);\
}

void ssDMA_HWSts(void)
{
//	uint32_t	reg;
	char	buf[128];
	struct file *filp;
	mm_segment_t oldfs;
	char	*cname = mdcname;
	uint32_t	*errCnt=&ssDMA_TestErrCnt;

	oldfs=get_fs();

	set_fs(KERNEL_DS);

	filp=filp_open(cname,O_CREAT|O_RDWR|O_APPEND,0777);
	if(IS_ERR(filp))
        {
		printk("[%s] File Open Error:%s\n", __FUNCTION__, cname);
		return;
        }

//	ssNTB_CROutput1("[%s] ----------------------------------------------------------------\n", __FUNCTION__);
//	reg = PLX_VIRT_MMIO_READ(0x04);if(reg & CmdStsErr) (*errCnt)++;
//	ssNTB_CROutput1("[%s] NTB Virtual I/F command/status(0x04):                0x%08x%s\n", __FUNCTION__, reg, (reg & CmdStsErr)?"(CHK)":"");


	ssNTB_CROutput1("[%s] Memory Bus stress test by DMA generates %d errors\n", __FUNCTION__, (*errCnt));

	//printk("\nWrite :%xh bytes Offset=%xh\n",r,(unsigned)filp->f_pos);

	filp_close(filp,NULL);

	set_fs(oldfs);
}

#define	UncorrectableErrSts	(BIT(12)|BIT(16)|BIT(17)|BIT(18)|BIT(19)|BIT(20)|BIT(22)|BIT(23))
#define CorrectableErrSts	(BIT(13)|BIT(14)|BIT(15))
#define CmdStsErr		(BIT(30)|BIT(31))
#define StsCtlErr		(BIT(16)|BIT(17)|BIT(18)|BIT(19))

void ssDMA_Stop(void)
{
	uint32_t	i;
	int	ret;
	struct ssNTB_TEST_CB *cb;

	for(i=0; i<DMA_ChCnt; i++) {
		cb=&ssDMA_Cb[i];
		if(cb->task != NULL) {
			ret = kthread_stop(cb->task);
//printk("%s kthread_should_stop is SENT\n", __FUNCTION__);
			cb->task = NULL;
		}
		ssDMA_TestErrCnt += cb->ErrCnt;
	}

	ssDMA_HWSts();
//	ssunlock_NTB_RunMap();
}

void ssNTB_HWSts(int opby)	// opby, 0: CPU, 1:DMA
{
	uint32_t	reg;
	char	buf[128];
	struct file *filp;
	mm_segment_t oldfs;
	char	*cname = (opby)? ndcname:nccname;
	uint32_t	*errCnt=(opby)? &ssNTBDMA_TestErrCnt:&ssNTBCPU_TestErrCnt;
	
	oldfs=get_fs();

	set_fs(KERNEL_DS);

	filp=filp_open(cname,O_CREAT|O_RDWR|O_APPEND,0777);
	if(IS_ERR(filp))
        {
		printk("[%s] File Open Error:%s\n", __FUNCTION__, cname);
		return;
        }

	ssNTB_CROutput1("[%s] ----------------------------------------------------------------\n", __FUNCTION__);
	reg = PLX_VIRT_MMIO_READ(0x04);if(reg & CmdStsErr) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Virtual I/F command/status(0x04):                0x%08x%s\n", __FUNCTION__, reg, (reg & CmdStsErr)?"(CHK)":"");
	reg = PLX_VIRT_MMIO_READ(0x70);if(reg & StsCtlErr) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Virtual I/F control/status(0x70):                0x%08x%s\n", __FUNCTION__, reg, (reg & StsCtlErr)?"(CHK)":"");
	reg = PLX_VIRT_MMIO_READ(0x78);
	ssNTB_CROutput1("[%s] NTB Virtual I/F link control and status(0x78):       0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFB8);if(reg & UncorrectableErrSts) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Virtual I/F Uncorrectable Error status(0xFB8):   0x%08x%s\n", __FUNCTION__, reg, (reg & UncorrectableErrSts)?"(CHK)":"");
	reg = PLX_VIRT_MMIO_READ(0xFBC);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Uncorrectable Error Mask(0xFBC):     0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFC0);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Uncorrectable Error Severity(0xFC0): 0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFC4);if(reg & CorrectableErrSts) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Virtual I/F Correctable Error Status(0xFC4):     0x%08x%s\n", __FUNCTION__, reg, (reg & CorrectableErrSts)?"(CHK)":"");
	reg = PLX_VIRT_MMIO_READ(0xFC8);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Correctable Error Mask(0xFC8):       0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFCC);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Advanced Error Cap. and Ctrl(0xFCC): 0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFE0);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Link Error Status Virtual(0xFE0):    0x%08x\n", __FUNCTION__, reg);
	reg = PLX_VIRT_MMIO_READ(0xFE4);
	ssNTB_CROutput1("[%s] NTB Virtual I/F Link Error Mask Virtual(0xFE4):      0x%08x\n", __FUNCTION__, reg);
	ssNTB_CROutput1("[%s] ----------------------------------------------------------------\n", __FUNCTION__);

	reg = PLX_LINK_MMIO_READ(0x04);if(reg & CmdStsErr) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Link I/F command/status(0x04):                0x%08x%s\n", __FUNCTION__, reg, (reg & CmdStsErr)?"(CHK)":"");
	reg = PLX_LINK_MMIO_READ(0x70);if(reg & StsCtlErr) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Link I/F control/status(0x70):                0x%08x%s\n", __FUNCTION__, reg, (reg & StsCtlErr)?"(CHK)":"");
	reg = PLX_LINK_MMIO_READ(0x78);
	ssNTB_CROutput1("[%s] NTB Link I/F Link control/status(0x78):           0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0x98);
	ssNTB_CROutput1("[%s] NTB Link I/F Link control/status 2(0x98):         0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0x160);
	ssNTB_CROutput1("[%s] NTB Link I/F VC0 Resource Status(0x160):          0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0x720);
	ssNTB_CROutput1("[%s] NTB Link I/F ECC Error Check Disable(0x720):      0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0xFB8);if(reg & UncorrectableErrSts) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Link I/F Uncorrectable Error status(0xFB8):   0x%08x%s\n", __FUNCTION__, reg, (reg & UncorrectableErrSts)?"(CHK)":"");
	reg = PLX_LINK_MMIO_READ(0xFBC);
	ssNTB_CROutput1("[%s] NTB Link I/F Uncorrectable Error Mask(0xFBC):     0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0xFC0);
	ssNTB_CROutput1("[%s] NTB Link I/F Uncorrectable Error Severity(0xFC0): 0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0xFC4);if(reg & CorrectableErrSts) (*errCnt)++;
	ssNTB_CROutput1("[%s] NTB Link I/F Correctable Error Status(0xFC4):     0x%08x%s\n", __FUNCTION__, reg, (reg & CorrectableErrSts)?"(CHK)":"");
	reg = PLX_LINK_MMIO_READ(0xFC8);
	ssNTB_CROutput1("[%s] NTB Link I/F Correctable Error Mask(0xFC8):       0x%08x\n", __FUNCTION__, reg);
	reg = PLX_LINK_MMIO_READ(0xFCC);
	ssNTB_CROutput1("[%s] NTB Link I/F Advanced Error Cap. and Ctrl(0xFCC): 0x%08x\n", __FUNCTION__, reg);
	ssNTB_CROutput1("[%s] ----------------------------------------------------------------\n", __FUNCTION__);
	
	ssNTB_CROutput1("[%s] NTB IO by %s generates %d errors\n", __FUNCTION__, opby?"DMA":"CPU", (*errCnt));

	//printk("\nWrite :%xh bytes Offset=%xh\n",r,(unsigned)filp->f_pos);

	filp_close(filp,NULL);

	set_fs(oldfs);
}


void ssNTBCPU_Stop(void)
{
	uint32_t	i;
	int	ret;
	struct ssNTB_TEST_CB *cb;

	for(i=0; i<ssCPU_Cores; i++) {
		cb=&ssNTBCPU_Cb[i];
		
		if(cb->task != NULL) {
			ret = kthread_stop(cb->task);
//printk("%s kthread_should_stop is SENT\n", __FUNCTION__);
			cb->task = NULL;
		}
		ssNTBCPU_TestErrCnt+=cb->ErrCnt;
	}

//	sslock_NTB_RunMap();
	ssNTB_HWSts(0);
//	ssunlock_NTB_RunMap();
}

void ssNTBDMA_Stop(void)
{
	uint32_t	i;
	int	ret;
	struct ssNTB_TEST_CB *cb;

	for(i=0; i<DMA_ChCnt; i++) {
		cb=&ssNTBDMA_Cb[i];
		
		if(cb->task != NULL) {
			ret = kthread_stop(cb->task);
			cb->task = NULL;
		}
		ssNTBDMA_TestErrCnt += cb->ErrCnt;
	}

//	sslock_NTB_RunMap();
	ssNTB_HWSts(1);
//	ssunlock_NTB_RunMap();
}

static ssize_t OpByCPU_err_counter_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
	return sprintf(buf, "%d", ssNTBCPU_TestErrCnt);
}

static ssize_t OpByDMA_err_counter_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
	return sprintf(buf, "%d", ssNTBDMA_TestErrCnt);
//uint32_t	ssDMA_TestErrCnt=0;
}

static ssize_t DMA_err_counter_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf)
{
	return sprintf(buf, "%d", ssDMA_TestErrCnt);
}


static struct kobj_attribute OpByCPU_err_counter = __ATTR(OpByCPU_err_counter, 0440, OpByCPU_err_counter_show, NULL);
static struct kobj_attribute OpByDMA_err_counter = __ATTR(OpByDMA_err_counter, 0440, OpByDMA_err_counter_show, NULL);
static struct kobj_attribute DMA_err_counter = __ATTR(DMA_err_counter, 0440, DMA_err_counter_show, NULL);

static struct attribute *ntbStressAttrs[] = {
	&OpByCPU_err_counter.attr,
	&OpByDMA_err_counter.attr,
	&DMA_err_counter.attr,
	NULL,
};

static struct attribute_group ntbStressAttrGroup = {
	.attrs = ntbStressAttrs,
};

static struct kobject *p_ntbStress_kobj = NULL;
static int __init ntbStressInit(void)
{
	int err=0;

	p_ntbStress_kobj = kobject_create_and_add("ntbStress", firmware_kobj);
	if(NULL == p_ntbStress_kobj) {
		goto errExit;
	}

	err = sysfs_create_group(p_ntbStress_kobj, &ntbStressAttrGroup);
	if(err) {
		kobject_put(p_ntbStress_kobj);
		goto errExit;
	}

	printk(KERN_INFO "NTB Stress Interface Driver");
	return 0;

	errExit:
		return -ENODEV;
}

static void __exit ntbStressExit(void)
{
	if(p_ntbStress_kobj) {
		sysfs_remove_group(p_ntbStress_kobj, &ntbStressAttrGroup);
		kobject_put(p_ntbStress_kobj);
	}
	return;
}

module_init(ntbStressInit);
module_exit(ntbStressExit);


MODULE_AUTHOR( "merck.hung@netapp.com" );
MODULE_DESCRIPTION( "NetApp NTB Stress Code" );
MODULE_LICENSE( "GPL" );
MODULE_VERSION( "0.1" );