This thread has been locked.

If you have a related question, please click the "Ask a related question" button in the top right corner. The newly created question will be automatically linked to this question.

SRIO 多核发送Doorbell包失败



大家好,

************************************************************************************************

我想实现的是多核对外发送 SRIO Doorbell包,遇到了LSU TX error的问题。如下所述

************************************************************************************************

SRIO配置如下:

UInt8           isAllocated;
Srio_DrvConfig  drvCfg;

System_printf ("Core Num = %d\n", corenum);

/* Initialize the SRIO Driver Configuration. */
memset ((Void *)&drvCfg, 0, sizeof(Srio_DrvConfig));

/* Initialize the OSAL */
if (Osal_dataBufferInitMemory(SRIO_MAX_MTU) < 0)
{
	System_printf ("Error: Unable to initialize the OSAL. \n");
	return -1;
}

/********************************************************************************
 * The SRIO Driver Instance is going to be created with the following properties:
 * - Driver Managed
 * - Interrupt Support (Pass the Rx Completion Queue as NULL)
 ********************************************************************************/
/* Setup the SRIO Driver Managed Configuration. */
drvCfg.bAppManagedConfig = FALSE;

/* Driver Managed: Receive Configuration */
drvCfg.u.drvManagedCfg.bIsRxCfgValid             = 1;
drvCfg.u.drvManagedCfg.rxCfg.rxMemRegion         = Qmss_MemRegion_MEMORY_REGION1;//Qmss_MemRegion_MEMORY_REGION1;//Qmss_MemRegion_MEMORY_REGION0
drvCfg.u.drvManagedCfg.rxCfg.numRxBuffers        = 4;
drvCfg.u.drvManagedCfg.rxCfg.rxMTU               = SRIO_MAX_MTU;

/* Accumulator Configuration. */
{
	int32_t coreToQueueSelector[4];

  /* This is the table which maps the core to a specific receive queue. */
	coreToQueueSelector[0] = 736;//736;//704
	coreToQueueSelector[1] = 737;
	coreToQueueSelector[2] = 738;
	coreToQueueSelector[3] = 739;

	/* Since we are programming the accumulator we want this queue to be a HIGH PRIORITY Queue */
	drvCfg.u.drvManagedCfg.rxCfg.rxCompletionQueue = Qmss_queueOpen (Qmss_QueueType_HIGH_PRIORITY_QUEUE,
																	 coreToQueueSelector[corenum], &isAllocated);
	System_printf ("Test Point 4\n");

	if (drvCfg.u.drvManagedCfg.rxCfg.rxCompletionQueue < 0)
	{
		System_printf ("Error: Unable to open the SRIO Receive Completion Queue\n");
		return 0;
	}

	/* Accumulator Configuration is VALID. */
	drvCfg.u.drvManagedCfg.rxCfg.bIsAccumlatorCfgValid = 1;

	/* Accumulator Configuration. */
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.channel             = corenum+16;//corenum;//1
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.command             = Qmss_AccCmd_ENABLE_CHANNEL;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.queueEnMask         = 0;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.queMgrIndex         = coreToQueueSelector[corenum];
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.maxPageEntries      = 2;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.timerLoadCount      = 0;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.interruptPacingMode = Qmss_AccPacingMode_LAST_INTERRUPT;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.listEntrySize       = Qmss_AccEntrySize_REG_D;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.listCountMode       = Qmss_AccCountMode_ENTRY_COUNT;
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.multiQueueMode      = Qmss_AccQueueMode_SINGLE_QUEUE;//Qmss_AccQueueMode_SINGLE_QUEUEQmss_AccQueueMode_MULTI_QUEUE

	/* Initialize the accumulator list memory */
	memset ((Void *)&gHiPriAccumList0[corenum], 0, sizeof(gHiPriAccumList0));
	drvCfg.u.drvManagedCfg.rxCfg.accCfg.listAddress = l2_global_address((UInt32)&gHiPriAccumList0[corenum]);
}
/* Driver Managed: Transmit Configuration */
drvCfg.u.drvManagedCfg.bIsTxCfgValid             = 1;
drvCfg.u.drvManagedCfg.txCfg.txMemRegion         = Qmss_MemRegion_MEMORY_REGION1;//Qmss_MemRegion_MEMORY_REGION0
drvCfg.u.drvManagedCfg.txCfg.numTxBuffers        = 4;
drvCfg.u.drvManagedCfg.txCfg.txMTU               = SRIO_MAX_MTU;

/* Start the Driver Managed SRIO Driver. */
hDrvManagedSrioDrv = Srio_start(&drvCfg);
if (hDrvManagedSrioDrv == NULL)
{
	System_printf ("Error(Core %d): SRIO Driver failed to start\n", corenum);
	return -1;
}

/* Enable Time Stamp Counter */
CSL_tscEnable();
int32_t	eventId;
UInt8 LSU_Int_HostIntNum;
/* Get the CSL SRIO Handle. */
hSrioCSL = CSL_SRIO_Open (0);
if (hSrioCSL == NULL)
	return -1;

//映射LSU TX完成中断
LSU_Int_HostIntNum = 8+corenum+16*corenum;
CSL_SRIO_DisableInterruptPacing (hSrioCSL, corenum);
CSL_SRIO_RouteLSUInterrupts (hSrioCSL, corenum, corenum);
CpIntc_dispatchPlug(CSL_INTC0_INTDST0+corenum, (CpIntc_FuncPtr)myDioTxCompletionIsr, (UArg)hDrvManagedSrioDrv, TRUE);
CpIntc_mapSysIntToHostInt(0, CSL_INTC0_INTDST0+corenum, LSU_Int_HostIntNum);
CpIntc_enableHostInt(0, LSU_Int_HostIntNum);
CpIntc_enableSysInt(0, CSL_INTC0_INTDST0+corenum);
eventId = CpIntc_getEventId(LSU_Int_HostIntNum);
EventCombiner_dispatchPlug (eventId, CpIntc_dispatch, LSU_Int_HostIntNum, TRUE);

//映射LSU TX失败中断
LSU_Int_HostIntNum = 12+corenum+16*corenum;
CSL_SRIO_DisableInterruptPacing (hSrioCSL, corenum+4);
CSL_SRIO_RouteLSUInterrupts (hSrioCSL, corenum+16, 4+corenum);
CpIntc_dispatchPlug(CSL_INTC0_INTDST0+corenum+4, (CpIntc_FuncPtr)myDioTxErrorIsr, (UArg)hDrvManagedSrioDrv, TRUE);
CpIntc_mapSysIntToHostInt(0, CSL_INTC0_INTDST0+corenum+4, LSU_Int_HostIntNum);
CpIntc_enableHostInt(0, LSU_Int_HostIntNum);
CpIntc_enableSysInt(0, CSL_INTC0_INTDST0+corenum+4);
eventId = CpIntc_getEventId(LSU_Int_HostIntNum);
EventCombiner_dispatchPlug (eventId, CpIntc_dispatch, LSU_Int_HostIntNum, TRUE);

EventCombiner_dispatchPlug (20, (CpIntc_FuncPtr)myDorrbellFunc, (UArg)hDrvManagedSrioDrv, TRUE);

return 0;

然后SRIO发送Doorbell的代码如下:

bindInfo.dio.doorbellValid  = 0;
bindInfo.dio.intrRequest    = 1;
bindInfo.dio.supInt         = 0;
bindInfo.dio.xambs          = 0;
bindInfo.dio.priority       = 0;
//bindInfo.dio.outPortID      = 3;//for 1X port3
bindInfo.dio.outPortID      = 0;//for 4X port0
bindInfo.dio.idSize         = 0;
bindInfo.dio.srcIDMap       = corenum;
bindInfo.dio.hopCount       = 0;
bindInfo.dio.doorbellReg    = 0;
bindInfo.dio.doorbellBit    = 0;

if (Srio_sockBind_DIO (srioSocket[corenum], &bindInfo) < 0)
{
	System_printf ("Error: Binding the SIO socket failed.\n");
	return -1;
}

/*********************************************************************************
* Run multiple iterations of the example to ensure multiple data transfers work
*********************************************************************************/
if(dio_ftype == Srio_Ftype_DOORBELL)
{
	//见Keystone SRIO手册的Figure2-22
	if(corenum == 0)
		doorbellInfo = SRIO_SET_DBELL_INFO(0,6);
	else if(corenum == 1)
		doorbellInfo = SRIO_SET_DBELL_INFO(1,6);
	else if(corenum == 2)
		doorbellInfo = SRIO_SET_DBELL_INFO(2,6);
	else if(corenum == 3)
		doorbellInfo = SRIO_SET_DBELL_INFO(3,6);
	else
	{
		System_printf("Doorbell Info set error!\n");
		return -1;
	}

	/* Populate the DIO Address Information where the data is to be sent. */
	to.dio.rapidIOMSB    = 0x0;
	to.dio.rapidIOLSB    = (uint32_t)READ_BASE_ADDR;
	to.dio.dstID         = DEVICE_DST_ID;		//DEVICE_ID2_16BIT;
	to.dio.ttype         = dio_ttype;
	to.dio.ftype         = dio_ftype;

	if (Srio_sockSend_DIO (srioSocket[corenum],(Srio_DrvBuffer)doorbellInfo, 0, (Srio_SockAddrInfo*)&to) < 0)
	{
		System_printf ("DIO Socket Example Failed\n");
		return -1;
	}
	while(! srioLsuIsrServiced);

	/* Read the completion code filled by the ISR */
	compCode = 0xFF;
	if (Srio_getSockOpt(srioSocket[corenum], Srio_Opt_DIO_READ_SOCK_COMP_CODE, &compCode, sizeof(uint8_t)) < 0)
	{
		System_printf ("Error: Unable to read the completion code in socket\n");
		return -1;
	}
	//System_printf("Completition Code is %d\n",compCode);

	CSL_SRIO_ClearDoorbellPendingInterrupt (hSrioCSL, corenum, 0xFFFF);
	CSL_SRIO_ClearLSUPendingInterrupt (hSrioCSL, 0xFFFFFFFF, 0xFFFFFFFF);
	srioLsuIsrServiced = 0;
	System_printf ("SRIO Doorbell Operation OK!\n");
}

************************************************************************************************

遇到的问题是:

Core0一切正常,能够对外发送Doorbell包,但是Core1~Core3没发送一个Doorbell包,就卡在

	while(! srioLsuIsrServiced);

就是说Core1~Core3在发送Doorbell包的时候,LSU没有正常进入TXComletionIsr。

************************************************************************************************

经过测试,4个Core加载代码,同时对外发送Doorbell包,结果如下

1. Core0上:每发送一次Doorbell包,LSU3_stat0~LSU3_stat3依次0、1变化,

Core0能够进入ICS0 – SRCID0, Transaction complete, No Errors (Posted/Non-posted)中断

2. Core1上:每发送一次Doorbell包,LSU0_stat0~LSU0_stat3依次2、3变化

Core1进入ICS17 – SRCID1, Transaction complete with error中断

3. Core2上:每发送一次Doorbell包,LSU1_stat0~LSU1_stat3依次2、3变化

Core2进入ICS18 – SRCID2, Transaction complete with error中断

4. Core3上:每发送一次Doorbell包,LSU2_stat0~LSU2_stat3依次2、3变化

Core3进入ICS19 – SRCID3, Transaction complete with error中断

顺便贴上LSUx_staty的寄存器值表格

应该是Core1~Core3在发送Doorbell包的时候,遇到了0b001 Timeout问题,所以无法进入myDioTxCompletionIsr中断处理函数,而进入了myDioTxErrorIsr中断处理函数。

可是这样的代码在Core0上运行无误,而且我测试如果4个Core对外发送的不是Doorbell而是SWRITE包,也不会发生这种Core1~Core3 LSU Transaction Timeout的问题。

************************************************************************************************

希望大家能帮我分析一下~~非常感谢

  • 你试试修改一下bindinfo里的关于doorbell的几个域看看是否有变化

  • 好像每个Core都有自己的DeviceID,我们的板子DSP是连接IDT SRIO Switch的,SWITCH与DSP连接的端口只映射了Core0的DeviceID,而没有映射Core1·~Core3的DeviceID,于是Core1~Core3发送Doorbell包的时候,Switch无法正常将目标器件的Response包路由回DSP,所以DSP Core1~Core3无法正常发送带Response的SRIO包,例如Doorbell、NWRITE_R等。

    我的解决方法是:把Core0~Core3的DeviceID都设置为同一个DeviceID,这样Switch就能正常路由了。

    当然理论上,如果修改Switch的路由表,添加Core1~Core3的不同的DeviceID,应该也能解决,但是感觉有点麻烦。

    有点疑问:4个Core都用同一个DeviceID是否会出现问题?目前测试倒是没有什么问题

  •   你这个问题和DEVICE ID  只占了以点关系

       Timeout 是由于 没有收到响应包的原因, 为什么CORE1 发的时候能收到 而其他收不到呢 , 都是一个DSP出去的 ,怎么就差这么多呢 。其实你忽略 了一个特别重要的地方就是SRCIDMAP

      你发现只有SRCIDMAP 为 0  的包能发送成功   ,那是因为  SRIO 在物理层中 会自动 把 Base device id 填入 SRIO 发出去的报文头,  resp 包也会发往对应的base device id 。

      而当使用其他SRCID MAP时,  如果你没有填充对应的SRCIDMAP 对应的ID时  默认是0 哦 ,你用这些SRCID_MAP发出去的包的响应 都会发往 destid 为 0 的设备,如果你的设备恰巧不是  0 的话当然会收不到了啦 ~~~~~~~~~

      被自己的聪明才智折服了

                                                                                   hello ,fantasy