TDA4VM: after add vxHarrisCornersNode, the program is block in 13 frame

bilin Dai

Part Number: TDA4VM

program data flow design is above, without the optical flow node, program is work normaly.

But when i add the opticalflow node, the program will block in the 3rd frame. my program is modify from the app_dof example.

opticalflow node cpp file is blow.

Fullscreen

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
vx_status app_init_opticalflow(vx_context context, OpticalFlowObj *opticalflowObj, char *objName)
{
    vx_status status = VX_SUCCESS;
    // tivx_dmpac_dof_params_init(&opticalflowObj->dof_params);
    vx_float32 optical_eps_val = 0.001;
    vx_float32 optical_num_iter_val = 100;
    vx_float32 optical_use_estimations_val = 1;
    opticalflowObj->eps = vxCreateScalar(context, VX_TYPE_FLOAT32, &optical_eps_val);
    opticalflowObj->num_iter = vxCreateScalar(context, VX_TYPE_UINT32, &optical_num_iter_val);
    opticalflowObj->use_estimations = vxCreateScalar(context, VX_TYPE_BOOL, &optical_use_estimations_val);
    if(opticalflowObj->enable_temporal_predicton_flow_vector)
    {
        vx_array corners_exemplar;
        corners_exemplar = vxCreateArray(context, VX_TYPE_KEYPOINT, opticalflowObj->num_corners);
        opticalflowObj->out_corners = vxCreateArray(context, VX_TYPE_KEYPOINT, opticalflowObj->num_corners);
        // APP_ASSERT_VALID_REF(corners_exemplar);
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

vx_status app_init_opticalflow(vx_context context, OpticalFlowObj *opticalflowObj, char *objName)
{
    vx_status status = VX_SUCCESS;

    // tivx_dmpac_dof_params_init(&opticalflowObj->dof_params);
    vx_float32 optical_eps_val = 0.001;
    vx_float32 optical_num_iter_val = 100;
    vx_float32 optical_use_estimations_val = 1;

    opticalflowObj->eps = vxCreateScalar(context, VX_TYPE_FLOAT32, &optical_eps_val);
    opticalflowObj->num_iter = vxCreateScalar(context, VX_TYPE_UINT32, &optical_num_iter_val);
    opticalflowObj->use_estimations = vxCreateScalar(context, VX_TYPE_BOOL, &optical_use_estimations_val);

    if(opticalflowObj->enable_temporal_predicton_flow_vector)
    {

        vx_array corners_exemplar;

        corners_exemplar = vxCreateArray(context, VX_TYPE_KEYPOINT, opticalflowObj->num_corners);
        opticalflowObj->out_corners = vxCreateArray(context, VX_TYPE_KEYPOINT, opticalflowObj->num_corners);
        // APP_ASSERT_VALID_REF(corners_exemplar);

        opticalflowObj->corners_delay = vxCreateDelay(context, (vx_reference)corners_exemplar, 2);
        APP_ASSERT_VALID_REF(opticalflowObj->corners_delay);

        opticalflowObj->corners_in = (vx_array)vxGetReferenceFromDelay(opticalflowObj->corners_delay, -1);
        APP_ASSERT_VALID_REF(opticalflowObj->corners_in);
        vxSetReferenceName((vx_reference)opticalflowObj->corners_in, "CornersIn");

        opticalflowObj->corners_out = (vx_array)vxGetReferenceFromDelay(opticalflowObj->corners_delay,  0);
        APP_ASSERT_VALID_REF(opticalflowObj->corners_out);
        vxSetReferenceName((vx_reference)opticalflowObj->corners_out, "CornersOut");
        vxReleaseArray(&corners_exemplar);
    }
    else
    {
        opticalflowObj->corners_delay =  NULL;
        opticalflowObj->corners_in = NULL;
        opticalflowObj->corners_out = vxCreateArray(context, VX_TYPE_KEYPOINT, opticalflowObj->num_corners);
        APP_ASSERT_VALID_REF(opticalflowObj->corners_out);
        vxSetReferenceName((vx_reference)opticalflowObj->corners_out, "CornersOut");

    }

    return status;
}

void app_deinit_opticalflow(OpticalFlowObj *opticalflowObj)
{

  if(opticalflowObj->enable_temporal_predicton_flow_vector)
  {
    vxReleaseDelay(&opticalflowObj->corners_delay); 
  }
  else
  {
    vxReleaseImage(&opticalflowObj->corners_out);
  }

}

void app_delete_opticalflow(OpticalFlowObj *opticalflowObj)
{

  if(opticalflowObj->node != NULL)
  {
    vxReleaseNode(&opticalflowObj->node);
  }else{

  }
 
}


vx_status app_create_graph_opticalflow(vx_graph graph, OpticalFlowObj *opticalflowObj,
                                   vx_pyramid old_images,
                                   vx_pyramid new_images,
                                   vx_array old_points,
                                   vx_array new_points_estimates,
                                   vx_scalar eps,
                                   vx_scalar num_iter,
                                   vx_scalar use_estimations)
{
  // printf("start set reference optic\n");

  vx_status status = VX_SUCCESS;

  opticalflowObj->node = vxOpticalFlowPyrLKNode(
      graph,
      old_images, new_images,
      old_points, new_points_estimates, opticalflowObj->out_corners,
      VX_TERM_CRITERIA_BOTH, eps, num_iter, use_estimations, opticalflowObj->winSize);
  APP_ASSERT_VALID_REF(opticalflowObj->node);
  // status = vxSetNodeTarget(opticalflowObj->node, VX_TARGET_STRING, TIVX_TARGET_DMPAC_DOF);
  // APP_ASSERT(status == VX_SUCCESS);
  vxSetReferenceName((vx_reference)opticalflowObj->node, "OpticalFlow");
  // printf("set reference optic\n");

  return status;
}

create node in main file

Fullscreen

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
    if (status == VX_SUCCESS)
    {
        status = app_create_graph_opticalflow(obj->graph, &obj->opticalflowObj,
                                   obj->pyramidObj.pyr_ref, obj->pyramidObj.pyr_cur, 
                                   obj->harrisObj.har_ref,
                                   obj->harrisObj.har_cur,
                                   obj->opticalflowObj.eps,
                                   obj->opticalflowObj.num_iter,
                                   obj->opticalflowObj.use_estimations);
    }
    
    
    if (obj->opticalflowObj.enable_temporal_predicton_flow_vector && (status == VX_SUCCESS))
    {
        status = vxRegisterAutoAging(obj->graph, obj->opticalflowObj.corners_delay);
    }
    
    
    if (status == VX_SUCCESS)
    {
        status = tivxSetNodeParameterNumBufByIndex(obj->opticalflowObj.node, 4, num_buf);
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

    
    if (status == VX_SUCCESS)
    {
        status = app_create_graph_opticalflow(obj->graph, &obj->opticalflowObj,
                                   obj->pyramidObj.pyr_ref, obj->pyramidObj.pyr_cur, 
                                   obj->harrisObj.har_ref,
                                   obj->harrisObj.har_cur,
                                   obj->opticalflowObj.eps,
                                   obj->opticalflowObj.num_iter,
                                   obj->opticalflowObj.use_estimations);
    }
    
    
    if (obj->opticalflowObj.enable_temporal_predicton_flow_vector && (status == VX_SUCCESS))
    {
        status = vxRegisterAutoAging(obj->graph, obj->opticalflowObj.corners_delay);
    }
    
    
    if (status == VX_SUCCESS)
    {
        status = tivxSetNodeParameterNumBufByIndex(obj->opticalflowObj.node, 4, num_buf);
    }

the program will block in the 3rd frame when the pipeline > 0

Fullscreen

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
static vx_status app_run_graph(AppObj *obj)
{
    char input_file_name[APP_MAX_FILE_PATH];
    char output_file_name_flow_img[APP_MAX_FILE_PATH];
    char output_file_name_conf_img[APP_MAX_FILE_PATH];
    uint32_t curFileNum;
    int32_t outputFileNum;
    uint32_t iterations = 1;
    vx_status status = VX_SUCCESS;
    uint32_t num_buf;
    num_buf = MAX_NUM_BUF;
    int32_t pipeline = -num_buf;
    int32_t enqueueCnt = 0;
    vx_int32 img_array_idx = -1, img_harray_idx = -1;
    /* create output directory is not already existing */
    mkdir(obj->output_file_path, S_IRWXU | S_IRWXG | S_IRWXO);
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

static vx_status app_run_graph(AppObj *obj)
{
    char input_file_name[APP_MAX_FILE_PATH];
    char output_file_name_flow_img[APP_MAX_FILE_PATH];
    char output_file_name_conf_img[APP_MAX_FILE_PATH];
    uint32_t curFileNum;
    int32_t outputFileNum;
    uint32_t iterations = 1;
    vx_status status = VX_SUCCESS;

    uint32_t num_buf;
    num_buf = MAX_NUM_BUF;

    int32_t pipeline = -num_buf;
    int32_t enqueueCnt = 0;

    vx_int32 img_array_idx = -1, img_harray_idx = -1;

    /* create output directory is not already existing */
    mkdir(obj->output_file_path, S_IRWXU | S_IRWXG | S_IRWXO);

    /* run pyramid only for 1st frame */
    status = app_run_pyramid_for_first_frame(obj);

    /* run DOF for 2nd frame onwards */

    vx_uint32 test_counter = 0;
    for (curFileNum = obj->start_fileno + 1; curFileNum <= obj->end_fileno; curFileNum++)
    {
        // TicToc t_process;
        if (obj->stop_task)
        {
            break;
        }

        appPerfPointBegin(&obj->total_perf);

        snprintf(input_file_name, APP_MAX_FILE_PATH, "%s/%d.%s",
                 obj->input_file_path,
                 curFileNum,
                 obj->in_file_ext);

        printf(" %d of %d: Loading [%s] ...\n", curFileNum, obj->end_fileno, input_file_name);                 

        // snprintf(input_file_name, APP_MAX_FILE_PATH, "%s/%s%05d%s.%s",
        //     obj->input_file_path,
        //     obj->input_file_prefix,
        //     curFileNum,
        //     obj->input_file_postfix,
        //     obj->in_file_ext
        //     );
        if (pipeline < 0)
        {
            /* Enqueue outpus */
            // if (status == VX_SUCCESS)
            // {
            //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->flow_vector_field_img_graph_parameter_index, (vx_reference *)&obj->dofvizObj.flow_vector_field_img[enqueueCnt], 1);
            //     if (status == VX_SUCCESS)
            //     {
            //         status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->confidence_img_graph_parameter_index, (vx_reference *)&obj->dofvizObj.confidence_img[enqueueCnt], 1);
            //     }
            // }

            if (0 == obj->is_interactive)
            {
                printf("1 %d of %d: Loading [%s] ...\n", curFileNum, obj->end_fileno, input_file_name);
            }
            appPerfPointBegin(&obj->fileio_perf);
            /* Read input */
            if (status == VX_SUCCESS)
            {
                status = app_load_vximage_from_file(obj->pyramidObj.input_img[enqueueCnt], input_file_name, obj->in_file_format);
            }
            // if (status == VX_SUCCESS)
            // {
            //     status = app_load_vximage_from_file(obj->harrisObj.input_img[enqueueCnt], input_file_name, obj->in_file_format);
            // }

            appPerfPointEnd(&obj->fileio_perf);

            /* Enqueue input - start execution */
            if (status == VX_SUCCESS)
            {
                status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->input_img_graph_parameter_index, (vx_reference *)&obj->pyramidObj.input_img[enqueueCnt], 1);
            }
            // if (status == VX_SUCCESS)
            // {
            //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->input_img_harris_graph_parameter_index, (vx_reference *)&obj->harrisObj.input_img[enqueueCnt], 1);
            // }

            enqueueCnt++;
            enqueueCnt = (enqueueCnt >= num_buf) ? 0 : enqueueCnt;
            pipeline++;
        }
        else if (pipeline >= 0)
        {
            vx_image input_image, input_image1;
            
            // vx_image flow_vector_field_image;
            // vx_image confidence_image;
            uint32_t num_refs;
            vx_uint32 actual_checksum = 0;

            /* Dequeue input */
            if (status == VX_SUCCESS)
            {
                status = vxGraphParameterDequeueDoneRef(obj->graph, obj->input_img_graph_parameter_index, (vx_reference *)&input_image, 1, &num_refs);
            }

            // if (status == VX_SUCCESS)
            // {
            //     status = vxGraphParameterDequeueDoneRef(obj->graph, obj->input_img_harris_graph_parameter_index, (vx_reference *)&input_image1, 1, &num_refs);
            // }
            if ((vx_true_e == tivxIsTargetEnabled(TIVX_TARGET_DISPLAY1)) && (1 == obj->displayObj.display_option) && (!obj->test_mode))
            {
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterDequeueDoneRef(obj->graph, obj->flow_vector_field_img_graph_parameter_index, (vx_reference *)&flow_vector_field_image, 1, &num_refs);
                // }
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterDequeueDoneRef(obj->graph, obj->confidence_img_graph_parameter_index, (vx_reference *)&confidence_image, 1, &num_refs);
                // }
                // /* Enqueue output */
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->flow_vector_field_img_graph_parameter_index, (vx_reference *)&flow_vector_field_image, 1);
                // }
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->confidence_img_graph_parameter_index, (vx_reference *)&confidence_image, 1);
                // }
            }
            else
            {
                /* Dequeue & Save output */
                outputFileNum = curFileNum - num_buf;
                outputFileNum = (outputFileNum <= obj->start_fileno) ? (obj->end_fileno - obj->start_fileno + outputFileNum + 1) : outputFileNum;

                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterDequeueDoneRef(obj->graph, obj->flow_vector_field_img_graph_parameter_index, (vx_reference *)&flow_vector_field_image, 1, &num_refs);
                //     snprintf(output_file_name_flow_img, APP_MAX_FILE_PATH, "%s/%sflov_%05d.%s",
                //              obj->output_file_path,
                //              obj->output_file_prefix,
                //              outputFileNum,
                //              obj->out_file_ext);
                // }

                // if ((status == VX_SUCCESS) && (!obj->test_mode))
                // {
                //     if (0 == obj->is_interactive)
                //     {
                //         printf(" %d of %d: Saving [%s] ...\n", outputFileNum, obj->end_fileno, output_file_name_flow_img);
                //     }
                //     status = app_save_vximage_to_file(output_file_name_flow_img, flow_vector_field_image, obj->out_file_format);
                // }
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterDequeueDoneRef(obj->graph, obj->confidence_img_graph_parameter_index, (vx_reference *)&confidence_image, 1, &num_refs);
                //     snprintf(output_file_name_conf_img, APP_MAX_FILE_PATH, "%s/%sconf_%05d.%s",
                //              obj->output_file_path,
                //              obj->output_file_prefix,
                //              outputFileNum,
                //              obj->out_file_ext);
                // }
                // if ((status == VX_SUCCESS) && (!obj->test_mode))
                // {
                //     if (0 == obj->is_interactive)
                //     {
                //         printf(" %d of %d: Saving [%s] ...\n", outputFileNum, obj->end_fileno, output_file_name_conf_img);
                //     }
                //     status = app_save_vximage_to_file(output_file_name_conf_img, confidence_image, obj->out_file_format);
                // }

                // if (obj->test_mode == 1)
                // {
                //     if (app_test_check_image(confidence_image, obj->width, obj->height, checksums_expected[0][test_counter], &actual_checksum) == vx_false_e)
                //     {
                //         test_result = vx_false_e;
                //         populate_gatherer(0, test_counter, actual_checksum);
                //     }
                //     if (app_test_check_image(flow_vector_field_image, obj->width, obj->height, checksums_expected[1][test_counter], &actual_checksum) == vx_false_e)
                //     {
                //         test_result = vx_false_e;
                //         populate_gatherer(1, test_counter, actual_checksum);
                //     }
                // }
                /* Enqueue output */
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->flow_vector_field_img_graph_parameter_index, (vx_reference *)&flow_vector_field_image, 1);
                // }
                // if (status == VX_SUCCESS)
                // {
                //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->confidence_img_graph_parameter_index, (vx_reference *)&confidence_image, 1);
                // }
            }
            app_find_image_array_index(obj->pyramidObj.input_img, (vx_reference)input_image, num_buf, &img_array_idx);
            // app_find_image_array_index(obj->harrisObj.input_img, (vx_reference)input_image1, num_buf, &img_harray_idx);
            // printf("harris id %i", img_harray_idx);

            if (img_array_idx != -1)
            {
                if (0 == obj->is_interactive)
                {
                    printf(" %d of %d: Loading [%s] ...\n", curFileNum, obj->end_fileno, input_file_name);
                }
                appPerfPointBegin(&obj->fileio_perf);
                if (status == VX_SUCCESS)
                {
                    status = app_load_vximage_from_file(obj->pyramidObj.input_img[img_array_idx], input_file_name, obj->in_file_format);
                }
                // if (status == VX_SUCCESS)
                // {
                //     status = app_load_vximage_from_file(obj->harrisObj.input_img[img_harray_idx], input_file_name, obj->in_file_format);
                // }
                appPerfPointEnd(&obj->fileio_perf);
            }


            // if (img_harray_idx != -1)
            // {
            //     if (0 == obj->is_interactive)
            //     {
            //         printf(" %d of %d: Loading [%s] ...\n", curFileNum, obj->end_fileno, input_file_name);
            //     }
            //     appPerfPointBegin(&obj->fileio_perf);
            //     if (status == VX_SUCCESS)
            //     {
            //         status = app_load_vximage_from_file(obj->harrisObj.input_img[img_harray_idx], input_file_name, obj->in_file_format);
            //     }
            //     appPerfPointEnd(&obj->fileio_perf);
            // }                

            /* Enqueue input - start execution */
            if (status == VX_SUCCESS)
            {
                status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->input_img_graph_parameter_index, (vx_reference *)&input_image, 1);//(vx_reference *)&input_image
            }

            // if (status == VX_SUCCESS)
            // {
            //     status = vxGraphParameterEnqueueReadyRef(obj->graph, obj->input_img_harris_graph_parameter_index, (vx_reference *)&input_image1, 1);
            // }
            test_counter++;
        }
        appPerfPointEnd(&obj->total_perf);
        /* Run for num_iterations - loop to first image if we're on the last image */
        if ((curFileNum == obj->end_fileno) && (iterations < obj->num_iterations))
        {
            test_counter = 0;
            if (0 == obj->is_interactive)
            {
                printf("Iteration %d of %d: Done !!!\n", iterations, obj->num_iterations);
            }

#if 1 //def APP_DEBUG
            appPerfPointPrintFPS(&obj->total_perf);
            appPerfPointReset(&obj->total_perf);
            if (iterations == 1)
            {
                /* after first iteration reset performance stats */
                appPerfStatsResetAll();
            }
#endif

            curFileNum = obj->start_fileno - 1u;
            iterations++;
        }
        printf("image process done.\n");
    }

    appPerfPointPrintFPS(&obj->total_perf);

    if (0 == obj->is_interactive)
    {
        printf("Ran %d times successfully!\n", obj->num_iterations * (obj->end_fileno - obj->start_fileno));
    }
    if (status == VX_SUCCESS)
    {
        status = vxWaitGraph(obj->graph);
    }
    obj->stop_task = 1;

    return status;
}

tda4 run result above, the pipeline depth is set to 10, and the program is block in 13 frame.

4 年多前

0 Nancy Wang 4 年多前

TI__Guru** 110395 points

建议您到英文论坛咨询，有相关工程师给您提供支持，谢谢！

https://e2e.ti.com/support/processors-group/processors/f/processors-forum

处理器

处理器论坛

TDA4VM: after add vxHarrisCornersNode, the program is block in 13 frame