proj source file

+/*
+-- (c) Copyright 2018 Xilinx, Inc. All rights reserved.
+--
+-- This file contains confidential and proprietary information
+-- of Xilinx, Inc. and is protected under U.S. and
+-- international copyright and other intellectual property
+-- laws.
+--
+-- DISCLAIMER
+-- This disclaimer is not a license and does not grant any
+-- rights to the materials distributed herewith. Except as
+-- otherwise provided in a valid license issued to you by
+-- Xilinx, and to the maximum extent permitted by applicable
+-- law: (1) THESE MATERIALS ARE MADE AVAILABLE "AS IS" AND
+-- WITH ALL FAULTS, AND XILINX HEREBY DISCLAIMS ALL WARRANTIES
+-- AND CONDITIONS, EXPRESS, IMPLIED, OR STATUTORY, INCLUDING
+-- BUT NOT LIMITED TO WARRANTIES OF MERCHANTABILITY, NON-
+-- INFRINGEMENT, OR FITNESS FOR ANY PARTICULAR PURPOSE; and
+-- (2) Xilinx shall not be liable (whether in contract or tort,
+-- including negligence, or under any other theory of
+-- liability) for any loss or damage of any kind or nature
+-- related to, arising under or in connection with these
+-- materials, including for any direct, or any indirect,
+-- special, incidental, or consequential loss or damage
+-- (including loss of data, profits, goodwill, or any type of
+-- loss or damage suffered as a result of any action brought
+-- by a third party) even if such damage or loss was
+-- reasonably foreseeable or Xilinx had been advised of the
+-- possibility of the same.
+--
+-- CRITICAL APPLICATIONS
+-- Xilinx products are not designed or intended to be fail-
+-- safe, or for use in any application requiring fail-safe
+-- performance, such as life-support or safety devices or
+-- systems, Class III medical devices, nuclear facilities,
+-- applications related to the deployment of airbags, or any
+-- other applications that could lead to death, personal
+-- injury, or severe property or environmental damage
+-- (individually and collectively, "Critical
+-- Applications"). Customer assumes the sole risk and
+-- liability of any use of Xilinx products in Critical
+-- Applications, subject only to applicable laws and
+-- regulations governing limitations on product liability.
+--
+-- THIS COPYRIGHT NOTICE AND DISCLAIMER MUST BE RETAINED AS
+-- PART OF THIS FILE AT ALL TIMES.
+*/
+
+#include <algorithm>
+#include <vector>
+#include <atomic>
+#include <queue>
+#include <string>
+#include <iostream>
+#include <fstream>
+#include <iomanip>
+#include <chrono>
+#include <mutex>
+#include <zconf.h>
+#include <thread>
+#include <sys/stat.h>
+#include <dirent.h>
+
+#include <dnndk/dnndk.h>
+
+#include "utils.h"
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <signal.h>
+#include <stdlib.h>
+
+#define Port_host 12300
+#define Port_dcam 12400
+#define host_addr "192.168.2.1"
+#define BUF_SIZE 1024
+
+
+using namespace std;
+using namespace cv;
+using namespace std::chrono;
+
+
+#define INPUT_NODE "layer0_conv"
+
+chrono::system_clock::time_point start_time;
+
+int Mutexnum = 0, Mutexdget = 0, Yolonum = 0;
+int index_yolo = 0, index_co = 1; // file index
+VideoCapture cam(cv::CAP_V4L);
+int Sockfd_host, Sockfd_dcam, newSock_host, newSock_dcam; // socket
+
+
+typedef pair<int, Mat> imagePair;
+class paircomp {
+public:
+    bool operator()(const imagePair& n1, const imagePair& n2) const {
+        if (n1.first == n2.first) {
+            return (n1.first > n2.first);
+        }
+
+        return n1.first > n2.first;
+    }
+};
+
+// 3d yolo result struct
+typedef struct {
+    char number[10];
+    float x;
+    float y;
+    float w;
+    float l;
+    float yaw;
+    float conf;
+    float cls_conf;
+    float cls_pred;
+} dinfo;
+
+// socket function
+void send_result() {
+    static int ind = 0;
+    int now_size = 0, read_size;
+    char file_name[256], msg_buf[BUF_SIZE], sync = '\0';
+    FILE* image;
+    unsigned int fsize;
+
+    sprintf(file_name, "img_%03d_result.jpg", ind); // save file name
+
+    image = fopen(file_name, "w");
+
+    // recv file size
+    if ((fsize = read(Sockfd_host, (char*)&msg_buf, sizeof(msg_buf) - 1)) < 0) {
+        perror("read");
+        exit(1);
+    }
+    msg_buf[fsize] = '\0';
+    fsize = atoi(msg_buf);
+    printf("%d\n", fsize);
+
+    // recv image file
+    while (1) {
+        memset(msg_buf, 0, BUF_SIZE);
+        read_size = read(Sockfd_host, (char*)&msg_buf, BUF_SIZE); // get image from socket
+        fwrite(msg_buf, 1, read_size, image); // write image
+        now_size += read_size;
+        if (now_size == fsize)
+            break;
+        else if (now_size > fsize) {
+            printf("file error\n");
+            exit(1);
+        }
+        write(Sockfd_host, (char*)&sync, sizeof(sync)); // for sync
+    }
+
+    printf("Receive done : %s\n", file_name);
+
+    if (write(Sockfd_host, (char*)&file_name, sizeof(file_name)) < 0) { // 종료 전송 
+        perror("write");
+        exit(1);
+    }
+
+    fclose(image);
+    ind++;
+    if (ind == 999) ind = 0; // max 1000 file
+}
+
+// input frames queue
+queue<pair<int, Mat>> queueInput;
+
+/**
+ * @brief Feed input frame into DPU for process
+ *
+ * @param task - pointer to DPU Task for YOLO-v3 network
+ * @param frame - pointer to input frame
+ * @param mean - mean value for YOLO-v3 network
+ *
+ * @return none
+ */
+void setInputImageForYOLO(DPUTask* task, const Mat& frame, float* mean) {
+    Mat img_copy;
+    int height = dpuGetInputTensorHeight(task, INPUT_NODE);
+    int width = dpuGetInputTensorWidth(task, INPUT_NODE);
+    int size = dpuGetInputTensorSize(task, INPUT_NODE);
+    int8_t* data = dpuGetInputTensorAddress(task, INPUT_NODE);
+
+    image img_new = load_image_cv(frame);
+    image img_yolo = letterbox_image(img_new, width, height);
+
+    vector<float> bb(size);
+    for (int b = 0; b < height; ++b) {
+        for (int c = 0; c < width; ++c) {
+            for (int a = 0; a < 3; ++a) {
+                bb[b * width * 3 + c * 3 + a] = img_yolo.data[a * height * width + b * width + c];
+            }
+        }
+    }
+
+    float scale = dpuGetInputTensorScale(task, INPUT_NODE);
+
+    for (int i = 0; i < size; ++i) {
+        data[i] = int(bb.data()[i] * scale);
+        if (data[i] < 0) data[i] = 127;
+    }
+
+    free_image(img_new);
+    free_image(img_yolo);
+}
+
+void co_detect(DPUTask* task, Mat& frame1, int sWidth, int sHeight, VideoCapture& cam) {  // in postProcess function
+
+    /*output nodes of YOLO-v3 */
+    const vector<string> outputs_node = { "layer81_conv", "layer93_conv", "layer105_conv" };
+
+    vector<vector<float>> boxes;
+    for (size_t i = 0; i < outputs_node.size(); i++) {
+        string output_node = outputs_node[i];
+        int channel = dpuGetOutputTensorChannel(task, output_node.c_str());
+        int width = dpuGetOutputTensorWidth(task, output_node.c_str());
+        int height = dpuGetOutputTensorHeight(task, output_node.c_str());
+
+        int sizeOut = dpuGetOutputTensorSize(task, output_node.c_str());
+        int8_t* dpuOut = dpuGetOutputTensorAddress(task, output_node.c_str());
+        float scale = dpuGetOutputTensorScale(task, output_node.c_str());
+        vector<float> result(sizeOut);
+        boxes.reserve(sizeOut);
+
+        /* Store every output node results */
+        get_output(dpuOut, sizeOut, scale, channel, height, width, result);
+
+        /* Store the object detection frames as coordinate information  */
+        detect(boxes, result, channel, height, width, i, sHeight, sWidth);
+    }
+
+    /* Restore the correct coordinate frame of the original image */
+    correct_region_boxes(boxes, boxes.size(), frame1.cols, frame1.rows, sWidth, sHeight);
+
+    /* Apply the computation for NMS */
+    cout << "boxes size: " << boxes.size() << endl;
+    vector<vector<float>> res = applyNMS(boxes, classificationCnt, NMS_THRESHOLD);
+
+    float h = frame1.rows;
+    float w = frame1.cols;
+    Mat frame2;
+    for (int i = 0; i < 3; i++)
+        cam.grab();
+    cam.read(frame2);
+
+    // for socket variable
+    char matchinfo[1024] = "", match_inframe_info[1024], match_n[10];
+    int hostAddrlen, countnum;
+    struct sockaddr_in hostAddr;
+    hostAddrlen = sizeof(hostAddr);
+    char tmpinfo[1024], count[5];
+    char* receive_info = new char[1024];
+    dinfo depthinfo;
+
+    // read 3dyolo result from udp socket
+    while (Mutexdget == 1)
+        usleep(10);
+    Mutexdget = 1; // lock
+    // get result count in image
+    if (recvfrom(Sockfd_dcam, &count, sizeof(count), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+        perror("recvfrom");
+        exit(1);
+    } // receive depth camera info
+    countnum = atoi(count);
+    for (int i = 0; i < countnum; i++) {
+        if (recvfrom(Sockfd_dcam, &depthinfo, sizeof(depthinfo), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+            perror("recvfrom");
+            exit(1);
+        } // receive depth camera info
+        sprintf(receive_info, "%s %f %f %f %f %f %f %f %f \n", depthinfo.number, depthinfo.x, depthinfo.y, depthinfo.w, depthinfo.l, depthinfo.yaw, depthinfo.conf, depthinfo.cls_conf, depthinfo.cls_pred);
+        strcat(tmpinfo, receive_info);
+    }
+    Mutexdget = 0; // unlock
+
+    for (size_t i = 0; i < res.size(); ++i) {
+        float xmin = (res[i][0] - res[i][2] / 2.0) * w + 1.0;
+        float ymin = (res[i][1] - res[i][3] / 2.0) * h + 1.0;
+        float xmax = (res[i][0] + res[i][2] / 2.0) * w + 1.0;
+        float ymax = (res[i][1] + res[i][3] / 2.0) * h + 1.0;
+
+        //cout<<xmin<<" "<<ymin<<" "<<xmax<<" "<<ymax<<endl;
+
+        Mat compare_part = frame1(Range((2 * ymin + 1 * ymax) / 3, (1 * ymin + 2 * ymax) / 3), Range((2 * xmin + 1 * xmax) / 3, (1 * xmin + 2 * xmax) / 3));
+
+        unsigned nextx, nexty;
+        double min = 100000000, error;
+        for (int x = (2 * xmin + 1 * xmax) / 3; x < (1 * xmin + 2 * xmax) / 3; x += 2) {
+            if (((xmax - xmin) > h / 2) || ((ymax - ymin) > w / 2))
+                break; // too large object reject
+            for (int y = (2 * ymin + 1 * ymax) / 3; y < (1 * ymin + 2 * ymax) / 3; y += 2) {
+                error = 0;
+                for (int k = -(int)(xmax - xmin) / 6; k < (int)(xmax - xmin) / 6; k++) {
+                    for (int u = -(int)(ymax - ymin) / 6; u < (int)(ymax - ymin) / 6; u++) {
+                        uchar b1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 0];
+                        uchar g1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 1];
+                        uchar r1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 2];
+                        int yu = y + u;
+                        int xk = x + k;
+                        if (y + u < 0 || x + k < 0 || y + u > w || x + k > h) { // for padding
+                            if (y + u < 0) yu = 0;
+                            if (x + k < 0) xk = 0;
+                            if (y + u > w) yu = w;
+                            if (x + k > h) xk = h;
+                        }
+                        uchar b2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 0];
+                        uchar g2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 1];
+                        uchar r2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 2];
+                        error += (pow(b1 - b2, 2) + pow(g1 - g2, 2) + pow(r1 - r2, 2));
+                    }
+                }
+                if (min > error) {
+                    min = error;
+                    nextx = x;
+                    nexty = y;
+                }
+            }
+        }
+        // create result string
+        sprintf(match_n, "MC%d-%d", index_co, i);
+        sprintf(match_inframe_info, "%s %f %f %f %f\n", match_n, nextx - (xmax - xmin) / 2, nexty - (ymax - ymin) / 2, nextx + (xmax - xmin) / 2, nexty + (ymax - ymin) / 2);
+        strcat(matchinfo, match_inframe_info);
+
+        rectangle(frame2, Rect(Point((2 * xmin + 1 * xmax) / 3, (2 * ymin + 1 * ymax) / 3), Point((1 * xmin + 2 * xmax) / 3, (1 * ymin + 2 * ymax) / 3)), (0, 255, 0), 1);
+        rectangle(frame2, Rect(Point(nextx - (xmax - xmin) / 2, nexty - (ymax - ymin) / 2), Point(nextx + (xmax - xmin) / 2, nexty + (ymax - ymin) / 2)), (0, 0, 255), 1);
+    }
+    // write tcp socket. to host
+    strcat(matchinfo, tmpinfo);
+    printf("%s\n", matchinfo);
+    if (write(newSock_host, (char*)&matchinfo, sizeof(matchinfo)) < 0) {
+        perror("write");
+        exit(1);
+    }
+
+    string file_ind = "result_" + to_string(index_co) + ".jpg";
+    imwrite(file_ind, frame2);
+    cout << "co_detect1 : " << file_ind << endl;
+    index_co += 2;
+    imshow("Xilinx DPU", frame2);
+    //if(waitKey(30) == 27) break;
+    Yolonum++; // mutex update*/
+}
+
+void co_detect2(DPUTask* task, Mat& frame1, int sWidth, int sHeight, VideoCapture& cam) {  // in postProcess function
+
+    /*output nodes of YOLO-v3 */
+    const vector<string> outputs_node = { "layer81_conv", "layer93_conv", "layer105_conv" };
+
+    vector<vector<float>> boxes;
+    for (size_t i = 0; i < outputs_node.size(); i++) {
+        string output_node = outputs_node[i];
+        int channel = dpuGetOutputTensorChannel(task, output_node.c_str());
+        int width = dpuGetOutputTensorWidth(task, output_node.c_str());
+        int height = dpuGetOutputTensorHeight(task, output_node.c_str());
+
+        int sizeOut = dpuGetOutputTensorSize(task, output_node.c_str());
+        int8_t* dpuOut = dpuGetOutputTensorAddress(task, output_node.c_str());
+        float scale = dpuGetOutputTensorScale(task, output_node.c_str());
+        vector<float> result(sizeOut);
+        boxes.reserve(sizeOut);
+
+        /* Store every output node results */
+        get_output(dpuOut, sizeOut, scale, channel, height, width, result);
+
+        /* Store the object detection frames as coordinate information  */
+        detect(boxes, result, channel, height, width, i, sHeight, sWidth);
+    }
+
+    /* Restore the correct coordinate frame of the original image */
+    correct_region_boxes(boxes, boxes.size(), frame1.cols, frame1.rows, sWidth, sHeight);
+
+    /* Apply the computation for NMS */
+    cout << "boxes size: " << boxes.size() << endl;
+    vector<vector<float>> res = applyNMS(boxes, classificationCnt, NMS_THRESHOLD);
+
+    float h = frame1.rows;
+    float w = frame1.cols;
+    Mat frame2;
+    for (int i = 0; i < 3; i++)
+        cam.grab();
+    cam.read(frame2);
+    char matchinfo[1024] = "", match_inframe_info[1024], match_n[10];
+
+    int hostAddrlen, countnum;
+    struct sockaddr_in hostAddr;
+    hostAddrlen = sizeof(hostAddr);
+    char tmpinfo[1024], count[5];
+    char* receive_info = new char[1024];
+    dinfo depthinfo;
+
+    while (Mutexdget == 1)
+        usleep(10);
+    Mutexdget = 1; // lock
+    if (recvfrom(Sockfd_dcam, &count, sizeof(count), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+        perror("recvfrom");
+        exit(1);
+    } // receive depth camera info
+    countnum = atoi(count);
+    for (int i = 0; i < countnum; i++) {
+        if (recvfrom(Sockfd_dcam, &depthinfo, sizeof(depthinfo), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+            perror("recvfrom");
+            exit(1);
+        } // receive depth camera info
+        sprintf(receive_info, "%s %f %f %f %f %f %f %f %f \n", depthinfo.number, depthinfo.x, depthinfo.y, depthinfo.w, depthinfo.l, depthinfo.yaw, depthinfo.conf, depthinfo.cls_conf, depthinfo.cls_pred);
+        strcat(tmpinfo, receive_info);
+    }
+    Mutexdget = 0; // unlock
+
+    for (size_t i = 0; i < res.size(); ++i) {
+        float xmin = (res[i][0] - res[i][2] / 2.0) * w + 1.0;
+        float ymin = (res[i][1] - res[i][3] / 2.0) * h + 1.0;
+        float xmax = (res[i][0] + res[i][2] / 2.0) * w + 1.0;
+        float ymax = (res[i][1] + res[i][3] / 2.0) * h + 1.0;
+
+        //cout<<xmin<<" "<<ymin<<" "<<xmax<<" "<<ymax<<endl;
+
+        Mat compare_part = frame1(Range((2 * ymin + 1 * ymax) / 3, (1 * ymin + 2 * ymax) / 3), Range((2 * xmin + 1 * xmax) / 3, (1 * xmin + 2 * xmax) / 3));
+
+        unsigned nextx, nexty;
+        double min = 100000000, error;
+        for (int x = (2 * xmin + 1 * xmax) / 3; x < (1 * xmin + 2 * xmax) / 3; x += 2) { // for padding
+            if (((xmax - xmin) > h / 2) || ((ymax - ymin) > w / 2))
+                break; // too large object reject
+            for (int y = (2 * ymin + 1 * ymax) / 3; y < (1 * ymin + 2 * ymax) / 3; y += 2) {
+                error = 0;
+                for (int k = -(int)(xmax - xmin) / 6; k < (int)(xmax - xmin) / 6; k++) {
+                    for (int u = -(int)(ymax - ymin) / 6; u < (int)(ymax - ymin) / 6; u++) {
+                        uchar b1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 0];
+                        uchar g1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 1];
+                        uchar r1 = compare_part.ptr<uchar>(u + (int)(ymax - ymin) / 6)[(k + (int)(xmax - xmin) / 6) * 3 + 2];
+                        int yu = y + u;
+                        int xk = x + k;
+                        if (y + u < 0 || x + k < 0 || y + u > w || x + k > h) {
+                            if (y + u < 0) yu = 0;
+                            if (x + k < 0) xk = 0;
+                            if (y + u > w) yu = w;
+                            if (x + k > h) xk = h;
+                        }
+                        uchar b2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 0];
+                        uchar g2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 1];
+                        uchar r2 = frame2.ptr<uchar>(yu)[(xk) * 3 + 2];
+                        error += (pow(b1 - b2, 2) + pow(g1 - g2, 2) + pow(r1 - r2, 2));
+                    }
+                }
+                if (min > error) {
+                    min = error;
+                    nextx = x;
+                    nexty = y;
+                }
+            }
+        }
+        sprintf(match_n, "MC%d-%d", index_co, i);
+        sprintf(match_inframe_info, "%s %f %f %f %f\n", match_n, nextx - (xmax - xmin) / 2, nexty - (ymax - ymin) / 2, nextx + (xmax - xmin) / 2, nexty + (ymax - ymin) / 2);
+        strcat(matchinfo, match_inframe_info);
+
+        rectangle(frame2, Rect(Point((2 * xmin + 1 * xmax) / 3, (2 * ymin + 1 * ymax) / 3), Point((1 * xmin + 2 * xmax) / 3, (1 * ymin + 2 * ymax) / 3)), (0, 0, 255), 1);
+        rectangle(frame2, Rect(Point(nextx - (xmax - xmin) / 2, nexty - (ymax - ymin) / 2), Point(nextx + (xmax - xmin) / 2, nexty + (ymax - ymin) / 2)), (0, 0, 255), 1);
+    }
+    strcat(matchinfo, tmpinfo);
+    printf("%s\n", matchinfo);
+    if (write(newSock_host, (char*)&matchinfo, sizeof(matchinfo)) < 0) {
+        perror("write");
+        exit(1);
+    }
+
+    string file_ind = "result_" + to_string(index_co) + ".jpg";
+    imwrite(file_ind, frame2);
+    cout << "co_detect2 : " << file_ind << endl;
+    index_co += 2;
+    imshow("Xilinx DPU", frame2);
+    //if(waitKey(30) == 27) break;
+    delete receive_info;
+    Yolonum++; // mutex update*/
+}
+
+/**
+ * @brief Post process after the runing of DPU for YOLO-v3 network
+ *
+ * @param task - pointer to DPU task for running YOLO-v3
+ * @param frame
+ * @param sWidth
+ * @param sHeight
+ *
+ * @return none
+ */
+void postProcess(DPUTask* task, Mat& frame, int sWidth, int sHeight) {
+
+    /*output nodes of YOLO-v3 */
+    const vector<string> outputs_node = { "layer81_conv", "layer93_conv", "layer105_conv" };
+
+    vector<vector<float>> boxes;
+    for (size_t i = 0; i < outputs_node.size(); i++) {
+        string output_node = outputs_node[i];
+        int channel = dpuGetOutputTensorChannel(task, output_node.c_str());
+        int width = dpuGetOutputTensorWidth(task, output_node.c_str());
+        int height = dpuGetOutputTensorHeight(task, output_node.c_str());
+
+        int sizeOut = dpuGetOutputTensorSize(task, output_node.c_str());
+        int8_t* dpuOut = dpuGetOutputTensorAddress(task, output_node.c_str());
+        float scale = dpuGetOutputTensorScale(task, output_node.c_str());
+        vector<float> result(sizeOut);
+        boxes.reserve(sizeOut);
+
+        /* Store every output node results */
+        get_output(dpuOut, sizeOut, scale, channel, height, width, result);
+
+        /* Store the object detection frames as coordinate information  */
+        detect(boxes, result, channel, height, width, i, sHeight, sWidth);
+    }
+
+    /* Restore the correct coordinate frame of the original image */
+    correct_region_boxes(boxes, boxes.size(), frame.cols, frame.rows, sWidth, sHeight);
+
+    /* Apply the computation for NMS */
+    cout << "boxes size: " << boxes.size() << endl;
+    vector<vector<float>> res = applyNMS(boxes, classificationCnt, NMS_THRESHOLD);
+
+    float h = frame.rows;
+    float w = frame.cols;
+    char yoloinfo[1024] = "YOLO-\n", yolo_inframe_info[1024], yolo_num[10];
+    for (size_t i = 0; i < res.size(); ++i) {
+
+        float xmin = (res[i][0] - res[i][2] / 2.0) * w + 1.0;
+        float ymin = (res[i][1] - res[i][3] / 2.0) * h + 1.0;
+        float xmax = (res[i][0] + res[i][2] / 2.0) * w + 1.0;
+        float ymax = (res[i][1] + res[i][3] / 2.0) * h + 1.0;
+
+        //cout<<res[i][res[i][4] + 6]<<" ";
+        //cout<<xmin<<" "<<ymin<<" "<<xmax<<" "<<ymax<<endl;
+
+    // write yolo result
+        sprintf(yolo_num, "YC%d-%d", index_yolo, i);
+        sprintf(yolo_inframe_info, "%s %f %f %f %f\n", yolo_num, xmin, ymin, xmax, ymax); // yolo info
+        strcat(yoloinfo, yolo_inframe_info);
+
+        if (res[i][res[i][4] + 6] > CONF) {
+            int type = res[i][4];
+
+            if (type == 0) {
+                rectangle(frame, cvPoint(xmin, ymin), cvPoint(xmax, ymax), Scalar(0, 0, 255), 1, 1, 0);
+            }
+            else if (type == 1) {
+                rectangle(frame, cvPoint(xmin, ymin), cvPoint(xmax, ymax), Scalar(255, 0, 0), 1, 1, 0);
+            }
+            else {
+                rectangle(frame, cvPoint(xmin, ymin), cvPoint(xmax, ymax), Scalar(0, 255, 255), 1, 1, 0);
+            }
+        }
+    }
+    int hostAddrlen, countnum;
+    struct sockaddr_in hostAddr;
+    hostAddrlen = sizeof(hostAddr);
+    char receive_info[1024], count[5];
+    dinfo depthinfo;
+
+    while (Mutexdget == 1)
+        usleep(10);
+    Mutexdget = 1; // lock
+
+    if (recvfrom(Sockfd_dcam, &count, sizeof(count), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+        perror("recvfrom");
+        exit(1);
+    } // receive depth camera info
+    countnum = atoi(count);
+    for (int i = 0; i < countnum; i++) {
+        if (recvfrom(Sockfd_dcam, &depthinfo, sizeof(depthinfo), 0, (struct sockaddr*)&hostAddr, (socklen_t*)&hostAddrlen) < 0) {
+            perror("recvfrom");
+            exit(1);
+        } // receive depth camera info
+        sprintf(receive_info, "%s %f %f %f %f %f %f %f %f \n", depthinfo.number, depthinfo.x, depthinfo.y, depthinfo.w, depthinfo.l, depthinfo.yaw, depthinfo.conf, depthinfo.cls_conf, depthinfo.cls_pred);
+        strcat(yoloinfo, receive_info);
+    }
+    Mutexdget = 0; // unlock 
+
+    printf("%s\n", yoloinfo);
+    if (write(newSock_host, (char*)&yoloinfo, sizeof(yoloinfo)) < 0) {
+        perror("write");
+        exit(1);
+    }
+}
+
+/**
+ * @brief Thread entry for running YOLO-v3 network on DPU for acceleration
+ *
+ * @param task - pointer to DPU task for running YOLO-v3
+ * @param img
+ *
+ * @return none
+ */
+thread runYOLO(DPUTask* task, Mat& img, VideoCapture& cam) {
+    /* mean values for YOLO-v3 */
+    float mean[3] = { 0.0f, 0.0f, 0.0f };
+
+    int height = dpuGetInputTensorHeight(task, INPUT_NODE);
+    int width = dpuGetInputTensorWidth(task, INPUT_NODE);
+
+    /* feed input frame into DPU Task with mean value */
+    setInputImageForYOLO(task, img, mean);
+
+    /* invoke the running of DPU for YOLO-v3 */
+    dpuRunTask(task);
+    thread t(co_detect, task, ref(img), width, height, ref(cam));
+    postProcess(task, img, width, height);
+    Yolonum++;
+    return t;
+}
+thread runYOLO2(DPUTask* task, Mat& img, VideoCapture& cam) {
+    /* mean values for YOLO-v3 */
+    float mean[3] = { 0.0f, 0.0f, 0.0f };
+
+    int height = dpuGetInputTensorHeight(task, INPUT_NODE);
+    int width = dpuGetInputTensorWidth(task, INPUT_NODE);
+
+    /* feed input frame into DPU Task with mean value */
+    setInputImageForYOLO(task, img, mean);
+
+    /* invoke the running of DPU for YOLO-v3 */
+    dpuRunTask(task);
+    thread t(co_detect2, task, ref(img), width, height, ref(cam));
+    postProcess(task, img, width, height);
+    Yolonum++;
+    return t;
+}
+
+/**
+ * @brief Entry for running YOLO-v3 neural network for ADAS object detection
+ *
+ */
+void Close(int signo)
+{
+    close(Sockfd_host);
+    close(Sockfd_dcam);
+    printf("\ndone\n");
+    exit(0);
+}
+int main(const int argc, const char** argv) {
+    int     len;
+    struct sockaddr_in      Addr, dcamAddr, hostAddr;
+
+    signal(SIGINT, Close);
+
+    if (argc != 1) {
+        cout << "Usage : ./yolo " << endl;
+        return -1;
+    }
+
+    // get current path
+    /*char cwd[256];
+    if(getcwd(cwd, 256) == NULL) {
+         perror("getcwd");
+         exit(1);
+    }*/
+
+    /* socket for host - tcp */
+    if ((Sockfd_host = socket(PF_INET, SOCK_STREAM, 0)) < 0) {
+        perror("socket");
+        exit(1);
+    }
+
+    cout << "~~";
+
+    bzero((char*)&Addr, sizeof(Addr));
+    Addr.sin_family = PF_INET;
+    Addr.sin_addr.s_addr = htonl(INADDR_ANY);
+    Addr.sin_port = htons(Port_host);
+
+    if (bind(Sockfd_host, (struct sockaddr*)&Addr, sizeof(Addr)) < 0) {
+        perror("bind");
+        exit(1);
+    }
+    listen(Sockfd_host, 5);
+
+    len = sizeof(hostAddr);
+    newSock_host = accept(Sockfd_host, (struct sockaddr*)&hostAddr, (socklen_t*)&len);
+    if (newSock_host < 0) {
+        perror("accept");
+        exit(1);
+    }
+
+    /*socket for depth camera - udp*/
+    if ((Sockfd_dcam = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
+        perror("socket");
+        exit(1);
+    }
+
+    bzero((char*)&dcamAddr, sizeof(dcamAddr));
+    dcamAddr.sin_family = PF_INET;
+    dcamAddr.sin_addr.s_addr = htonl(INADDR_ANY);
+    dcamAddr.sin_port = htons(Port_dcam);
+
+    if (bind(Sockfd_dcam, (struct sockaddr*)&dcamAddr, sizeof(dcamAddr)) < 0) {
+        perror("bind");
+        exit(1);
+    }
+
+    /*
+        if(write(Sockfd, (char*)&cwd, sizeof(cwd)) < 0) { // cwd 전송
+            perror("write");
+            exit(1);
+        }*/
+
+    if (!cam.isOpened()) {
+        cout << "Fail to open webcam" << endl;
+        exit(-1);
+    }
+    string file_ind;
+
+    // init Yolo2 before while 
+    dpuOpen();
+    DPUKernel* kernel = dpuLoadKernel("yolo");
+    DPUTask* task = dpuCreateTask(kernel, 0);
+    Mat img;
+    thread t1, t2;
+    cam.read(img);
+    t2 = runYOLO2(task, img, cam);
+    file_ind = "result_" + to_string(index_yolo) + ".jpg";
+    imwrite(file_ind, img);
+    cout << "Yolo2 : " << file_ind << endl;
+    index_yolo += 2;
+    while (1) {
+        file_ind = "result_" + to_string(index_yolo) + ".jpg";
+        // run yolo1
+        for (int i = 0; i < 5; i++) // buffer claer
+            cam.grab();
+        cam.read(img);
+        t1 = runYOLO(task, img, cam);
+        if (Mutexnum + 1 < Yolonum) // create mutex
+            t2.join();
+        imwrite(file_ind, img);
+        cout << "Yolo1 : " << file_ind << endl;
+        index_yolo += 2;
+        imshow("Xilinx DPU", img);
+        if (waitKey(30) == 27) break;
+        // run yolo2
+        for (int i = 0; i < 5; i++) // buffer claer
+            cam.grab();
+        cam.read(img);
+        t2 = runYOLO2(task, img, cam);
+        if (Mutexnum + 1 < Yolonum) // create mutex
+            t1.join();
+        file_ind = "result_" + to_string(index_yolo) + ".jpg";
+        imwrite(file_ind, img);
+        cout << "Yolo2 : " << file_ind << endl;
+        index_yolo += 2;
+        imshow("Xilinx DPU", img);
+        if (waitKey(30) == 27) break;
+    }
+    dpuDestroyTask(task);
+    /* Destroy DPU Kernels & free resources */
+    dpuDestroyKernel(kernel);
+
+    /* Dettach from DPU driver & free resources */
+    dpuClose();
+    return 0;
+}
+

+// 최종 결합된 data를 socket을 이용해 수신
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <signal.h>
+
+#define Port 12300
+#define Board_addr "192.168.2.99"
+#define BUF_SIZE 1024
+
+int Sockfd;
+
+void receive_result() {
+	static int ind = 0;
+
+	char info[2000];
+	strcpy(info, "");
+	if (read(Sockfd, (char*)&info, sizeof(info)) <= 0) {
+		perror("read");
+		exit(1);
+	}
+	printf("%s\n", info);
+
+	/*int now_size = 0, read_size;
+	char file_name[256], msg_buf[BUF_SIZE], sync = '\0';
+	FILE* image;
+	unsigned int fsize;
+
+	sprintf(file_name, "img_%03d_result.jpg", ind); // save file name
+	
+	image = fopen(file_name, "w");
+	
+	// recv file size
+	msg_buf[fsize] = '\0';
+	fsize = atoi(msg_buf);
+	printf("%d\n", fsize);
+
+	// recv image file
+	while(1) {
+		memset(msg_buf, 0, BUF_SIZE);
+		read_size = read(Sockfd, (char*)&msg_buf, BUF_SIZE); // get image from socket
+		fwrite(msg_buf, 1, read_size, image); // write image
+		now_size += read_size;
+		if(now_size == fsize)
+			break;
+		else if(now_size > fsize) {
+			printf("file error\n");
+			exit(1);
+		}
+		write(Sockfd, (char*)&sync, sizeof(sync)); // for sync
+	}
+	
+	printf("Receive done : %s\n", file_name);
+	
+	if(write(Sockfd, (char*)&file_name, sizeof(file_name)) < 0) { // 종료 전송 
+		perror("write");
+		exit(1);
+	}
+	
+	fclose(image);
+	ind++;
+	if(ind == 999) ind = 0; // max 1000 file
+*/
+}
+
+main(int argc, char *argv[])
+{
+	int	n, len;
+	struct sockaddr_in	boardAddr;
+	char cwd[256];
+
+	if(getcwd(cwd, 256) == NULL) {
+		perror("getcwd");
+		exit(1);
+	}
+	
+	if ((Sockfd = socket(PF_INET, SOCK_STREAM, 0)) < 0)  {
+		perror("socket");
+		exit(1);
+	}
+	
+	bzero((char *)&boardAddr, sizeof(boardAddr));
+	boardAddr.sin_family = PF_INET;
+	boardAddr.sin_addr.s_addr = inet_addr(Board_addr);
+	boardAddr.sin_port = htons(Port);
+		
+	if (connect(Sockfd, (struct sockaddr*)&boardAddr, sizeof(boardAddr)) < 0)  {
+		perror("connect");
+		exit(1);
+	}
+	
+	printf("connect\n");
+	/*
+	if(write(Sockfd, (char*)&cwd, sizeof(cwd)) < 0) { // cwd 전송 
+		perror("write");
+		exit(1);
+	}
+	*/
+	while(1) {
+		printf("\n\n");
+		receive_result(); // 받은 파일이름 출력
+	}
+	close(Sockfd);
+	return 0;
+}