File ExampleClsRuntimeWithMetadata.cpp

Go to the documentation of this file.

#include <limits.h>
#include <stdlib.h>

#include <filesystem>
#include <iostream>
#include <string>
#include <vector>

#include "ExampleMain.h"
#include "SaigeVisionAPI.h"
#include "SaigeVisionUtil.h"
namespace cls {
int run_with_meta() {
    std::string dataDir =
        get_env_var("SAIGE_HOME") + "/SaigeVision/engine/Examples/data/sfaw2023/images/test/";

    // trained model path
    std::string trainedModelPath = dataDir + "../../models/cls_example.saigecls";

    // image paths to classify
    // used filesystem library of c++17 standard.
    // u8string means 8byte character string so, it doesn't support unicode system of filesystem.
    std::vector<std::string> imagePaths;

    for (const auto& file : std::filesystem::recursive_directory_iterator(dataDir)) {
        if (file.is_regular_file() && file.path().extension() == ".png") {
            imagePaths.push_back(std::string(file.path().u8string()));
        }
    }

    int retval;
    int verbosity = 2;
    int deviceIndex = 0;
    bool detailedTime = false;
    bool optimize = false;

    int waitkeyMilliSecond = 1000;

    /***********************************************************
     *    1. 모델 파일 및 검사 결과 handle 생성,
     *       Create handles for model files and inspection results
     ***********************************************************/
    SaigeInferenceHandle handle = nullptr;
    SaigeHandleConfig config;
    config.batchSize = 1;
    unsigned int handleFlags = SAIGE_SET_BATCH_SIZE;
    if (optimize) {
        handleFlags |= SAIGE_OPTIMIZE_HANDLE;
    }
    retval = SaigeCreateInferenceHandle(&handle, kProjectTypeCLS, trainedModelPath.c_str(),
                                        deviceIndex, &config, handleFlags);

    if (retval < 0) {
        std::cerr << "SaigeCreateInferenceHandle() failed - " << SaigeStrError(retval) << std::endl;
        return retval;
    }

    SaigeInferenceResult result = nullptr;
    retval = SaigeCreateInferenceResult(&result);
    if (retval < 0) {
        std::cerr << "SaigeCreateInferenceResult() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        return retval;
    }

    /***********************************************************
     *    2. 클래스 정보 확인 (Optional)
     *       Check class information
     ***********************************************************/
    std::cout << SaigeGetModuleVersion() << std::endl;
    SaigeModelInfo* modelInfo = new SaigeModelInfo;
    retval = SaigeGetModelInfo(handle, modelInfo);
    if (retval < 0) {
        std::cerr << "SaigeGetModelInfo() failed - " << SaigeStrError(retval) << std::endl;
    } else if (verbosity >= 1) {
        std::cout << "[Classification Model Info]" << std::endl;
        std::cout << "Network Type: " << modelInfo->networkType << std::endl;
        std::cout << "Number of Classes: " << modelInfo->numOfClasses << std::endl;

        for (auto i = 0; i < modelInfo->numOfClasses; ++i) {
            uint8_t red, green, blue;
            SaigeColorCodeToRGB(modelInfo->classColors[i], &red, &green, &blue);
            std::cout << modelInfo->classNames[i] << ": color RGB(" << (int)red << ", "
                      << (int)green << ", " << (int)blue << ") - "
                      << (modelInfo->classIsNg[i] ? "NG" : "OK") << std::endl;
        }
        std::cout << "Trained Developer Version: " << modelInfo->developerVersion << std::endl;
        std::cout << "Server Version: " << modelInfo->serverVersion << std::endl;
        std::cout << "Module Version: " << modelInfo->moduleVersion << std::endl;
        std::cout << "TrainingTimeStarted: " << modelInfo->trainingTimeStarted << std::endl;
        std::cout << "TrainingTimeCompleted: " << modelInfo->trainingTimeCompleted << std::endl;
        std::cout << "TrainingTimeElapsed: " << modelInfo->trainingTimeElapsed << std::endl;
        std::cout << "UsedGpuName: " << modelInfo->usedGpuName << std::endl;
        std::cout << "Number of Train Images: " << modelInfo->numOfTrainImages << std::endl;
        std::cout << "Number of Validation Images: " << modelInfo->numOfValidationImages
                  << std::endl;
        std::cout << "Number of Iterations: " << modelInfo->numOfIterations << std::endl;
        std::cout << "ROI Info" << std::endl
                  << "- ROI Mode: " << modelInfo->roiInfo.roiMode << std::endl
                  << "- ROI LTRB Ratio: " << modelInfo->roiInfo.left << ", "
                  << modelInfo->roiInfo.top << ", " << modelInfo->roiInfo.right << ", "
                  << modelInfo->roiInfo.bottom << std::endl;
        std::cout << "Image Resize Width, Height: " << modelInfo->imageResizeWidth << ", "
                  << modelInfo->imageResizeHeight << std::endl;
        if (modelInfo->roiInfo.blindMask != nullptr) {
            // show blind mask
            SaigeShowImage(modelInfo->roiInfo.blindMask, modelInfo->roiInfo.blindMaskWidth,
                           modelInfo->roiInfo.blindMaskHeight, 1, SaigeImageType::SV_U8, false,
                           true, 255.0, 0);
        }
    }

    /***********************************************************
     *    3. 검사 옵션 확인 및 수정 (Optional)
     *       Check inspection options and set new values
     ***********************************************************/
    // additionalScores[numOfClasses]
    // 각 클래스 별로 스코어 값(int)을 보정해 주는 옵션입니다. (default= {0,0,0,...}
    SaigeInferenceOption* options = new SaigeInferenceOption;
    retval = SaigeGetInferenceOption(handle, options);
    if (retval < 0) {
        std::cerr << "SaigeGetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        SaigeDestroyInferenceResult(result);
        return retval;
    }
    if (detailedTime) {
        options->calcTime = true;
        retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_CALC_TIME);
        if (retval < 0) {
            std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }
    }
    // 값을 변경하여 설정할 수 있습니다.
    // 핸들의 모델 타입과 관련없는 값은 설정이 반영되지 않습니다.
    // Classification에서는 Additional Scores를 설정해줄 수 있습니다.
    /*options->additionalScores[1] = 50;

    retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_OBJECT_AREA_THRESHOLD);
    if (retval < 0) {
        std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        SaigeDestroyInferenceResult(result);
        return retval;
    }*/

    if (verbosity >= 3 && optimize == false) {
        options->calcClassActivationMap = true;
        retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_CALC_CLASS_ACTIVATION_MAP);
        if (retval < 0) {
            std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }
    }

    retval = SaigeGetInferenceOption(handle, options);
    if (retval < 0) {
        std::cerr << "SaigeGetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        SaigeDestroyInferenceResult(result);
        return retval;
    }
    if (verbosity >= 1) {
        std::cout << "[Classification Inference Options]" << std::endl;
        std::cout << "Calc Time: " << (options->calcTime ? "True" : "False") << std::endl;
        std::cout << "Batch Size: " << options->batchSize << std::endl;
        std::cout << "Additional Scores: (" << options->additionalScores[0];
        for (int i = 1; i < modelInfo->numOfClasses; ++i) {
            std::cout << ", " << options->additionalScores[i];
        }
        std::cout << ")" << std::endl;
        std::cout << "Calculate Class Activation Map: "
                  << (options->calcClassActivationMap ? "True" : "False") << std::endl;
    }

    /***********************************************************
     *    4. 이미지 검사하기, Inspect images
     *       이미지 사이즈는 같다고 가정한다.
     ***********************************************************/
    int h = 0, w = 0, c = 0, type = 0;
    int nbatch = 1;
    unsigned char* ucImageData = nullptr;
    long long imreadTime, inferenceTime, postProcessingTime;
    std::vector<long long> inspectionTimes;
    retval = SaigeSetHandleMetadata(handle, "Manufacturing Line", "test");
    if (retval < 0) {
        std::cerr << "SaigeSetHandleMetadata() failed - " << SaigeStrError(retval) << std::endl;
    }
    for (auto& img : imagePaths) {
        // 이미지 데이터 메모리를 해제합니다. (루프일관성)
        if (ucImageData != nullptr) {
            retval = SaigeFreeImageData(ucImageData);
            if (retval < 0) {
                std::cerr << "SaigeFreeImageData() Failed - " << SaigeStrError(retval) << std::endl;
            }
            ucImageData = nullptr;
        }
        // 이미지 데이터를 로드합니다.
        ucImageData = (unsigned char*)SaigeGetImageData(img.c_str(), &w, &h, &c, &type);
        if (ucImageData == nullptr) {
            std::cerr << "SaigeGetImageData() failed - " << img << std::endl;
            continue;
        }
        /***********************************************************
         *    4-1. 검사 함수 호출하기, Call Infer API
         ***********************************************************/
        if (verbosity >= 2) {
            std::cout << "\n================================" << std::endl;
            std::cout << "(Start): Infer From Image: " << img.c_str() << std::endl;
            std::cout << "---" << std::endl;
        }
        unsigned int imageFlag = SaigeConvertImageType(c, type);

        if (img.find("1") > 0) {
           retval = SaigeSetInferenceMetadata(result, "Test", "test1");
        } 
        std::string file_name = img.substr(img.find_last_of("/")+1);
        std::string pid = file_name.substr(0, file_name.find_last_of("."));
        SaigeSetInferenceMetadata(result, "_product_id",
                                  file_name.substr(0, file_name.find(".")).c_str());
        std::string file_path = "V:\\image\\runtime_example\\images\\test\\" + file_name;
        if (file_name.find("01.png") != std::string::npos) {
           SaigeSetInferenceMetadata(result, "_file_path", file_path.c_str());
        }

        // 검사를 두 번 진행한다.
        /*retval = SaigeInferFromByteArray(handle, ucImageData, w, h, imageFlag, result);
        if (retval < 0) {
            std::cerr << "SaigeInferFromByteArray() failed - " << SaigeStrError(retval)
                      << std::endl;
            continue;
        }*/

        retval = SaigeInferFromImageFile(handle, img.c_str(), result);
        if (retval < 0) {
            std::cerr << "SaigeInferFromImageFile() failed - " << SaigeStrError(retval)
                      << std::endl;
            continue;
        }
        /***********************************************************
         *    4-2. 검사 결과 받아오기, Retrieve inspection results
         ***********************************************************/
        std::vector<float> score;
        int classId;
        score.resize(modelInfo->numOfClasses);
        retval = SaigeGetInferenceResultScore(result, score.data(), true);
        if (retval < 0) {
            std::cerr << "SaigeGetInferenceResultScore() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }

        retval = SaigeGetInferenceResultClass(result, &classId);
        if (retval < 0) {
            std::cerr << "SaigeGetInferenceResultClass() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }

        // Set result metadata
        retval = SaigeSetResultMetadata(result, "Result Class", modelInfo->classNames[classId]);
        if (retval < 0) {
            std::cerr << "SaigeSetResultMetadata() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }

        retval = SaigeGetInferenceResultInspectionTime(result, &imreadTime, &inferenceTime,
                                                       &postProcessingTime);
        if (retval < 0) {
            std::cerr << "SaigeGetInferenceResultClass() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            SaigeDestroyInferenceResult(result);
            return retval;
        }

        inspectionTimes.push_back(imreadTime + inferenceTime + postProcessingTime);

        // Display and draw
        if (verbosity >= 2) {
            for (int j = 0; j < modelInfo->numOfClasses; j++) {
                std::cout << "  - " << j << "-th class score: " << score[j] << std::endl;
            }
            std::cout << "  - Result Class: " << modelInfo->classNames[classId] << std::endl;

            retval = SaigeGetInferenceResultScore(result, score.data(), 0);
            if (retval < 0) {
                std::cerr << "SaigeGetInferenceResultClass() failed - " << SaigeStrError(retval)
                          << std::endl;
                SaigeDestroyInferenceHandle(handle);
                SaigeDestroyInferenceResult(result);
                return retval;
            }
            std::cout << "Without additional score" << std::endl;
            for (int j = 0; j < modelInfo->numOfClasses; j++) {
                std::cout << "  - " << j << "-th class score: " << score[j] << std::endl;
            }
        }
        if (verbosity >= 3 && optimize == false) {
            unsigned char* classActivationMap = nullptr;
            retval = SaigeGetInferenceResultClassActivationMap(result, &classActivationMap);
            if (retval < 0) {
                std::cerr << "SaigeGetInferenceResultClassActivationMap() failed - "
                          << SaigeStrError(retval) << std::endl;
                SaigeDestroyInferenceHandle(handle);
                SaigeDestroyInferenceResult(result);
                return retval;
            }
            if (verbosity >= 4) {
                SaigeDrawImage(classActivationMap, w, h, 1, 1, SaigeImageType::SV_U8, false,
                               waitkeyMilliSecond);
                SaigeDrawImageOverBase(ucImageData, classActivationMap, w, h, c, 1, type,
                                       SaigeImageType::SV_U8, 0.5, false, waitkeyMilliSecond);
            }
        }
        if (verbosity >= 2) {
            if (detailedTime) {
                std::cout << "---\n(End) Image Read, Infer, Postprocess Times (ms): (" << imreadTime
                          << ", " << inferenceTime << ", " << postProcessingTime << ")"
                          << std::endl;
            } else {
                std::cout << "---\n(End) Inspection Time: " << inferenceTime << " ms" << std::endl;
            }
        }
    }

    if (verbosity >= 1) {
        long long totalInferenceTime = 0;
        for (auto time : inspectionTimes) {
            totalInferenceTime += time;
        }
        std::cout << "Total Elapsed Time: " << totalInferenceTime << " ms" << std::endl;
    }

    if (ucImageData != nullptr) {
        retval = SaigeFreeImageData(ucImageData);
        if (retval < 0) {
            std::cerr << "SaigeFreeImageData() Failed - " << SaigeStrError(retval) << std::endl;
        }
        ucImageData = nullptr;
    }

    delete modelInfo;
    delete options;
    /***********************************************************
     *    5. 핸들 파괴, Destruct handles
     ***********************************************************/
    retval = SaigeDestroyInferenceResult(result);
    if (retval < 0) {
        std::cerr << "SaigeDestroyInferenceResult() failed - " << SaigeStrError(retval)
                  << std::endl;
        return retval;
    }

    retval = SaigeDestroyInferenceHandle(handle);
    if (retval < 0) {
        std::cerr << "SaigeDestroyInferenceHandle() failed - " << SaigeStrError(retval)
                  << std::endl;
        return retval;
    }

    return 0;
}
}  // namespace cls