File ExampleClsRuntimeBatch.cpp

Go to the documentation of this file.

#include <limits.h>
#include <stdlib.h>

#include <filesystem>
#include <iostream>
#include <string>
#include <vector>
#include <cmath>

#include "ExampleMain.h"
#include "SaigeVisionAPI.h"
#include "SaigeVisionUtil.h"

void clear_array(void** array, const int num) {
    for (int i = 0; i < num; ++i) {
        if (array[i] != nullptr) {
            delete[] array[i];
        }
    }
}
int clearImageBatch(unsigned char** images, const int num) {
    int retval = 0;
    for (int i = 0; i < num; ++i) {
        retval = SaigeFreeImageData(images[i]);
        if (retval < 0) {
            break;
        }
        images[i] = nullptr;
    }
    return retval;
}

int clearResults(SaigeInferenceResult* results, const int num) {
    int retval = 0;
    for (int i = 0; i < num; ++i) {
        retval = SaigeDestroyInferenceResult(results[i]);
        if (retval < 0) break;
    }
    return retval;
}

namespace cls {
int run_batch() {
    std::string dataDir =
        get_env_var("SAIGE_HOME") + "/SaigeVision/engine/Examples/data/sfaw2023/images/test/";

    // trained model path
    std::string trainedModelPath = dataDir + "../../models/cls_example.saigecls";

    // image paths to classify
    // used filesystem library of c++17 standard.
    // u8string means 8byte character string so, it doesn't support unicode system of filesystem.
    std::vector<std::string> imagePaths;

    for (const auto& file : std::filesystem::recursive_directory_iterator(dataDir)) {
        if (file.is_regular_file() && file.path().extension() == ".png") {
            imagePaths.push_back(std::string(file.path().u8string()));
        }
    }

    int retval;
    int verbosity = 2;
    int deviceIndex = 0;
    bool optimize = false;
    bool detailedTime = false;
    int batchSize = 4;

    int waitkeyMilliSecond = 0;
    /***********************************************************
     *    1. 모델 파일 및 검사 결과 핸들 생성,
     *       Create handles for model files and inference results
     ***********************************************************/
    SaigeInferenceHandle handle = nullptr;
    SaigeHandleConfig config;
    config.batchSize = batchSize;
    unsigned int handleFlags = SAIGE_SET_BATCH_SIZE;
    if (optimize) {
        handleFlags |= SAIGE_OPTIMIZE_HANDLE;
    }
    retval = SaigeCreateInferenceHandle(&handle, kProjectTypeCLS, trainedModelPath.c_str(),
                                        deviceIndex, &config, handleFlags);
    if (retval < 0) {
        std::cerr << "SaigeCreateInferenceHandle failed - " << SaigeStrError(retval) << std::endl;
        return retval;
    }

    SaigeInferenceResult* results = new SaigeInferenceResult[batchSize];
    for (int i = 0; i < batchSize; ++i) {
        retval = SaigeCreateInferenceResult(&results[i]);
        if (retval < 0) {
            std::cerr << "SaigeCreateInferenceResult() failed - " << SaigeStrError(retval)
                      << std::endl;
            for (int j = 0; j < i; ++j) {
                SaigeDestroyInferenceResult(results[j]);
            }
            delete[] results;
            SaigeDestroyInferenceHandle(handle);
            return retval;
        }
    }

    /***********************************************************
     *    2. 모델 정보 확인 (Optional)
     *       Check model information
     ***********************************************************/
    SaigeModelInfo* modelInfo = new SaigeModelInfo;
    retval = SaigeGetModelInfo(handle, modelInfo);
    if (retval < 0) {
        std::cerr << "SaigeGetModelInfo() failed - " << SaigeStrError(retval) << std::endl;
    } else if (verbosity >= 1) {
        std::cout << "[Classification Model Info]" << std::endl;
        std::cout << "Network Type: " << modelInfo->networkType << std::endl;
        std::cout << "Number of Classes: " << modelInfo->numOfClasses << std::endl;

        for (auto i = 0; i < modelInfo->numOfClasses; ++i) {
            uint8_t red, green, blue;
            SaigeColorCodeToRGB(modelInfo->classColors[i], &red, &green, &blue);
            std::cout << modelInfo->classNames[i] << ": color RGB(" << (int)red << ", " << (int)green
                      << ", " << (int)blue << ")" << std::endl;
        }
    }
    /***********************************************************
     *    3. 검사 옵션 확인 및 수정 (Optional)
     *       Check inference options and set new values
     ***********************************************************/
    // 각 클래스 별로 스코어 값(int)을 보정해 주는 옵션입니다. (default= {0,0,0,...}
    SaigeInferenceOption* options = new SaigeInferenceOption;
    retval = SaigeGetInferenceOption(handle, options);
    if (retval < 0) {
        std::cerr << "SaigeGetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        clearResults(results, batchSize);
        return retval;
    }
    if (detailedTime) {
        options->calcTime = true;
        retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_CALC_TIME);
        if (retval < 0) {
            std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            clearResults(results, batchSize);
            return retval;
        }
    }

    options->additionalScores[1] = 50;

    retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_ADDITIONAL_SCORES);
    if (retval < 0) {
        std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        clearResults(results, batchSize);
        return retval;
    }

    if (verbosity >= 4) {
        options->calcClassActivationMap = true;
        retval = SaigeSetInferenceOption(handle, options, SAIGE_SET_CALC_CLASS_ACTIVATION_MAP);
        if (retval < 0) {
            std::cerr << "SaigeSetInferenceOption() failed - " << SaigeStrError(retval)
                      << std::endl;
            SaigeDestroyInferenceHandle(handle);
            clearResults(results, batchSize);
            return retval;
        }
    }

    retval = SaigeGetInferenceOption(handle, options);
    if (retval < 0) {
        std::cerr << "SaigeGetInferenceOption() failed - " << SaigeStrError(retval) << std::endl;
        SaigeDestroyInferenceHandle(handle);
        clearResults(results, batchSize);
        return retval;
    }
    if (verbosity >= 1) {
        std::cout << "[Classification Inference Options]" << std::endl;
        std::cout << "Calc Time: " << (options->calcTime ? "True" : "False") << std::endl;
        std::cout << "Batch Size: " << options->batchSize << std::endl;
        std::cout << "Additional Scores: (" << options->additionalScores[0];
        for (int i = 1; i < modelInfo->numOfClasses; ++i) {
            std::cout << ", " << options->additionalScores[i];
        }
        std::cout << ")" << std::endl;
        std::cout << "Calculate Class Activation Map: "
                  << (options->calcClassActivationMap ? "True" : "False") << std::endl;
    }
    /***********************************************************
     *    4. 이미지 검사하기, Inspect images
     *       이미지 사이즈는 같다고 가정한다.
     ***********************************************************/
    int w, h, c, type;
    long long imreadTime, inferenceTime, postProcessingTime;
    long long inferenceTime_batch = 0;

    // 검사 이미지와 검사 결과의 배치를 받아오기 위한 변수를 할당합니다.
    std::vector<int> classIds;
    std::vector<std::vector<float> > scores;
    classIds.resize(batchSize);
    scores.resize(batchSize);
    for (auto& score : scores) {
        score.resize(modelInfo->numOfClasses);
    }

    unsigned char** ucBatchImageData = new unsigned char*[batchSize];
    for (int i = 0; i < batchSize; ++i) {
        ucBatchImageData[i] = nullptr;
    }

    int numIter = (int)std::ceil((float)imagePaths.size() / (float)batchSize);
    int numFinalImages = imagePaths.size() % batchSize;
    if (numFinalImages == 0) numFinalImages = batchSize;
    int numBatch = batchSize;

    // start iteration
    for (int iter = 0; iter < numIter; ++iter) {
        if (iter == numIter - 1) {
            numBatch = numFinalImages;
        }
        // 이미지 데이터 메모리를 해제합니다. (루프일관성)
        clearImageBatch(ucBatchImageData, batchSize);

        // 이미지 batch를 생성합니다.
        int w_batch = 0, h_batch = 0, c_batch = 0, type_batch = -1;
        bool invalid_batch = false;
        for (int i = 0; i < numBatch; ++i) {
            invalid_batch = false;
            int img_idx = i + iter * batchSize;
            ucBatchImageData[i] =
                (unsigned char*)SaigeGetImageData(imagePaths[img_idx].c_str(), &w, &h, &c, &type);
            if (i == 0) {
                w_batch = w;
                h_batch = h;
                c_batch = c;
                type_batch = type;
            }
            if (ucBatchImageData[i] == nullptr) {
                // 로드에 실패한 이미지가 있으면 다음 배치로 넘어갑니다.
                invalid_batch = true;
                std::cerr << "SaigeGetImageData() failed - " << imagePaths[img_idx] << std::endl;
                break;
            }
            if (w != w_batch || h != h_batch || c != c_batch) {
                // 한 배치 내의 이미지 크기는 모두 같아야 합니다.
                invalid_batch = true;
                std::cerr << "Image sizes in a same batch should be same." << std::endl;
                break;
            }
        }
        unsigned int imageFlag = SaigeConvertImageType(c_batch, type_batch);
        /***********************************************************
         *    4-1. 검사 함수 호출하기, Call Infer API
         ***********************************************************/
        if (verbosity >= 2) {
            std::cout << "\n================================" << std::endl;
            std::cout << "(Start): Infer From Image Batch: " << std::endl;
            for (int i = 0; i < numBatch; ++i) {
                std::cout << "\t: " << imagePaths[i + iter * batchSize] << std::endl;
            }
            std::cout << "---" << std::endl;
        }
        if (invalid_batch) {
            continue;
        }
        retval = SaigeInferFromMultipleByteArray(
            handle, numBatch, (const unsigned char**)ucBatchImageData, w, h, imageFlag, results);
        if (retval < 0) {
            std::cerr << "SaigeInferFromByteArray() failed - " << SaigeStrError(retval)
                      << std::endl;
            continue;
        }

        /***********************************************************
         *    4-2. 검사 결과 받아오기, Retrieve inference results
         ***********************************************************/
        for (int i = 0; i < numBatch; ++i) {  // DEFAULT
            retval = SaigeGetInferenceResultScore(results[i], scores[i].data(), true);
            if (retval < 0) {
                std::cerr << "SaigeGetInferenceResultScore() failed - " << SaigeStrError(retval)
                          << std::endl;
                clearImageBatch(ucBatchImageData, batchSize);
                delete[] ucBatchImageData;
                SaigeDestroyInferenceHandle(handle);
                clearResults(results, batchSize);
                return retval;
            }

            retval = SaigeGetInferenceResultClass(results[i], &classIds[i]);
            if (retval < 0) {
                std::cerr << "SaigeGetInferenceResultClass() failed - " << SaigeStrError(retval)
                          << std::endl;
                clearImageBatch(ucBatchImageData, batchSize);
                delete[] ucBatchImageData;
                SaigeDestroyInferenceHandle(handle);
                clearResults(results, batchSize);
                return retval;
            }

            retval = SaigeGetInferenceResultInspectionTime(results[i], &imreadTime, &inferenceTime,
                                                           &postProcessingTime);
            if (retval < 0) {
                std::cerr << "SaigeGetInferenceResultInspectionTime() failed - "
                          << SaigeStrError(retval) << std::endl;
                clearImageBatch(ucBatchImageData, batchSize);
                delete[] ucBatchImageData;
                SaigeDestroyInferenceHandle(handle);
                clearResults(results, batchSize);
                return retval;
            }
            inferenceTime_batch += imreadTime + inferenceTime + postProcessingTime;

            if (verbosity >= 2) {
                int img_idx = i + iter * batchSize;
                std::cout << imagePaths[img_idx] << std::endl;
                for (int j = 0; j < modelInfo->numOfClasses; j++) {
                    std::cout << "  - " << j << "-th class score: " << scores[i][j] << std::endl;
                }
                std::cout << "  - Result Class: " << classIds[i] << std::endl;
                if (detailedTime) {
                    std::cout << "---\n(End) Image Read, Infer, Postprocess Times (ms): ("
                              << imreadTime << ", " << inferenceTime << ", " << postProcessingTime
                              << ")" << std::endl;
                } else {
                    std::cout << "---\n(End) Inspection Time: " << inferenceTime << " ms"
                              << std::endl;
                }
            }
            if (verbosity >= 4) {
                unsigned char* classActivationMap = nullptr;
                retval = SaigeGetInferenceResultClassActivationMap(results[i], &classActivationMap);
                if (retval < 0) {
                    std::cerr << "SaigeGetInferenceResultClassActivationMap() failed - "
                              << SaigeStrError(retval) << std::endl;
                    clearImageBatch(ucBatchImageData, batchSize);
                    delete[] ucBatchImageData;
                    SaigeDestroyInferenceHandle(handle);
                    clearResults(results, batchSize);
                    return retval;
                }
                SaigeDrawImage(classActivationMap, w, h, 1, 1, SaigeImageType::SV_U8, false,
                               waitkeyMilliSecond);
                SaigeDrawImageOverBase(ucBatchImageData[i], classActivationMap, w, h, c, 1, type,
                                       SaigeImageType::SV_U8, 0.5, false, waitkeyMilliSecond);
            }
        }
    }
    clearImageBatch(ucBatchImageData, batchSize);
    delete[] ucBatchImageData;
    std::cout << "Total time to infer with batch images: " << inferenceTime_batch << " ms"
              << std::endl;

    delete modelInfo;
    delete options;
    /***********************************************************
     *    6. 핸들 파괴, Destruct handles
     ***********************************************************/
    for (int i = 0; i < batchSize; ++i) {
        retval = SaigeDestroyInferenceResult(results[i]);
        if (retval < 0) {
            std::cerr << "SaigeDestroyInferenceResult() Failed -" << SaigeStrError(retval)
                      << std::endl;
            return retval;
        }
    }
    retval = SaigeDestroyInferenceHandle(handle);
    if (retval < 0) {
        std::cerr << "SaigeDestroyInferenceHandle() Failed -" << SaigeStrError(retval) << std::endl;
        return retval;
    }
    return 0;
}
}  // namespace cls