diff --git a/BUILD.gn b/BUILD.gn index fce5c6565..7098eb385 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -56,6 +56,9 @@ group("plugins") { if (enable_jpeg_hw_decode) { deps += [ "frameworks/innerkitsimpl/test/unittest/jpeg_hw_decode/demo:jpeg_hw_decoder_demo" ] } + if (enable_heif_hw_decode) { + deps += [ "frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo:heif_hw_decoder_demo" ] + } } } diff --git a/bundle.json b/bundle.json index 6cebd9d09..df2bf5539 100644 --- a/bundle.json +++ b/bundle.json @@ -42,6 +42,7 @@ "drivers_interface_codec", "drivers_interface_display", "drivers_peripheral_display", + "drivers_peripheral_codec", "hdf_core", "memmgr_override", "libjpeg-turbo", diff --git a/frameworks/innerkitsimpl/test/BUILD.gn b/frameworks/innerkitsimpl/test/BUILD.gn index ed491f67e..e63bc63fb 100644 --- a/frameworks/innerkitsimpl/test/BUILD.gn +++ b/frameworks/innerkitsimpl/test/BUILD.gn @@ -183,7 +183,7 @@ ohos_unittest("imagesourcetest") { ] if (enable_heif_hw_decode) { - source += [ "$image_subsystem/frameworks/innerkitsimpl/test/unittest/image_source_test/image_source_heif_test.cpp" ] + sources += [ "$image_subsystem/frameworks/innerkitsimpl/test/unittest/image_source_test/image_source_heif_test.cpp" ] } if (DUAL_ADAPTER) { @@ -1189,6 +1189,60 @@ ohos_unittest("jpeg_hw_decoder_test") { ] } +ohos_unittest("heif_hw_decoder_test") { + module_out_path = module_output_path + + sources = [ + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/heif_hw_decoder.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/codec_state.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/format.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_buffer.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_dfx.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_list.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_decoder.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/msg_handle_loop.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/param_bundle.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/state_machine.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/type_converter.cpp", + "unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.cpp", + "unittest/heif_hw_decode/unittest/heif_hw_decoder_test.cpp", + ] + include_dirs = [ + "${image_subsystem}/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/", + "${image_subsystem}/frameworks/innerkitsimpl/utils/include/", + "${image_subsystem}/interfaces/innerkits/include/", + "${image_subsystem}/plugins/common/libs/image/libextplugin/include/", + "${image_subsystem}/../../../base/hiviewdfx/hilog/interfaces/native/innerkits/include/", + "${image_subsystem}/../../../base/hiviewdfx/hitrace/interfaces/native/innerkits/include/hitrace_meter/", + "${image_subsystem}/../../../base/startup/init/interfaces/innerkits/include/", + "${image_subsystem}/../../../commonlibrary/c_utils/base/include/", + "${image_subsystem}/../../../drivers/hdf_core/interfaces/inner_api/", + "${image_subsystem}/../../../drivers/peripheral/codec/interfaces/include/", + "${image_subsystem}/../../../foundation/graphic/graphic_surface/interfaces/inner_api/surface/", + "${image_subsystem}/../../../foundation/resourceschedule/qos_manager/interfaces/inner_api", + "foundation/multimedia/image_framework/interfaces/innerkits/include", + ] + deps = [ + "${image_subsystem}/frameworks/innerkitsimpl/utils:image_utils", + "//third_party/googletest:gtest_main", + ] + external_deps = [ + "c_utils:utils", + "drivers_interface_codec:codec_idl_headers", + "drivers_interface_codec:libcodec_proxy_3.0", + "graphic_surface:surface", + "hdf_core:libhdi", + "hilog:libhilog", + "hitrace:hitrace_meter", + "init:libbegetutil", + "ipc:ipc_core", + "openmax:libopenmax_static", + "qos_manager:qos", + ] + resource_config_file = "$image_subsystem/test/resource/image/ohos_test.xml" +} + ohos_unittest("heif_parser_test") { module_out_path = module_output_path @@ -1212,7 +1266,6 @@ ohos_unittest("heif_parser_test") { "${image_subsystem}/../../../foundation/graphic/graphic_surface/interfaces/inner_api/surface/", "${image_subsystem}/../../../foundation/multimedia/image_framework/interfaces/innerkits/include", "${image_subsystem}/../../../foundation/resourceschedule/qos_manager/interfaces/inner_api/", - "${image_subsystem}/../../../third_party/openmax/api/1.1.2/", ] deps = [ @@ -1232,6 +1285,7 @@ ohos_unittest("heif_parser_test") { "hitrace:hitrace_meter", "init:libbegetutil", "ipc:ipc_core", + "openmax:libopenmax_static", "qos_manager:qos", ] @@ -1752,7 +1806,12 @@ group("unittest") { } if (enable_heif_hw_decode) { - deps += [ ":heifyuvtest" ] + deps += [ + ":heif_hw_decoder_test", + ":heifyuvtest", + ] } } + ################################################ + diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.cpp b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.cpp new file mode 100644 index 000000000..09129afb5 --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.cpp @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "command_parser.h" +#include +#include + +namespace OHOS::ImagePlugin { +using namespace std; + +enum ShortOption { + OPT_UNKONWN = 0, + OPT_HELP, + OPT_INPUT = 'i', + OPT_PIXEL_FORMAT +}; + +static struct option g_longOptions[] = { + {"help", no_argument, nullptr, static_cast(ShortOption::OPT_HELP)}, + {"in", required_argument, nullptr, static_cast(ShortOption::OPT_INPUT)}, + {"pixelFormat", required_argument, nullptr, static_cast(ShortOption::OPT_PIXEL_FORMAT)}, + {nullptr, no_argument, nullptr, static_cast(ShortOption::OPT_UNKONWN)}, +}; + +void ShowUsage() +{ + std::cout << "Heif Hardware decode Demo Options:" << std::endl; + std::cout << " --help help info." << std::endl; + std::cout << " -i, --in full file path for input file." << std::endl; + std::string pixFmtHelpInfo = "pixel format of output. 0 is NV12, 1 is NV21, 2 is NV12_10bit, 3 is NV21_10bit"; + std::cout << " --pixelFormat " << pixFmtHelpInfo << std::endl; +} + +CommandOpt Parse(int argc, char *argv[]) +{ + CommandOpt opt; + int c; + while ((c = getopt_long(argc, argv, "i:", g_longOptions, nullptr)) != -1) { + switch (static_cast(c)) { + case ShortOption::OPT_HELP: + ShowUsage(); + break; + case ShortOption::OPT_INPUT: + opt.inputPath = string(optarg); + break; + case ShortOption::OPT_PIXEL_FORMAT: + opt.pixelFormat = static_cast(stol(optarg)); + break; + default: + break; + } + } + return opt; +} + +void CommandOpt::Print() const +{ + std::string pixelFmtDesc = "unknown"; + switch (pixelFormat) { + case UserPixelFormat::NV12: + pixelFmtDesc = "NV12"; + break; + case UserPixelFormat::NV21: + pixelFmtDesc = "NV21"; + break; + case UserPixelFormat::NV12_10bit: + pixelFmtDesc = "NV12_10bit"; + break; + case UserPixelFormat::NV21_10bit: + pixelFmtDesc = "NV21_10bit"; + break; + default: + break; + } + std::cout << "=========================== OPT INFO ===========================" << endl; + std::cout << " inputPath : " << inputPath << endl; + std::cout << " pixelFormat : " << pixelFmtDesc << endl; + std::cout << "=================================================================" << endl; +} + +} // OHOS::ImagePlugin diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.h b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.h new file mode 100644 index 000000000..197b9b34e --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HEIF_COMMAND_PARSER_H +#define HEIF_COMMAND_PARSER_H + +#include + +namespace OHOS::ImagePlugin { +enum class UserPixelFormat { + NV12 = 0, + NV21 = 1, + NV12_10bit = 2, + NV21_10bit = 3 +}; + +struct CommandOpt { + UserPixelFormat pixelFormat; + std::string inputPath; + void Print() const; +}; + +CommandOpt Parse(int argc, char *argv[]); +void ShowUsage(); +} // OHOS::ImagePlugin + +#endif // HEIF_COMMAND_PARSER_H diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.cpp b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.cpp new file mode 100644 index 000000000..66ea34bf5 --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mock_heif_hw_decode_flow.h" +#include "hardware/imagecodec/image_codec_log.h" +#include "media_errors.h" +#include +#include +#include +#include +#include +#include + +namespace OHOS::ImagePlugin { +using namespace std; + +void HeifHwDecoderFlow::InputParser::SplitString(const std::string& src, char sep, std::vector& vec) +{ + vec.clear(); + string::size_type startPos = 0; + while (true) { + string::size_type endPos = src.find_first_of(sep, startPos); + if (endPos == string::npos) { + break; + } + vec.emplace_back(src.substr(startPos, endPos - startPos)); + startPos = endPos + 1; + } + if (startPos != string::npos) { + vec.emplace_back(src.substr(startPos)); + } +} + +std::string HeifHwDecoderFlow::InputParser::JoinPath(const std::string& base, const std::string& append) +{ + return (filesystem::path(base) / append).string(); +} + +bool HeifHwDecoderFlow::InputParser::ParseGridInfo(GridInfo& gridInfo) +{ + // source_ demo: + // 1. has grid: 3072x4096_grid_512x512_6x8 + // 2. no grid: 3072x4096_nogrid + string baseDir = filesystem::path(source_).filename().string(); + vector vec; + SplitString(baseDir, MAIN_SEP, vec); + IF_TRUE_RETURN_VAL_WITH_MSG(vec.size() < MIN_MAIN_SEG_CNT, false, + "invalid source: %{public}s", source_.c_str()); + + vector vecTmp; + SplitString(vec[DISPLAY_SIZE], SUB_SEP, vecTmp); + IF_TRUE_RETURN_VAL_WITH_MSG(vecTmp.size() != SUB_SEG_CNT, false, "invalid source: %{public}s", source_.c_str()); + gridInfo.displayWidth = static_cast(stol(vecTmp[HORIZONTAL].c_str())); + gridInfo.displayHeight = static_cast(stol(vecTmp[VERTICAL].c_str())); + + if (vec[GRID_FLAG].find(NO_GRID_INDICATOR) != string::npos) { + gridInfo.enableGrid = false; + gridInfo.cols = 0; + gridInfo.rows = 0; + gridInfo.tileWidth = 0; + gridInfo.tileHeight = 0; + } else { + IF_TRUE_RETURN_VAL_WITH_MSG(vec.size() < MAX_MAIN_SEG_CNT, false, + "invalid source: %{public}s", source_.c_str()); + + gridInfo.enableGrid = true; + + SplitString(vec[TILE_SIZE], SUB_SEP, vecTmp); + IF_TRUE_RETURN_VAL_WITH_MSG(vecTmp.size() != SUB_SEG_CNT, false, + "invalid source: %{public}s", source_.c_str()); + gridInfo.tileWidth = static_cast(stol(vecTmp[HORIZONTAL].c_str())); + gridInfo.tileHeight = static_cast(stol(vecTmp[VERTICAL].c_str())); + + SplitString(vec[GRID_SIZE], SUB_SEP, vecTmp); + IF_TRUE_RETURN_VAL_WITH_MSG(vecTmp.size() != SUB_SEG_CNT, false, + "invalid source: %{public}s", source_.c_str()); + gridInfo.cols = static_cast(stol(vecTmp[HORIZONTAL].c_str())); + gridInfo.rows = static_cast(stol(vecTmp[VERTICAL].c_str())); + } + return true; +} + +void HeifHwDecoderFlow::InputParser::FindXpsAndIFrameFile() +{ + DIR *dirp = opendir(source_.c_str()); + IF_TRUE_RETURN_VOID_WITH_MSG(dirp == nullptr, "failed to open: %{public}s, errno=%{public}d", + source_.c_str(), errno); + struct dirent *dp; + while ((dp = readdir(dirp)) != nullptr) { + if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) { + continue; + } + string path = JoinPath(source_, dp->d_name); + struct stat st{}; + if (stat(path.c_str(), &st) != 0 || !S_ISREG(st.st_mode)) { + continue; + } + string fileName(dp->d_name); + if (fileName.find(XPS_INDICATOR) != string::npos) { + xpsFile_ = path; + } else if (fileName.find(I_FRAME_INDICATOR) != string::npos) { + iFrameFile_.emplace_back(path); + } + } + closedir(dirp); +} + +bool HeifHwDecoderFlow::InputParser::ReadFileToVec(const string& filePath, vector>& inputs) +{ + ifstream ifs(filePath, ios::binary); + IF_TRUE_RETURN_VAL_WITH_MSG(!ifs.is_open(), false, "failed to open file: %{public}s", filePath.c_str()); + + ifs.seekg(0, ifstream::end); + size_t fileSize = static_cast(ifs.tellg()); + ifs.seekg(0, ifstream::beg); + + vector vec(fileSize); + ifs.read(reinterpret_cast(vec.data()), static_cast(fileSize)); + ifs.close(); + inputs.emplace_back(vec); + return true; +} + +int HeifHwDecoderFlow::InputParser::ExtractIFrameNum(const string& filePath) +{ + string fileName = filesystem::path(filePath).filename().string(); + string::size_type pos = fileName.find(I_FRAME_INDICATOR); + if (pos == string::npos) { + return -1; + } + return stoi(fileName.substr(pos + string(I_FRAME_INDICATOR).size())); +} + +bool HeifHwDecoderFlow::InputParser::ReadInput(vector>& inputs) +{ + FindXpsAndIFrameFile(); + IF_TRUE_RETURN_VAL_WITH_MSG(xpsFile_.empty(), false, "no xps file in %{public}s", source_.c_str()); + IF_TRUE_RETURN_VAL_WITH_MSG(iFrameFile_.empty(), false, "no iframe file in %{public}s", source_.c_str()); + + IF_TRUE_RETURN_VAL_WITH_MSG(!ReadFileToVec(xpsFile_, inputs), false, + "failed to read xps file: %{public}s", xpsFile_.c_str()); + std::sort(iFrameFile_.begin(), iFrameFile_.end(), [](const string& a, const string& b) { + return ExtractIFrameNum(a) < ExtractIFrameNum(b); + }); + for (const string& one : iFrameFile_) { + IF_TRUE_RETURN_VAL_WITH_MSG(!ReadFileToVec(one, inputs), false, + "failed to read iframe file: %{public}s", one.c_str()); + } + return true; +} + +HeifHwDecoderFlow::~HeifHwDecoderFlow() +{ + output_ = nullptr; +} + +bool HeifHwDecoderFlow::Run(const CommandOpt& opt) +{ + bool ret = PrepareInput(opt.inputPath); + ret = ret && AllocOutput(opt.pixelFormat); + ret = ret && DoDecode(); + if (ret) { + LOGI("demo succeed"); + } else { + LOGE("demo failed"); + } + return ret; +} + +bool HeifHwDecoderFlow::PrepareInput(const std::string& inputPath) +{ + InputParser parser(inputPath); + bool ret = parser.ParseGridInfo(gridInfo_); + ret = ret && parser.ReadInput(input_); + return ret; +} + +bool HeifHwDecoderFlow::AllocOutput(UserPixelFormat userPixelFormat) +{ + static const map userPixelFmtToGraphicPixelFmt = { + { UserPixelFormat::NV12, GRAPHIC_PIXEL_FMT_YCBCR_420_SP }, + { UserPixelFormat::NV21, GRAPHIC_PIXEL_FMT_YCRCB_420_SP }, + { UserPixelFormat::NV12_10bit, GRAPHIC_PIXEL_FMT_YCBCR_P010 }, + { UserPixelFormat::NV21_10bit, GRAPHIC_PIXEL_FMT_YCRCB_P010 }, + }; + auto iter = userPixelFmtToGraphicPixelFmt.find(userPixelFormat); + if (iter == userPixelFmtToGraphicPixelFmt.end()) { + LOGE("unsupported pixel format: %{public}d", static_cast(userPixelFormat)); + return false; + } + GraphicPixelFormat pixelFmt = iter->second; + output_ = hwDecoder_.AllocateOutputBuffer(gridInfo_.displayWidth, gridInfo_.displayHeight, pixelFmt); + if (output_ == nullptr) { + LOGE("failed to alloc output"); + return false; + } + return true; +} + +bool HeifHwDecoderFlow::DoDecode() +{ + uint32_t ret = hwDecoder_.DoDecode(gridInfo_, input_, output_); + if (ret != Media::SUCCESS) { + LOGE("failed to decode"); + return false; + } + return true; +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.h b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.h new file mode 100644 index 000000000..0b624ab0f --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HEIF_HW_DECODER_DEMO_H +#define HEIF_HW_DECODER_DEMO_H + +#include +#include "command_parser.h" +#include "hardware/heif_hw_decoder.h" + +namespace OHOS::ImagePlugin { +class HeifHwDecoderFlow { +public: + HeifHwDecoderFlow() = default; + ~HeifHwDecoderFlow(); + bool Run(const CommandOpt& opt); +private: + class InputParser { + public: + explicit InputParser(const std::string& inputPath) : source_(inputPath) {} + ~InputParser() = default; + bool ParseGridInfo(GridInfo& gridInfo); + bool ReadInput(std::vector>& inputs); + private: + void FindXpsAndIFrameFile(); + static void SplitString(const std::string& src, char sep, std::vector& vec); + static std::string JoinPath(const std::string& base, const std::string& append); + static bool ReadFileToVec(const std::string& filePath, std::vector>& inputs); + static int ExtractIFrameNum(const std::string& filePath); + + static constexpr char MAIN_SEP = '_'; + static constexpr size_t MIN_MAIN_SEG_CNT = 2; + static constexpr size_t MAX_MAIN_SEG_CNT = 4; + static constexpr char SUB_SEP = 'x'; + static constexpr size_t SUB_SEG_CNT = 2; + static constexpr char NO_GRID_INDICATOR[] = "nogrid"; + static constexpr char XPS_INDICATOR[] = "_hevc_xps"; + static constexpr char I_FRAME_INDICATOR[] = "_hevc_I"; + enum MainSeg { + DISPLAY_SIZE = 0, + GRID_FLAG, + TILE_SIZE, + GRID_SIZE + }; + enum SubSeg { + HORIZONTAL = 0, + VERTICAL + }; + + std::string source_; + std::string xpsFile_; + std::vector iFrameFile_; + }; +private: + bool PrepareInput(const std::string& inputPath); + bool AllocOutput(UserPixelFormat userPixelFormat); + bool DoDecode(); +private: + GridInfo gridInfo_; + std::vector> input_; + sptr output_; + HeifHardwareDecoder hwDecoder_; +}; +} // OHOS::ImagePlugin + +#endif // HEIF_HW_DECODER_DEMO_H diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/BUILD.gn b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/BUILD.gn new file mode 100644 index 000000000..dcdfff7ec --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/BUILD.gn @@ -0,0 +1,76 @@ +# Copyright (c) 2023 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import("//build/ohos.gni") +import("//foundation/multimedia/image_framework/ide/image_decode_config.gni") + +ohos_executable("heif_hw_decoder_demo") { + install_enable = false + subsystem_name = "multimedia" + part_name = "image_framework" + + if (!use_clang_android && !use_clang_ios) { + sanitize = { + cfi = true + cfi_cross_dso = true + debug = false + } + } + + if (build_variant == "root") { + defines = [ "BUILD_ENG_VERSION" ] + } + + sources = [ + "${image_subsystem}/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/command_parser.cpp", + "${image_subsystem}/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/mock_heif_hw_decode_flow.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/heif_hw_decoder.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/codec_state.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/format.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_buffer.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_dfx.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_list.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_decoder.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/msg_handle_loop.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/param_bundle.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/state_machine.cpp", + "${image_subsystem}/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/type_converter.cpp", + "heif_hw_decoder_demo.cpp", + ] + + include_dirs = [ + "${image_subsystem}/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/common/", + "${image_subsystem}/interfaces/innerkits/include/", + "${image_subsystem}/plugins/common/libs/image/libextplugin/include/", + ] + + deps = [ "${image_subsystem}/interfaces/innerkits:image_native" ] + + external_deps = [ + "c_utils:utils", + "drivers_interface_codec:codec_idl_headers", + "drivers_interface_codec:libcodec_proxy_3.0", + "drivers_peripheral_codec:libcodec_hdi_omx_client", + "graphic_surface:surface", + "hdf_core:libhdi", + "hilog:libhilog", + "init:libbegetutil", + "ipc:ipc_core", + "openmax:libopenmax_static", + "qos_manager:qos", + ] + if (build_variant == "root") { + external_deps += [ "hitrace:hitrace_meter" ] + } +} diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/heif_hw_decoder_demo.cpp b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/heif_hw_decoder_demo.cpp new file mode 100644 index 000000000..a5d51f30d --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/demo/heif_hw_decoder_demo.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "mock_heif_hw_decode_flow.h" + +namespace OHOS::ImagePlugin { +using namespace std; + +extern "C" { +int main(int argc, char *argv[]) +{ + CommandOpt opt = Parse(argc, argv); + if (!opt.inputPath.empty()) { + opt.Print(); + HeifHwDecoderFlow demo; + (void)demo.Run(opt); + } + return 0; +} +} +} diff --git a/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/unittest/heif_hw_decoder_test.cpp b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/unittest/heif_hw_decoder_test.cpp new file mode 100644 index 000000000..7ade49de9 --- /dev/null +++ b/frameworks/innerkitsimpl/test/unittest/heif_hw_decode/unittest/heif_hw_decoder_test.cpp @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "hardware/heif_hw_decoder.h" +#include "image_system_properties.h" +#include "mock_heif_hw_decode_flow.h" +#include "media_errors.h" // foundation/multimedia/image_framework/interfaces/innerkits/include/ + +namespace OHOS::Media { +using namespace testing::ext; +using namespace OHOS::ImagePlugin; + +class HeifHwDecoderTest : public testing::Test { +public: + static constexpr char TEST_HEIF_IMG_NO_GRID[] = "/data/local/tmp/image/heif_test/1024x1024_nogrid"; + static constexpr char TEST_HEIF_IMG_WITH_GRID[] = "/data/local/tmp/image/heif_test/1024x1024_grid_512x512_2x2"; +}; + +HWTEST_F(HeifHwDecoderTest, AllocOutputBufferWithInvalidSize, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + sptr output = testObj.AllocateOutputBuffer( + 0, 512, static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + ASSERT_TRUE(output == nullptr); +} + +HWTEST_F(HeifHwDecoderTest, AllocOutputBufferWithInvalidPixelFmt, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + sptr output = testObj.AllocateOutputBuffer( + 512, 512, static_cast(GRAPHIC_PIXEL_FMT_RGBA_8888)); + ASSERT_TRUE(output == nullptr); +} + +HWTEST_F(HeifHwDecoderTest, AllocOutputBufferOk, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + sptr output = testObj.AllocateOutputBuffer( + 1024, 512, static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + ASSERT_TRUE(output != nullptr); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidInputs, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 1024, + .displayHeight = 512, + .enableGrid = false, + .cols = 0, + .rows = 0, + .tileWidth = 0, + .tileHeight = 0 + }; + std::vector> inputs; + sptr output = testObj.AllocateOutputBuffer(gridInfo.displayWidth, gridInfo.displayHeight, + static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidOutputBuffer, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 1024, + .displayHeight = 512, + .enableGrid = false, + .cols = 0, + .rows = 0, + .tileWidth = 0, + .tileHeight = 0 + }; + std::vector> inputs; + inputs.emplace_back(std::vector(1)); + inputs.emplace_back(std::vector(1)); + sptr output = nullptr; + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidGridInfo1, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 512, + .displayHeight = 512, + .enableGrid = true, + .cols = 0, + .rows = 0, + .tileWidth = 0, + .tileHeight = 0 + }; + std::vector> inputs; + inputs.emplace_back(std::vector(1)); + inputs.emplace_back(std::vector(1)); + sptr output = testObj.AllocateOutputBuffer(gridInfo.displayWidth, gridInfo.displayHeight, + static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidGridInfo2, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 1024, + .displayHeight = 1024, + .enableGrid = true, + .cols = 1, + .rows = 2, + .tileWidth = 512, + .tileHeight = 512 + }; + std::vector> inputs; + inputs.emplace_back(std::vector(1)); + inputs.emplace_back(std::vector(1)); + sptr output = testObj.AllocateOutputBuffer(gridInfo.displayWidth, gridInfo.displayHeight, + static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidGridInfo3, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 512, + .displayHeight = 512, + .enableGrid = true, + .cols = 1, + .rows = 1, + .tileWidth = 512, + .tileHeight = 0 + }; + std::vector> inputs; + inputs.emplace_back(std::vector(1)); + inputs.emplace_back(std::vector(1)); + sptr output = testObj.AllocateOutputBuffer(gridInfo.displayWidth, gridInfo.displayHeight, + static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeWithInvalidGridInfo4, TestSize.Level1) +{ + HeifHardwareDecoder testObj; + GridInfo gridInfo = { + .displayWidth = 513, + .displayHeight = 512, + .enableGrid = true, + .cols = 1, + .rows = 1, + .tileWidth = 512, + .tileHeight = 512 + }; + std::vector> inputs; + inputs.emplace_back(std::vector(1)); + inputs.emplace_back(std::vector(1)); + sptr output = testObj.AllocateOutputBuffer(gridInfo.displayWidth, gridInfo.displayHeight, + static_cast(GRAPHIC_PIXEL_FMT_YCBCR_420_SP)); + uint32_t ret = testObj.DoDecode(gridInfo, inputs, output); + ASSERT_TRUE(ret != Media::SUCCESS); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeOkNoGrid, TestSize.Level1) +{ + bool ret = true; + if (ImageSystemProperties::GetHardWareDecodeEnabled()) { + CommandOpt opt = { + .pixelFormat = UserPixelFormat::NV21, + .inputPath = TEST_HEIF_IMG_NO_GRID + }; + HeifHwDecoderFlow testObj; + ret = testObj.Run(opt); + } + ASSERT_TRUE(ret); +} + +HWTEST_F(HeifHwDecoderTest, DoDecodeOkWithGrid, TestSize.Level1) +{ + bool ret = true; + if (ImageSystemProperties::GetHardWareDecodeEnabled()) { + CommandOpt opt = { + .pixelFormat = UserPixelFormat::NV21, + .inputPath = TEST_HEIF_IMG_WITH_GRID + }; + HeifHwDecoderFlow testObj; + ret = testObj.Run(opt); + } + ASSERT_TRUE(ret); +} +} // namespace OHOS::Media \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/BUILD.gn b/plugins/common/libs/image/libextplugin/BUILD.gn index 7d242fc8a..cb5230627 100644 --- a/plugins/common/libs/image/libextplugin/BUILD.gn +++ b/plugins/common/libs/image/libextplugin/BUILD.gn @@ -455,7 +455,7 @@ ohos_shared_library("heifimpl") { external_deps += [ "c_utils:utils", "drivers_interface_codec:codec_idl_headers", - "drivers_interface_codec:libcodec_proxy_2.0", + "drivers_interface_codec:libcodec_proxy_3.0", "drivers_peripheral_codec:libcodec_hdi_omx_client", "ffmpeg:libohosffmpeg", "hdf_core:libhdi", diff --git a/plugins/common/libs/image/libextplugin/include/hardware/heif_hw_decoder.h b/plugins/common/libs/image/libextplugin/include/hardware/heif_hw_decoder.h new file mode 100644 index 000000000..fd9669a08 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/heif_hw_decoder.h @@ -0,0 +1,118 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef HEIF_HW_DECODER_H +#define HEIF_HW_DECODER_H + +#include +#include +#include +#include +#include + #include "imagecodec/image_codec.h" + +namespace OHOS { +namespace ImagePlugin { +struct GridInfo { + uint32_t displayWidth = 0; + uint32_t displayHeight = 0; + bool enableGrid = false; + uint32_t cols = 0; + uint32_t rows = 0; + uint32_t tileWidth = 0; + uint32_t tileHeight = 0; + + bool IsValid() const; +}; + +class HeifHardwareDecoder { +public: + HeifHardwareDecoder(); + ~HeifHardwareDecoder(); + sptr AllocateOutputBuffer(uint32_t width, uint32_t height, int32_t pixelFmt); + uint32_t DoDecode(const GridInfo& gridInfo, std::vector>& inputs, sptr& output); +private: + class HeifDecoderCallback : public ImageCodecCallback { + public: + HeifDecoderCallback(HeifHardwareDecoder* heifDecoder); + void OnError(ImageCodecError err) override; + void OnOutputFormatChanged(const Format &format) override; + void OnInputBufferAvailable(uint32_t index, std::shared_ptr buffer) override; + void OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer) override; + HeifHardwareDecoder* heifDecoder_; + }; +private: + struct RawYuvCopyInfo { + uint8_t* yStart = 0; + uint8_t* uvStart = 0; + uint32_t stride = 0; + uint32_t yStride = 0; + uint32_t yOffset = 0; + uint32_t uvOffset = 0; + }; +private: + static bool GetUvPlaneOffsetFromSurfaceBuffer(sptr& surfaceBuffer, uint64_t& offset); + bool IsHardwareDecodeSupported(const GridInfo& gridInfo); + bool SetCallbackForDecoder(); + bool ConfigureDecoder(const GridInfo& gridInfo, sptr& output); + bool SetOutputBuffer(const GridInfo& gridInfo, sptr output); + bool WaitForOmxToReturnInputBuffer(uint32_t& bufferId, std::shared_ptr& buffer); + int32_t PrepareInputCodecBuffer(const std::vector>& inputs, size_t inputIndex, + std::shared_ptr& buffer); + void SendInputBufferLoop(const std::vector>& inputs); + bool WaitForOmxToReturnOutputBuffer(uint32_t& bufferId, std::shared_ptr& buffer); + void AssembleOutput(uint32_t outputIndex, std::shared_ptr& buffer); + static uint32_t CalculateDirtyLen(uint32_t displayLen, uint32_t gridLen, uint32_t totalGrid, uint32_t curGrid); + static bool CopyRawYuvData(const RawYuvCopyInfo& src, const RawYuvCopyInfo& dst, + uint32_t dirtyWidth, uint32_t dirtyHeight); + void ReceiveOutputBufferLoop(); + static int64_t GetTimestampInUs(); + void ReleaseDecoder(); + + void SignalError(); + bool HasError(); + void Reset(); + + void FlushOutput(); + std::string GetOutputPixelFmtDesc(); + void DumpOutput(); +private: + static constexpr int32_t BUFFER_CIRCULATE_TIMEOUT_IN_MS = 1000; + static constexpr uint32_t SAMPLE_RATIO_FOR_YUV420_SP = 2; + static constexpr size_t MIN_SIZE_OF_INPUT = 2; + + std::shared_ptr heifDecoderImpl_; + + sptr output_; + uint64_t uvOffsetForOutput_; + GridInfo gridInfo_; + + std::mutex errMtx_; + bool hasErr_ = false; + + std::mutex inputMtx_; + std::condition_variable inputCond_; + std::list>> inputList_; + + std::mutex outputMtx_; + std::condition_variable outputCond_; + std::list>> outputList_; + + std::thread releaseThread_; +}; +} // namespace ImagePlugin +} // namespace OHOS + +#endif // HEIF_HW_DECODER_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/format.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/format.h new file mode 100644 index 000000000..5327049e4 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/format.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_FORMAT_H +#define IMAGE_CODEC_FORMAT_H + +#include +#include +#include +#include + +namespace OHOS::ImagePlugin { +class Format { +public: + Format() = default; + ~Format() = default; + explicit Format(const Format& src); + Format& operator=(const Format& src); + + bool ContainKey(const std::string &key) const; + + template + void SetValue(const std::string &key, const T &value) + { + m_items[key] = value; + } + + template + bool GetValue(const std::string &key, T &value) const + { + const auto it = m_items.find(key); + if (it == m_items.end()) { + return false; + } + value = std::any_cast(it->second); + return true; + } + +private: + std::unordered_map m_items; +}; +}; // namespace OHOS::ImagePlugin +#endif // IMAGE_CODEC_FORMAT_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/hdi_define.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/hdi_define.h new file mode 100644 index 000000000..7ca966595 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/hdi_define.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_HDI_DEFINE_H +#define IMAGE_HDI_DEFINE_H + +#include "v3_0/codec_types.h" +#include "v3_0/icodec_callback.h" +#include "v3_0/icodec_component.h" +#include "v3_0/icodec_component_manager.h" + +namespace OHOS::ImagePlugin { +namespace HdiCodecNamespace = OHOS::HDI::Codec::V3_0; +} // namespace OHOS::ImagePlugin + +#endif // IMAGE_HDI_DEFINE_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec.h new file mode 100644 index 000000000..4b6420d4c --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec.h @@ -0,0 +1,423 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_H +#define IMAGE_CODEC_H + +#include +#include +#include +#include "securec.h" +#include "OMX_Component.h" // third_party/openmax/api/1.1.2 +#include "param_bundle.h" +#include "image_codec_buffer.h" +#include "image_codec_common.h" +#include "format.h" +#include "state_machine.h" +#include "type_converter.h" +#include "codec_omx_ext.h" +#include "hdi_define.h" + +namespace OHOS::ImagePlugin { +inline constexpr int TIME_RATIO_S_TO_MS = 1000; +inline constexpr double US_TO_MS = 1000.0; +inline constexpr double US_TO_S = 1000000.0; +inline constexpr uint32_t STRIDE_ALIGNMENT = 32; + +inline uint32_t GetYuv420Size(uint32_t w, uint32_t h) +{ + return w * h * 3 / 2; // 3: nom of ratio, 2: denom of ratio +} + +class ImageCodec : protected StateMachine { +public: + static std::shared_ptr Create(); + std::string GetComponentName() const { return componentName_; } + int32_t SetCallback(const std::shared_ptr &callback); + int32_t Configure(const Format &format); + int32_t QueueInputBuffer(uint32_t index); + int32_t ReleaseOutputBuffer(uint32_t index); + int32_t GetInputFormat(Format& format); + int32_t GetOutputFormat(Format& format); + int32_t Start(); + int32_t Release(); + int32_t GetOutputBufferUsage(uint64_t& usage); + int32_t SetOutputBuffer(sptr output); +protected: + enum MsgWhat : MsgType { + INIT, + SET_CALLBACK, + CONFIGURE, + START, + GET_INPUT_FORMAT, + GET_OUTPUT_FORMAT, + QUEUE_INPUT_BUFFER, + RELEASE_OUTPUT_BUFFER, + RELEASE, + GET_OUTPUT_BUFFER_USAGE, + SET_OUTPUT_BUFFER, + + INNER_MSG_BEGIN = 1000, + CODEC_EVENT, + OMX_EMPTY_BUFFER_DONE, + OMX_FILL_BUFFER_DONE, + CHECK_IF_STUCK, + FORCE_SHUTDOWN, + }; + + enum BufferOperationMode { + KEEP_BUFFER, + RESUBMIT_BUFFER, + FREE_BUFFER, + }; + + enum BufferOwner { + OWNED_BY_US = 0, + OWNED_BY_USER = 1, + OWNED_BY_OMX = 2, + OWNER_CNT = 3, + }; + + struct PortInfo { + uint32_t width; + uint32_t height; + OMX_VIDEO_CODINGTYPE codingType; + std::optional pixelFmt; + double frameRate; + std::optional inputBufSize; + std::optional bufferCnt; + }; + + struct BufferInfo { + BufferInfo() : lastOwnerChangeTime(std::chrono::steady_clock::now()) {} + bool isInput = true; + BufferOwner owner = OWNED_BY_US; + std::chrono::time_point lastOwnerChangeTime; + uint32_t bufferId = 0; + std::shared_ptr omxBuffer; + sptr surfaceBuffer; + std::shared_ptr imgCodecBuffer; + + void CleanUpUnusedInfo(); + void BeginCpuAccess(); + void EndCpuAccess(); + bool IsValidFrame() const; + void Dump(const std::string& prefix, bool dumpMode) const; + + private: + void Dump(const std::string& prefix) const; + void DumpSurfaceBuffer(const std::string& prefix) const; + void DumpLinearBuffer(const std::string& prefix) const; + static constexpr char DUMP_PATH[] = "/data/misc/imagecodecdump"; + }; +protected: + ImageCodec(OMX_VIDEO_CODINGTYPE codingType, bool isEncoder); + ~ImageCodec() override; + static const char* ToString(MsgWhat what); + static const char* ToString(BufferOwner owner); + void ReplyErrorCode(MsgId id, int32_t err); + void PrintAllBufferInfo(); + std::array CountOwner(bool isInput); + void ChangeOwner(BufferInfo& info, BufferOwner newOwner); + void UpdateInputRecord(const BufferInfo& info, std::chrono::time_point now); + void UpdateOutputRecord(const BufferInfo& info, std::chrono::time_point now); + + // configure + virtual int32_t OnConfigure(const Format &format) = 0; + bool GetPixelFmtFromUser(const Format &format); + static std::optional GetFrameRateFromUser(const Format &format); + int32_t SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info); + virtual int32_t UpdateInPortFormat() = 0; + virtual int32_t UpdateOutPortFormat() = 0; + virtual void UpdateColorAspects() {} + void PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def); + int32_t SetFrameRateAdaptiveMode(const Format &format); + int32_t SetProcessName(const Format &format); + virtual int32_t ReConfigureOutputBufferCnt() = 0; + virtual uint64_t OnGetOutputBufferUsage() = 0; + virtual int32_t OnSetOutputBuffer(sptr output) = 0; + + // start + virtual bool ReadyToStart() = 0; + virtual int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex) = 0; + virtual void UpdateFormatFromSurfaceBuffer() = 0; + int32_t GetPortDefinition(OMX_DIRTYPE portIndex, OMX_PARAM_PORTDEFINITIONTYPE& def); + int32_t AllocateSurfaceBuffers(OMX_DIRTYPE portIndex, sptr output = nullptr); + int32_t AllocateHardwareBuffers(OMX_DIRTYPE portIndex); + std::shared_ptr SurfaceBufferToOmxBuffer( + const sptr& surfaceBuffer); + std::shared_ptr DynamicSurfaceBufferToOmxBuffer(); + + virtual int32_t SubmitAllBuffersOwnedByUs() = 0; + virtual int32_t SubmitOutputBuffersToOmxNode() { return IC_ERR_UNSUPPORT; } + BufferInfo* FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId); + std::optional FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId); + + // input buffer circulation + virtual void NotifyUserToFillThisInBuffer(BufferInfo &info); + virtual void OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode); + void OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info); + int32_t NotifyOmxToEmptyThisInBuffer(BufferInfo& info); + virtual void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) = 0; + + // output buffer circulation + int32_t NotifyOmxToFillThisOutBuffer(BufferInfo &info); + void OnOMXFillBufferDone(const HdiCodecNamespace::OmxCodecBuffer& omxBuffer, BufferOperationMode mode); + void OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx); + void NotifyUserOutBufferAvaliable(BufferInfo &info); + void OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode); + + // // stop/release + void ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner, bool erase = false); + bool IsAllBufferOwnedByUs(OMX_DIRTYPE portIndex); + bool IsAllBufferOwnedByUs(); + void EraseOutBuffersOwnedByUs(); + void ClearBufferPool(OMX_DIRTYPE portIndex); + virtual void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) = 0; + void FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info); + + // template + template + static inline void InitOMXParam(T& param) + { + (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); + param.nSize = sizeof(T); + param.nVersion.s.nVersionMajor = 1; + } + + template + static inline void InitOMXParamExt(T& param) + { + (void)memset_s(¶m, sizeof(T), 0x0, sizeof(T)); + param.size = sizeof(T); + param.version.s.nVersionMajor = 1; + } + + template + bool GetParameter(uint32_t index, T& param, bool isCfg = false) + { + int8_t* p = reinterpret_cast(¶m); + std::vector inVec(p, p + sizeof(T)); + std::vector outVec; + int32_t ret = isCfg ? compNode_->GetConfig(index, inVec, outVec) : + compNode_->GetParameter(index, inVec, outVec); + if (ret != HDF_SUCCESS) { + return false; + } + if (outVec.size() != sizeof(T)) { + return false; + } + ret = memcpy_s(¶m, sizeof(T), outVec.data(), outVec.size()); + if (ret != EOK) { + return false; + } + return true; + } + + template + bool SetParameter(uint32_t index, const T& param, bool isCfg = false) + { + const int8_t* p = reinterpret_cast(¶m); + std::vector inVec(p, p + sizeof(T)); + int32_t ret = isCfg ? compNode_->SetConfig(index, inVec) : + compNode_->SetParameter(index, inVec); + if (ret != HDF_SUCCESS) { + return false; + } + return true; + } + +protected: + bool isEncoder_; + OMX_VIDEO_CODINGTYPE codingType_; + uint32_t componentId_ = 0; + std::string componentName_; + std::string compUniqueStr_; + bool debugMode_ = false; + bool dumpMode_ = false; + sptr compCb_ = nullptr; + sptr compNode_ = nullptr; + sptr compMgr_ = nullptr; + + std::shared_ptr callback_; + PixelFmt configuredFmt_; + BufferRequestConfig requestCfg_; + std::shared_ptr configFormat_; + std::shared_ptr inputFormat_; + std::shared_ptr outputFormat_; + + std::vector inputBufferPool_; + std::vector outputBufferPool_; + bool isBufferCirculating_ = false; + bool inputPortEos_ = false; + bool outputPortEos_ = false; + + struct TotalCntAndCost { + uint64_t totalCnt = 0; + uint64_t totalCostUs = 0; + }; + std::array, OWNER_CNT> inputHoldTimeRecord_; + std::chrono::time_point firstInTime_; + uint64_t inTotalCnt_ = 0; + std::array, OWNER_CNT> outputHoldTimeRecord_; + std::chrono::time_point firstOutTime_; + TotalCntAndCost outRecord_; + std::unordered_map> inTimeMap_; + + static constexpr char BUFFER_ID[] = "buffer-id"; + static constexpr uint32_t WAIT_FENCE_MS = 1000; +private: + struct BaseState : State { + protected: + BaseState(ImageCodec *codec, const std::string &stateName, + BufferOperationMode inputMode = KEEP_BUFFER, BufferOperationMode outputMode = KEEP_BUFFER) + : State(stateName), codec_(codec), inputMode_(inputMode), outputMode_(outputMode) {} + void OnMsgReceived(const MsgInfo &info) override; + void ReplyErrorCode(MsgId id, int32_t err); + void OnCodecEvent(const MsgInfo &info); + virtual void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2); + void OnGetFormat(const MsgInfo &info); + virtual void OnShutDown(const MsgInfo &info) = 0; + void OnCheckIfStuck(const MsgInfo &info); + void OnForceShutDown(const MsgInfo &info); + void OnStateExited() override { codec_->stateGeneration_++; } + + protected: + ImageCodec *codec_; + BufferOperationMode inputMode_; + BufferOperationMode outputMode_; + }; + + struct UninitializedState : BaseState { + explicit UninitializedState(ImageCodec *codec) : BaseState(codec, "Uninitialized") {} + private: + void OnStateEntered() override; + void OnMsgReceived(const MsgInfo &info) override; + int32_t OnAllocateComponent(const std::string &name); + void OnShutDown(const MsgInfo &info) override; + }; + + struct InitializedState : BaseState { + explicit InitializedState(ImageCodec *codec) : BaseState(codec, "Initialized") {} + private: + void OnStateEntered() override; + void ProcessShutDownFromRunning(); + void OnMsgReceived(const MsgInfo &info) override; + void OnSetCallBack(const MsgInfo &info); + void OnConfigure(const MsgInfo &info); + void OnGetOutputBufferUsage(const MsgInfo &info); + void OnSetOutputBuffer(const MsgInfo &info); + void OnStart(const MsgInfo &info); + void OnShutDown(const MsgInfo &info) override; + }; + + struct StartingState : BaseState { + explicit StartingState(ImageCodec *codec) : BaseState(codec, "Starting") {} + private: + void OnStateEntered() override; + void OnStateExited() override; + void OnMsgReceived(const MsgInfo &info) override; + int32_t AllocateBuffers(); + void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override; + void OnShutDown(const MsgInfo &info) override; + void ReplyStartMsg(int32_t errCode); + bool hasError_ = false; + }; + + struct RunningState : BaseState { + explicit RunningState(ImageCodec *codec) : BaseState(codec, "Running", RESUBMIT_BUFFER, RESUBMIT_BUFFER) {} + private: + void OnStateEntered() override; + void OnMsgReceived(const MsgInfo &info) override; + void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override; + void OnShutDown(const MsgInfo &info) override; + }; + + struct OutputPortChangedState : BaseState { + explicit OutputPortChangedState(ImageCodec *codec) + : BaseState(codec, "OutputPortChanged", RESUBMIT_BUFFER, FREE_BUFFER) {} + private: + void OnStateEntered() override; + void OnMsgReceived(const MsgInfo &info) override; + void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override; + void OnShutDown(const MsgInfo &info) override; + void HandleOutputPortDisabled(); + void HandleOutputPortEnabled(); + }; + + struct StoppingState : BaseState { + explicit StoppingState(ImageCodec *codec) : BaseState(codec, "Stopping"), + omxNodeInIdleState_(false), + omxNodeIsChangingToLoadedState_(false) {} + private: + void OnStateEntered() override; + void OnMsgReceived(const MsgInfo &info) override; + void OnCodecEvent(HdiCodecNamespace::CodecEventType event, uint32_t data1, uint32_t data2) override; + void OnShutDown(const MsgInfo &info) override; + void ChangeStateIfWeOwnAllBuffers(); + void ChangeOmxNodeToLoadedState(bool forceToFreeBuffer); + bool omxNodeInIdleState_; + bool omxNodeIsChangingToLoadedState_; + }; + + class HdiCallback : public HdiCodecNamespace::ICodecCallback { + public: + explicit HdiCallback(ImageCodec* codec) : codec_(codec) { } + virtual ~HdiCallback() = default; + int32_t EventHandler(HdiCodecNamespace::CodecEventType event, const HdiCodecNamespace::EventInfo& info); + int32_t EmptyBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer); + int32_t FillBufferDone(int64_t appData, const HdiCodecNamespace::OmxCodecBuffer& buffer); + private: + ImageCodec* codec_; + }; +private: + int32_t DoSyncCall(MsgWhat msgType, std::function oper); + int32_t DoSyncCallAndGetReply(MsgWhat msgType, std::function oper, ParamSP &reply); + int32_t InitWithName(const std::string &name); + void ReleaseComponent(); + void CleanUpOmxNode(); + void ChangeOmxToTargetState(HdiCodecNamespace::CodecStateType &state, + HdiCodecNamespace::CodecStateType targetState); + bool RollOmxBackToLoaded(); + + int32_t ForceShutdown(int32_t generation); + void SignalError(ImageCodecError err); + void DeferMessage(const MsgInfo &info); + void ProcessDeferredMessages(); + void ReplyToSyncMsgLater(const MsgInfo& msg); + bool GetFirstSyncMsgToReply(MsgInfo& msg); + +private: + static constexpr size_t MAX_IMAGE_CODEC_BUFFER_SIZE = 8192 * 4096 * 4; // 8K RGBA + static constexpr uint32_t THREE_SECONDS_IN_US = 3'000'000; + static constexpr double FRAME_RATE_COEFFICIENT = 65536.0; + + std::shared_ptr uninitializedState_; + std::shared_ptr initializedState_; + std::shared_ptr startingState_; + std::shared_ptr runningState_; + std::shared_ptr outputPortChangedState_; + std::shared_ptr stoppingState_; + + int32_t stateGeneration_ = 0; + bool isShutDownFromRunning_ = false; + bool notifyCallerAfterShutdownComplete_ = false; + bool hasFatalError_ = false; + std::list deferredQueue_; + std::map>> syncMsgToReply_; +}; // class ImageCodec +} // namespace OHOS::ImagePlugin + +#endif // IMAGE_CODEC_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_buffer.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_buffer.h new file mode 100644 index 000000000..7bf439896 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_buffer.h @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_BUFFER_H +#define IMAGE_CODEC_BUFFER_H + +#include "surface_type.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_type.h +#include "surface_buffer.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_buffer.h + +namespace OHOS::ImagePlugin { +enum YuvSemiPlanarArrangement { + PLANE_Y = 0, + PLANE_U, + PLANE_V, + PLANE_BUTT +}; + +class ImageCodecBuffer { +public: + static std::shared_ptr CreateDmaBuffer(int fd, int32_t capacity, int32_t stride); + static std::shared_ptr CreateSurfaceBuffer(const BufferRequestConfig &config); + static std::shared_ptr CreateSurfaceBuffer(sptr surface); + + virtual ~ImageCodecBuffer() = default; + void GetBufferCirculateInfo(int64_t& pts, uint32_t& flag, uint32_t& size, uint32_t& offset) const + { + pts = pts_; + flag = flag_; + size = size_; + offset = offset_; + } + void SetBufferCirculateInfo(int64_t pts, uint32_t flag, uint32_t size, uint32_t offset) + { + pts_ = pts; + flag_ = flag; + size_ = size; + offset_ = offset; + } + uint32_t GetBufferFlag() const { return flag_; } + int64_t GetPts() const { return pts_; } + int32_t GetCapacity() const { return capacity_; } + int32_t GetStride() const { return stride_; } + virtual bool Init() = 0; + virtual int32_t GetFileDescriptor() = 0; + virtual sptr GetSurfaceBuffer() = 0; + virtual uint8_t* GetAddr() = 0; +protected: + ImageCodecBuffer() = default; +protected: + int64_t pts_ = 0; + uint32_t size_ = 0; + uint32_t offset_ = 0; + uint32_t flag_; // OMX_BUFFERFLAG_X defined in OMX_Core.h + int32_t capacity_ = 0; + int32_t stride_ = 0; +}; + +class ImageDmaBuffer : public ImageCodecBuffer { +public: + ImageDmaBuffer(int fd, int32_t capacity, int32_t stride); + ~ImageDmaBuffer(); + bool Init() override { return true; } + int32_t GetFileDescriptor() override { return fd_; } + sptr GetSurfaceBuffer() override { return nullptr; } + uint8_t* GetAddr() override; +private: + int32_t fd_ = -1; + uint8_t* addr_ = nullptr; +}; + +class ImageSurfaceBuffer : public ImageCodecBuffer { +public: + explicit ImageSurfaceBuffer(const BufferRequestConfig &config); + explicit ImageSurfaceBuffer(sptr surface); + ~ImageSurfaceBuffer(); + bool Init() override; + int32_t GetFileDescriptor() override; + uint8_t* GetAddr() override; + sptr GetSurfaceBuffer() override; +private: + BufferRequestConfig config_; + sptr surfaceBuffer_ = nullptr; + uint8_t* addr_ = nullptr; +}; + +} // namespace OHOS::ImagePlugin +#endif \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_common.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_common.h new file mode 100644 index 000000000..64504e407 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_common.h @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_COMMON_H +#define IMAGE_CODEC_COMMON_H + +#include "surface_buffer.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_buffer.h +#include "surface_type.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_type.h +#include "format.h" + +namespace OHOS::ImagePlugin { +enum ImageCodecError : int32_t { + IC_ERR_OK, + IC_ERR_SERVICE_DIED, + IC_ERR_INVALID_VAL, + IC_ERR_INVALID_OPERATION, + IC_ERR_INVALID_STATE, + IC_ERR_NO_MEMORY, + IC_ERR_UNSUPPORT, + IC_ERR_UNKNOWN +}; + +class ImageCodecCallback { +public: + virtual ~ImageCodecCallback() = default; + virtual void OnError(ImageCodecError err) = 0; + virtual void OnOutputFormatChanged(const Format &format) = 0; + virtual void OnInputBufferAvailable(uint32_t index, std::shared_ptr buffer) = 0; + virtual void OnOutputBufferAvailable(uint32_t index, std::shared_ptr buffer) = 0; +}; + +class ImageCodecDescriptionKey { +public: + static constexpr char WIDTH[] = "width"; + static constexpr char HEIGHT[] = "height"; + static constexpr char VIDEO_DISPLAY_WIDTH[] = "video_display_width"; + static constexpr char VIDEO_DISPLAY_HEIGHT[] = "video_display_height"; + static constexpr char MAX_INPUT_SIZE[] = "max_input_size"; + static constexpr char INPUT_BUFFER_COUNT[] = "input_buffer_count"; + static constexpr char OUTPUT_BUFFER_COUNT[] = "output_buffer_count"; + static constexpr char ENABLE_HEIF_GRID[] = "enable_heif_grid"; + static constexpr char PIXEL_FORMAT[] = "pixel_format"; + static constexpr char RANGE_FLAG[] = "range_flag"; + static constexpr char COLOR_PRIMARIES[] = "color_primaries"; + static constexpr char TRANSFER_CHARACTERISTICS[] = "transfer_characteristics"; + static constexpr char MATRIX_COEFFICIENTS[] = "matrix_coefficients"; + static constexpr char FRAME_RATE[] = "frame_rate"; + static constexpr char VIDEO_FRAME_RATE_ADAPTIVE_MODE[] = "video_frame_rate_adaptive_mode"; + static constexpr char PROCESS_NAME[] = "process_name"; +}; +} // namespace OHOS::ImagePlugin +#endif // IMAGE_CODEC_COMMON_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_list.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_list.h new file mode 100644 index 000000000..63a2bdecd --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_list.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_LIST_H +#define IMAGE_CODEC_LIST_H + +#include "hdi_define.h" + +namespace OHOS::ImagePlugin { +sptr GetManager(); +std::vector GetCapList(); +} // namespace OHOS::ImagePlugin + +#endif // IMAGE_CODEC_LIST_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_log.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_log.h new file mode 100644 index 000000000..2114151d3 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_codec_log.h @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_LOG_H +#define IMAGE_CODEC_LOG_H + +#include +#include +#include "log_tags.h" +#include "hilog/log.h" +#ifdef BUILD_ENG_VERSION +#include "hitrace_meter.h" +#endif + +inline constexpr OHOS::HiviewDFX::HiLogLabel IMAGE_CODEC_LABEL = { + LOG_CORE, LOG_TAG_DOMAIN_ID_PLUGIN, "HEIF_HW_DECODER"}; + +#ifdef __FILE_NAME__ +#define FILENAME __FILE_NAME__ +#else +#define FILENAME __FILE__ +#endif + +#define LOG_FMT "[%{public}s][%{public}s %{public}d] " +#define LOGE(x, ...) \ + OHOS::HiviewDFX::HiLog::Error(IMAGE_CODEC_LABEL, LOG_FMT x, FILENAME, __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define LOGW(x, ...) \ + OHOS::HiviewDFX::HiLog::Warn(IMAGE_CODEC_LABEL, LOG_FMT x, FILENAME, __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define LOGI(x, ...) \ + OHOS::HiviewDFX::HiLog::Info(IMAGE_CODEC_LABEL, LOG_FMT x, FILENAME, __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define LOGD(x, ...) \ + OHOS::HiviewDFX::HiLog::Debug(IMAGE_CODEC_LABEL, LOG_FMT x, FILENAME, __FUNCTION__, __LINE__, ##__VA_ARGS__) + +// for ImageCodecBuffer +#define HLOG_FMT "%{public}s[%{public}s][%{public}s %{public}d] " +#define HLOGE(x, ...) OHOS::HiviewDFX::HiLog::Error(IMAGE_CODEC_LABEL, HLOG_FMT x, compUniqueStr_.c_str(), \ + currState_->GetName().c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define HLOGW(x, ...) OHOS::HiviewDFX::HiLog::Warn(IMAGE_CODEC_LABEL, HLOG_FMT x, compUniqueStr_.c_str(), \ + currState_->GetName().c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define HLOGI(x, ...) OHOS::HiviewDFX::HiLog::Info(IMAGE_CODEC_LABEL, HLOG_FMT x, compUniqueStr_.c_str(), \ + currState_->GetName().c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define HLOGD(x, ...) \ + do { \ + if (debugMode_) { \ + OHOS::HiviewDFX::HiLog::Debug(IMAGE_CODEC_LABEL, HLOG_FMT x, compUniqueStr_.c_str(), \ + currState_->GetName().c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__); \ + } \ + } while (0) + +// for ImageCodecBuffer inner state +#define SLOGE(x, ...) OHOS::HiviewDFX::HiLog::Error(IMAGE_CODEC_LABEL, HLOG_FMT x, \ + codec_->compUniqueStr_.c_str(), stateName_.c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define SLOGW(x, ...) OHOS::HiviewDFX::HiLog::Warn(IMAGE_CODEC_LABEL, HLOG_FMT x, \ + codec_->compUniqueStr_.c_str(), stateName_.c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define SLOGI(x, ...) OHOS::HiviewDFX::HiLog::Info(IMAGE_CODEC_LABEL, HLOG_FMT x, \ + codec_->compUniqueStr_.c_str(), stateName_.c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__) +#define SLOGD(x, ...) \ + do { \ + if (codec_->debugMode_) { \ + OHOS::HiviewDFX::HiLog::Debug(IMAGE_CODEC_LABEL, HLOG_FMT x, \ + codec_->compUniqueStr_.c_str(), stateName_.c_str(), __FUNCTION__, __LINE__, ##__VA_ARGS__); \ + } \ + } while (0) + +#define IF_TRUE_RETURN_VAL(cond, val) \ + do { \ + if (cond) { \ + return val; \ + } \ + } while (0) +#define IF_TRUE_RETURN_VAL_WITH_MSG(cond, val, msg, ...) \ + do { \ + if (cond) { \ + LOGE(msg, ##__VA_ARGS__); \ + return val; \ + } \ + } while (0) +#define IF_TRUE_RETURN_VOID(cond) \ + do { \ + if (cond) { \ + return; \ + } \ + } while (0) +#define IF_TRUE_RETURN_VOID_WITH_MSG(cond, msg, ...) \ + do { \ + if (cond) { \ + LOGE(msg, ##__VA_ARGS__); \ + return; \ + } \ + } while (0) + +#ifdef BUILD_ENG_VERSION +#ifdef H_SYSTRACE_TAG +#undef H_SYSTRACE_TAG +#endif +#define H_SYSTRACE_TAG HITRACE_TAG_ZMEDIA +#endif // BUILD_ENG_VERSION + +class HeifPerfTracker { +public: + explicit HeifPerfTracker(std::string desc) : desc_(desc) + { + startTimeInUs_ = GetCurrentTimeInUs(); +#ifdef BUILD_ENG_VERSION + StartTrace(H_SYSTRACE_TAG, desc); +#endif + } + ~HeifPerfTracker() + { +#ifdef BUILD_ENG_VERSION + FinishTrace(H_SYSTRACE_TAG); +#endif + static constexpr float MILLISEC_TO_MICROSEC = 1000.0f; + int64_t timeSpanInUs = GetCurrentTimeInUs() - startTimeInUs_; + LOGI("%{public}s cost: %{public}.2f ms", + desc_.c_str(), static_cast(timeSpanInUs / MILLISEC_TO_MICROSEC)); + } +private: + int64_t GetCurrentTimeInUs() + { + auto now = std::chrono::steady_clock::now(); + return std::chrono::duration_cast(now.time_since_epoch()).count(); + } + + int64_t startTimeInUs_; + std::string desc_; +}; + +#endif // IMAGE_CODEC_LOG_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_decoder.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_decoder.h new file mode 100644 index 000000000..d164cdb7e --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/image_decoder.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_DECODER_H +#define IMAGE_DECODER_H + + #include "hardware/imagecodec/image_codec.h" + +namespace OHOS::ImagePlugin { +class ImageDecoder : public ImageCodec { +public: + ImageDecoder(); +private: + // configure + int32_t OnConfigure(const Format &format) override; + int32_t SetupPort(const Format &format); + int32_t UpdateInPortFormat() override; + int32_t UpdateOutPortFormat() override; + bool UpdateConfiguredFmt(OMX_COLOR_FORMATTYPE portFmt); + void UpdateColorAspects() override; + void UpdateDisplaySizeByCrop(); + int32_t ReConfigureOutputBufferCnt() override; + uint64_t OnGetOutputBufferUsage() override; + int32_t OnSetOutputBuffer(sptr output) override; + + // start + int32_t AllocateBuffersOnPort(OMX_DIRTYPE portIndex) override; + void UpdateFormatFromSurfaceBuffer() override; + int32_t SubmitAllBuffersOwnedByUs() override; + int32_t SubmitOutputBuffersToOmxNode() override; + bool ReadyToStart() override; + + // input buffer circulation + void OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) override; + + // output buffer circulation + uint64_t GetProducerUsage(); + + // stop/release + void EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) override; +private: + static constexpr uint64_t BUFFER_MODE_REQUEST_USAGE = + BUFFER_USAGE_MEM_DMA | BUFFER_USAGE_VIDEO_DECODER | BUFFER_USAGE_CPU_READ | BUFFER_USAGE_MEM_MMZ_CACHE; + bool enableHeifGrid_ = false; + sptr outputBuffer_; +}; +} // namespace OHOS::ImagePlugin + +#endif // IMAGE_DECODER_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/msg_handle_loop.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/msg_handle_loop.h new file mode 100644 index 000000000..c7b288111 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/msg_handle_loop.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_MSGQUEUETHREAD_H +#define IMAGE_CODEC_MSGQUEUETHREAD_H + +#include +#include +#include +#include +#include +#include +#include "param_bundle.h" + +namespace OHOS::ImagePlugin { +using MsgType = int32_t; +using MsgId = uint64_t; +struct MsgInfo { + MsgType type; + MsgId id; + ParamSP param; +}; + +class MsgHandleLoop { +protected: + MsgHandleLoop(); + ~MsgHandleLoop(); + void SendAsyncMsg(MsgType type, const ParamSP &msg, uint32_t delayUs = 0); + bool SendSyncMsg(MsgType type, const ParamSP &msg, ParamSP &reply, uint32_t waitMs = 0); + virtual void OnMsgReceived(const MsgInfo &info) = 0; + void PostReply(MsgId id, const ParamSP &reply); + void Stop(); + static constexpr MsgId ASYNC_MSG_ID = 0; + +private: + void MainLoop(); + MsgId GenerateMsgId(); + using TimeUs = int64_t; + static TimeUs GetNowUs(); + +private: + std::thread m_thread; + std::mutex m_mtx; + bool m_threadNeedStop = false; + MsgId m_lastMsgId = 0; + std::map m_msgQueue; // msg will be sorted by timeUs + std::condition_variable m_threadCond; + + std::mutex m_replyMtx; + std::map m_replies; + std::condition_variable m_replyCond; +}; +} // namespace OHOS::ImagePlugin +#endif // IMAGE_CODEC_MSGQUEUETHREAD_H diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/param_bundle.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/param_bundle.h new file mode 100644 index 000000000..9a30566a4 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/param_bundle.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_PARAM_BUNDLE_H +#define IMAGE_CODEC_PARAM_BUNDLE_H + +#include +#include +#include +#include +#include + +namespace OHOS::ImagePlugin { +class ParamBundle; +using ParamSP = std::shared_ptr; + +class ParamBundle { +public: + static ParamSP Create(); + + template + void SetValue(const std::string &key, const T &value) + { + std::lock_guard lock(m_mtx); + m_items[key] = value; + } + + template + bool GetValue(const std::string &key, T &value) const + { + std::lock_guard lock(m_mtx); + const auto it = m_items.find(key); + if (it == m_items.end()) { + return false; + } + value = std::any_cast(it->second); + return true; + } + + ParamBundle(const ParamBundle &) = delete; + ParamBundle &operator=(const ParamBundle &) = delete; + ParamBundle(ParamBundle &&) = delete; + ParamBundle &operator=(ParamBundle &&) = delete; + +private: + ParamBundle() = default; + ~ParamBundle() = default; + + mutable std::mutex m_mtx; + std::unordered_map m_items; +}; +} // namespace OHOS::ImagePlugin +#endif // IMAGE_CODEC_PARAM_BUNDLE_H diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/state_machine.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/state_machine.h new file mode 100644 index 000000000..64fbe1105 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/state_machine.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_STATE_H +#define IMAGE_CODEC_STATE_H + +#include +#include +#include +#include "msg_handle_loop.h" + +namespace OHOS::ImagePlugin { +class State { +public: + explicit State(std::string stateName) : stateName_(std::move(stateName)) {} + const std::string GetName() const { return stateName_; } + +protected: + virtual ~State() = default; + virtual void OnStateEntered() {}; + virtual void OnStateExited() {}; + virtual void OnMsgReceived(const MsgInfo &info) = 0; + + friend class StateMachine; + + std::string stateName_; +}; + +class StateMachine : public MsgHandleLoop { +public: + StateMachine() = default; + +protected: + virtual ~StateMachine() = default; + void ChangeStateTo(const std::shared_ptr &targetState); + void OnMsgReceived(const MsgInfo &info) override; + + std::shared_ptr currState_; +}; +} // namespace OHOS::ImagePlugin +#endif // IMAGE_CODEC_STATE_H diff --git a/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/type_converter.h b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/type_converter.h new file mode 100644 index 000000000..97cd6cd03 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/include/hardware/imagecodec/type_converter.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef IMAGE_CODEC_TYPE_CONVERTOR_H +#define IMAGE_CODEC_TYPE_CONVERTOR_H + +#include +#include "surface_type.h" // foundation/graphic/graphic_2d/interfaces/inner_api/surface/ + +namespace OHOS::ImagePlugin { +struct PixelFmt { + GraphicPixelFormat graphicFmt; // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_type.h + std::string strFmt; +}; + +class TypeConverter { +public: + // pixel format + static std::optional GraphicFmtToFmt(GraphicPixelFormat format); +}; +} // namespace OHOS::ImagePlugin + +#endif // IMAGE_CODEC_TYPE_CONVERTOR_H \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/heif_hw_decoder.cpp b/plugins/common/libs/image/libextplugin/src/hardware/heif_hw_decoder.cpp new file mode 100644 index 000000000..c9c9a0986 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/heif_hw_decoder.cpp @@ -0,0 +1,566 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/heif_hw_decoder.h" +#include "hardware/imagecodec/image_codec_list.h" +#include "hardware/imagecodec/image_codec_log.h" +#include "hardware/imagecodec/type_converter.h" +#include "media_errors.h" // foundation/multimedia/image_framework/interfaces/innerkits/include/ +#include "syspara/parameters.h" // base/startup/init/interfaces/innerkits/include/ +#include +#include + +namespace OHOS::ImagePlugin { +using namespace std; +using namespace HdiCodecNamespace; + +bool GridInfo::IsValid() const +{ + IF_TRUE_RETURN_VAL_WITH_MSG((displayWidth == 0 || displayHeight == 0), false, + "invalid displaySize: [%{public}ux%{public}u]", displayWidth, displayHeight); + IF_TRUE_RETURN_VAL(!enableGrid, true); + IF_TRUE_RETURN_VAL_WITH_MSG((cols == 0 || rows == 0), false, + "invalid gridSize: [%{public}ux%{public}u]", cols, rows); + IF_TRUE_RETURN_VAL_WITH_MSG((tileWidth == 0 || tileHeight == 0), false, + "invalid tileSize: [%{public}ux%{public}u]", tileWidth, tileHeight); + uint32_t expCols = static_cast(ceil(static_cast(displayWidth) / static_cast(tileWidth))); + IF_TRUE_RETURN_VAL_WITH_MSG(expCols != cols, false, + "invalid cols, expect %{public}u, get %{public}u", expCols, cols); + uint32_t expRows = static_cast(ceil(static_cast(displayHeight) / static_cast(tileHeight))); + IF_TRUE_RETURN_VAL_WITH_MSG(expRows != rows, false, + "invalid rows, expect %{public}u, get %{public}u", expRows, rows); + return true; +} + +HeifHardwareDecoder::HeifDecoderCallback::HeifDecoderCallback(HeifHardwareDecoder* heifDecoder) + : heifDecoder_(heifDecoder) +{} + +void HeifHardwareDecoder::HeifDecoderCallback::OnError(ImageCodecError err) +{ + heifDecoder_->SignalError(); +} + +void HeifHardwareDecoder::HeifDecoderCallback::OnOutputFormatChanged(const Format &format) +{} + +void HeifHardwareDecoder::HeifDecoderCallback::OnInputBufferAvailable(uint32_t index, + std::shared_ptr buffer) +{ + lock_guard lk(heifDecoder_->inputMtx_); + heifDecoder_->inputList_.emplace_back(index, buffer); + heifDecoder_->inputCond_.notify_all(); +} + +void HeifHardwareDecoder::HeifDecoderCallback::OnOutputBufferAvailable(uint32_t index, + std::shared_ptr buffer) +{ + lock_guard lk(heifDecoder_->outputMtx_); + heifDecoder_->outputList_.emplace_back(index, buffer); + heifDecoder_->outputCond_.notify_all(); +} + +HeifHardwareDecoder::HeifHardwareDecoder() : uvOffsetForOutput_(0) +{ + heifDecoderImpl_ = ImageCodec::Create(); +} + +HeifHardwareDecoder::~HeifHardwareDecoder() +{ + if (releaseThread_.joinable()) { + releaseThread_.join(); + } + Reset(); +} + +sptr HeifHardwareDecoder::AllocateOutputBuffer(uint32_t width, uint32_t height, int32_t pixelFmt) +{ + HeifPerfTracker tracker(__FUNCTION__); + LOGI("BufferInfo: width=%{public}u, height=%{public}u, pixelFmt=%{public}d", width, height, pixelFmt); + IF_TRUE_RETURN_VAL_WITH_MSG(heifDecoderImpl_ == nullptr, nullptr, "failed to create heif decoder"); + IF_TRUE_RETURN_VAL_WITH_MSG((width == 0 || height == 0), nullptr, + "invalid size=[%{public}ux%{public}u]", width, height); + optional fmt = TypeConverter::GraphicFmtToFmt(static_cast(pixelFmt)); + IF_TRUE_RETURN_VAL(!fmt.has_value(), nullptr); + + uint64_t usage; + int32_t err = heifDecoderImpl_->GetOutputBufferUsage(usage); + IF_TRUE_RETURN_VAL_WITH_MSG(err != IC_ERR_OK, nullptr, "failed to get output buffer usage, err=%{public}d", err); + sptr output = SurfaceBuffer::Create(); + IF_TRUE_RETURN_VAL_WITH_MSG(output == nullptr, nullptr, "failed to create output"); + BufferRequestConfig config = { + .width = width, + .height = height, + .strideAlignment = STRIDE_ALIGNMENT, + .format = pixelFmt, + .usage = usage, + .timeout = 0 + }; + GSError ret = output->Alloc(config); + IF_TRUE_RETURN_VAL_WITH_MSG(ret != GSERROR_OK, nullptr, "failed to alloc output, ret=%{public}d", ret); + return output; +} + +static bool IsValueInRange(uint32_t value, HdiCodecNamespace::RangeValue range) +{ + return (value >= static_cast(range.min)) && (value <= static_cast(range.max)); +} + +bool HeifHardwareDecoder::IsHardwareDecodeSupported(const GridInfo& gridInfo) +{ + string decoderName = heifDecoderImpl_->GetComponentName(); + vector capList = GetCapList(); + auto it = find_if(capList.begin(), capList.end(), [decoderName](const CodecCompCapability& cap) { + return (cap.compName == decoderName); + }); + if (it == capList.end()) { + LOGE("can not find heif hw decoder"); + return false; + } + uint32_t widthToCheck = gridInfo.enableGrid ? gridInfo.tileWidth : gridInfo.displayWidth; + uint32_t heightToCheck = gridInfo.enableGrid ? gridInfo.tileHeight : gridInfo.displayHeight; + HdiCodecNamespace::RangeValue widthRange = { + .min = it->port.video.minSize.width, + .max = it->port.video.maxSize.width + }; + HdiCodecNamespace::RangeValue heightRange = { + .min = it->port.video.minSize.height, + .max = it->port.video.maxSize.height + }; + bool isValidSize = false; + if (it->canSwapWidthHeight) { + LOGI("decoder support swap width and height"); + isValidSize = (IsValueInRange(widthToCheck, widthRange) && IsValueInRange(heightToCheck, heightRange)) || + (IsValueInRange(heightToCheck, widthRange) && IsValueInRange(widthToCheck, heightRange)); + } else { + isValidSize = IsValueInRange(widthToCheck, widthRange) && IsValueInRange(heightToCheck, heightRange); + } + if (!isValidSize) { + LOGE("unsupported size: [%{public}ux%{public}u]", widthToCheck, heightToCheck); + return false; + } + return true; +} + +bool HeifHardwareDecoder::SetCallbackForDecoder() +{ + HeifPerfTracker tracker(__FUNCTION__); + shared_ptr cb = make_shared(this); + int32_t ret = heifDecoderImpl_->SetCallback(cb); + if (ret != IC_ERR_OK) { + LOGE("failed to set callback for decoder, err=%{public}d", ret); + return false; + } + return true; +} + +bool HeifHardwareDecoder::ConfigureDecoder(const GridInfo& gridInfo, sptr& output) +{ + HeifPerfTracker tracker(__FUNCTION__); + Format format; + if (gridInfo.enableGrid) { + format.SetValue(ImageCodecDescriptionKey::WIDTH, gridInfo.tileWidth); + format.SetValue(ImageCodecDescriptionKey::HEIGHT, gridInfo.tileHeight); + } else { + format.SetValue(ImageCodecDescriptionKey::WIDTH, gridInfo.displayWidth); + format.SetValue(ImageCodecDescriptionKey::HEIGHT, gridInfo.displayHeight); + } + static constexpr double OUTPUT_FRAME_RATE = 120.0; + format.SetValue(ImageCodecDescriptionKey::FRAME_RATE, OUTPUT_FRAME_RATE); + format.SetValue(ImageCodecDescriptionKey::VIDEO_FRAME_RATE_ADAPTIVE_MODE, true); + int32_t pixelFmt = output->GetFormat(); + pixelFmt = (pixelFmt == GRAPHIC_PIXEL_FMT_YCBCR_P010) ? GRAPHIC_PIXEL_FMT_YCBCR_420_SP : pixelFmt; + pixelFmt = (pixelFmt == GRAPHIC_PIXEL_FMT_YCRCB_P010) ? GRAPHIC_PIXEL_FMT_YCRCB_420_SP : pixelFmt; + format.SetValue(ImageCodecDescriptionKey::PIXEL_FORMAT, pixelFmt); + format.SetValue(ImageCodecDescriptionKey::ENABLE_HEIF_GRID, gridInfo.enableGrid); + if (!gridInfo.enableGrid) { + static constexpr uint32_t INPUT_BUFFER_CNT_WHEN_NO_GRID = 3; + format.SetValue(ImageCodecDescriptionKey::INPUT_BUFFER_COUNT, INPUT_BUFFER_CNT_WHEN_NO_GRID); + } + static constexpr uint32_t OUTPUT_BUFFER_CNT = 1; + format.SetValue(ImageCodecDescriptionKey::OUTPUT_BUFFER_COUNT, OUTPUT_BUFFER_CNT); + int32_t ret = heifDecoderImpl_->Configure(format); + if (ret != IC_ERR_OK) { + LOGE("failed to configure decoder, err=%{public}d", ret); + return false; + } + return true; +} + +bool HeifHardwareDecoder::SetOutputBuffer(const GridInfo& gridInfo, sptr output) +{ + HeifPerfTracker tracker(__FUNCTION__); + if (gridInfo.enableGrid) { + return true; + } + int32_t ret = heifDecoderImpl_->SetOutputBuffer(output); + if (ret != IC_ERR_OK) { + LOGE("failed to set output buffer, err=%{public}d", ret); + return false; + } + return true; +} + +bool HeifHardwareDecoder::GetUvPlaneOffsetFromSurfaceBuffer(sptr& surfaceBuffer, uint64_t& offset) +{ + IF_TRUE_RETURN_VAL_WITH_MSG(surfaceBuffer == nullptr, false, "invalid surface buffer"); + OH_NativeBuffer_Planes* outputPlanes = nullptr; + GSError ret = surfaceBuffer->GetPlanesInfo(reinterpret_cast(&outputPlanes)); + IF_TRUE_RETURN_VAL_WITH_MSG((ret != GSERROR_OK || outputPlanes == nullptr), false, + "GetPlanesInfo failed, GSError=%{public}d", ret); + IF_TRUE_RETURN_VAL_WITH_MSG(outputPlanes->planeCount < PLANE_BUTT, false, + "invalid yuv buffer, %{public}u", outputPlanes->planeCount); + int32_t pixelFmt = surfaceBuffer->GetFormat(); + if (pixelFmt == GRAPHIC_PIXEL_FMT_YCBCR_420_SP || pixelFmt == GRAPHIC_PIXEL_FMT_YCBCR_P010) { + offset = outputPlanes->planes[PLANE_U].offset; + } else { + offset = outputPlanes->planes[PLANE_V].offset; + } + return true; +} + +uint32_t HeifHardwareDecoder::DoDecode(const GridInfo& gridInfo, std::vector>& inputs, + sptr& output) +{ + LOGI("GridInfo: displayWidth=%{public}u, displayHeight=%{public}u, enableGrid=%{public}d, " \ + "cols=%{public}u, rows=%{public}u, tileWidth=%{public}u, tileHeight=%{public}u", + gridInfo.displayWidth, gridInfo.displayHeight, gridInfo.enableGrid, gridInfo.cols, gridInfo.rows, + gridInfo.tileWidth, gridInfo.tileHeight); + HeifPerfTracker tracker(__FUNCTION__); + IF_TRUE_RETURN_VAL(!gridInfo.IsValid(), Media::ERR_IMAGE_INVALID_PARAMETER); + IF_TRUE_RETURN_VAL_WITH_MSG(output == nullptr, Media::ERR_IMAGE_INVALID_PARAMETER, "null output"); + IF_TRUE_RETURN_VAL_WITH_MSG(inputs.size() < MIN_SIZE_OF_INPUT, Media::ERR_IMAGE_INVALID_PARAMETER, + "input size < %{public}zu", MIN_SIZE_OF_INPUT); + IF_TRUE_RETURN_VAL_WITH_MSG(heifDecoderImpl_ == nullptr, Media::ERR_IMAGE_DECODE_FAILED, + "failed to create heif decoder"); + IF_TRUE_RETURN_VAL(!IsHardwareDecodeSupported(gridInfo), Media::ERR_IMAGE_HW_DECODE_UNSUPPORT); + IF_TRUE_RETURN_VAL(!SetCallbackForDecoder(), Media::ERR_IMAGE_DECODE_FAILED); + IF_TRUE_RETURN_VAL(!ConfigureDecoder(gridInfo, output), Media::ERR_IMAGE_DECODE_FAILED); + IF_TRUE_RETURN_VAL(!SetOutputBuffer(gridInfo, output), Media::ERR_IMAGE_DECODE_FAILED); + Reset(); + output_ = output; + IF_TRUE_RETURN_VAL(!GetUvPlaneOffsetFromSurfaceBuffer(output_, uvOffsetForOutput_), + Media::ERR_IMAGE_DECODE_FAILED); + gridInfo_ = gridInfo; + thread inputThread(&HeifHardwareDecoder::SendInputBufferLoop, this, inputs); + thread outputThread(&HeifHardwareDecoder::ReceiveOutputBufferLoop, this); + int32_t ret = heifDecoderImpl_->Start(); + if (ret != IC_ERR_OK) { + LOGE("failed to start decoder, err=%{public}d", ret); + SignalError(); + } + if (inputThread.joinable()) { + inputThread.join(); + } + if (outputThread.joinable()) { + outputThread.join(); + } + releaseThread_ = thread(&HeifHardwareDecoder::ReleaseDecoder, this); + IF_TRUE_RETURN_VAL_WITH_MSG(hasErr_, Media::ERR_IMAGE_DECODE_FAILED, "err occured during decode"); + FlushOutput(); + if (OHOS::system::GetBoolParameter("image.codec.dump", false)) { + DumpOutput(); + } + return Media::SUCCESS; +} + +void HeifHardwareDecoder::ReleaseDecoder() +{ + HeifPerfTracker tracker(__FUNCTION__); + int32_t ret = heifDecoderImpl_->Release(); + if (ret != IC_ERR_OK) { + LOGE("failed to release decoder, err=%{public}d", ret); + } +} + +void HeifHardwareDecoder::Reset() +{ + hasErr_ = false; + output_ = nullptr; + inputList_.clear(); + outputList_.clear(); +} + +void HeifHardwareDecoder::FlushOutput() +{ + if (output_->GetUsage() & BUFFER_USAGE_MEM_MMZ_CACHE) { + GSError err = output_->Map(); + if (err != GSERROR_OK) { + LOGW("Map failed, GSError=%{public}d", err); + return; + } + err = output_->FlushCache(); + if (err != GSERROR_OK) { + LOGW("FlushCache failed, GSError=%{public}d", err); + } + } +} + +string HeifHardwareDecoder::GetOutputPixelFmtDesc() +{ + optional fmt = TypeConverter::GraphicFmtToFmt(static_cast(output_->GetFormat())); + if (fmt.has_value()) { + return fmt->strFmt; + } + return "unknown"; +} + +void HeifHardwareDecoder::DumpOutput() +{ + static constexpr char DUMP_PATH[] = "/data/misc/imagecodecdump"; + string pixelFmtDesc = GetOutputPixelFmtDesc(); + static constexpr int MAX_PATH_LEN = 256; + char outputFilePath[MAX_PATH_LEN] = {0}; + int ret = 0; + if (gridInfo_.enableGrid) { + ret = sprintf_s(outputFilePath, sizeof(outputFilePath), "%s/out_%s_%ux%u_grid_%ux%u_%ux%u.bin", + DUMP_PATH, pixelFmtDesc.c_str(), gridInfo_.displayWidth, gridInfo_.displayHeight, + gridInfo_.tileWidth, gridInfo_.tileHeight, gridInfo_.cols, gridInfo_.rows); + } else { + ret = sprintf_s(outputFilePath, sizeof(outputFilePath), "%s/out_%s_%ux%u_nogrid.bin", + DUMP_PATH, pixelFmtDesc.c_str(), gridInfo_.displayWidth, gridInfo_.displayHeight); + } + if (ret == -1) { + LOGE("failed to create dump file"); + return; + } + LOGI("dump result to: %{public}s", outputFilePath); + + std::ofstream dumpOutFile; + dumpOutFile.open(std::string(outputFilePath), std::ios_base::binary | std::ios_base::trunc); + if (!dumpOutFile.is_open()) { + LOGE("failed to dump decode result"); + return; + } + + GSError err = output_->InvalidateCache(); + if (err != GSERROR_OK) { + LOGW("InvalidateCache failed, GSError=%{public}d", err); + } + dumpOutFile.write(reinterpret_cast(output_->GetVirAddr()), output_->GetSize()); + dumpOutFile.close(); +} + +int64_t HeifHardwareDecoder::GetTimestampInUs() +{ + auto now = chrono::steady_clock::now(); + return chrono::duration_cast(now.time_since_epoch()).count(); +} + +int32_t HeifHardwareDecoder::PrepareInputCodecBuffer(const vector>& inputs, size_t inputIndex, + shared_ptr& buffer) +{ + HeifPerfTracker tracker(__FUNCTION__); + int64_t pts = GetTimestampInUs(); + if (inputIndex >= inputs.size()) { + buffer->SetBufferCirculateInfo(pts, OMX_BUFFERFLAG_EOS, 0, 0); + return 0; + } + const vector& one = inputs[inputIndex]; + if (one.empty()) { + LOGW("inputs[%{public}zu] is empty", inputIndex); + return -1; + } + errno_t ret = memcpy_s(buffer->GetAddr(), static_cast(buffer->GetCapacity()), one.data(), one.size()); + if (ret != EOK) { + LOGE("failed to get input"); + return -1; + } + uint32_t flag = inputIndex == 0 ? OMX_BUFFERFLAG_CODECCONFIG : 0; + int32_t size = static_cast(one.size()); + buffer->SetBufferCirculateInfo(pts, flag, static_cast(size), 0); + return size; +} + +bool HeifHardwareDecoder::WaitForOmxToReturnInputBuffer(uint32_t& bufferId, shared_ptr& buffer) +{ + unique_lock lk(inputMtx_); + bool ret = inputCond_.wait_for(lk, chrono::milliseconds(BUFFER_CIRCULATE_TIMEOUT_IN_MS), [this] { + return !inputList_.empty(); + }); + if (!ret) { + return false; + } + std::tie(bufferId, buffer) = inputList_.front(); + inputList_.pop_front(); + return true; +} + +void HeifHardwareDecoder::SendInputBufferLoop(const vector>& inputs) +{ + LOGI("in"); + size_t inputIndex = 0; + bool eos = false; + while (!eos && !HasError()) { + uint32_t bufferId; + shared_ptr buffer; + if (!WaitForOmxToReturnInputBuffer(bufferId, buffer)) { + LOGE("input time out"); + continue; + } + if (buffer == nullptr) { + LOGE("got null input buffer"); + break; + } + int32_t size = PrepareInputCodecBuffer(inputs, inputIndex, buffer); + if (size >= 0) { + int32_t ret = heifDecoderImpl_->QueueInputBuffer(bufferId); + if (ret != IC_ERR_OK) { + LOGE("failed to queue input buffer"); + } + } + ++inputIndex; + eos = (size == 0); + } + LOGI("out"); +} + +bool HeifHardwareDecoder::WaitForOmxToReturnOutputBuffer(uint32_t& bufferId, shared_ptr& buffer) +{ + unique_lock lk(outputMtx_); + bool ret = outputCond_.wait_for(lk, chrono::milliseconds(BUFFER_CIRCULATE_TIMEOUT_IN_MS), [this] { + return !outputList_.empty(); + }); + if (!ret) { + return false; + } + std::tie(bufferId, buffer) = outputList_.front(); + outputList_.pop_front(); + return true; +} + +bool HeifHardwareDecoder::CopyRawYuvData(const RawYuvCopyInfo& src, const RawYuvCopyInfo& dst, + uint32_t dirtyWidth, uint32_t dirtyHeight) +{ + IF_TRUE_RETURN_VAL_WITH_MSG((dst.yStart == nullptr || dst.uvStart == nullptr), + false, "can not get addr from dst buffer"); + IF_TRUE_RETURN_VAL_WITH_MSG((src.yStart == nullptr || src.uvStart == nullptr), + false, "can not get addr from src buffer"); + errno_t ret = EOK; + // copy Y plane + for (uint32_t row = 0; (row < dirtyHeight) && (ret == EOK); ++row) { + ret = memcpy_s(dst.yStart + dst.yOffset + row * dst.stride, static_cast(dirtyWidth), + src.yStart + src.yOffset + row * src.stride, static_cast(dirtyWidth)); + } + // copy UV plane + uint32_t dirtyHeightForUvPlane = (dirtyHeight + SAMPLE_RATIO_FOR_YUV420_SP - 1) / SAMPLE_RATIO_FOR_YUV420_SP; + for (uint32_t row = 0; (row < dirtyHeightForUvPlane) && (ret == EOK); ++row) { + ret = memcpy_s(dst.uvStart + dst.uvOffset + row * dst.stride, static_cast(dirtyWidth), + src.uvStart + src.uvOffset + row * src.stride, static_cast(dirtyWidth)); + } + if (ret != EOK) { + LOGE("failed to copy grid data, err=%{public}d", ret); + return false; + } + return true; +} + +uint32_t HeifHardwareDecoder::CalculateDirtyLen(uint32_t displayLen, uint32_t gridLen, + uint32_t totalGrid, uint32_t curGrid) +{ + uint32_t dirtyLen = 0; + if (gridLen >= displayLen) { + dirtyLen = displayLen; + } else { + dirtyLen = gridLen; + if (curGrid + 1 == totalGrid) { + dirtyLen = displayLen - curGrid * gridLen; + } + } + return dirtyLen; +} + +void HeifHardwareDecoder::AssembleOutput(uint32_t outputIndex, shared_ptr& buffer) +{ + HeifPerfTracker tracker(__FUNCTION__); + + uint64_t srcUvOffset = 0; + sptr srcSurfaceBuffer = buffer->GetSurfaceBuffer(); + if (!GetUvPlaneOffsetFromSurfaceBuffer(srcSurfaceBuffer, srcUvOffset)) { + SignalError(); + return; + } + + RawYuvCopyInfo dst; + dst.yStart = static_cast(output_->GetVirAddr()); + dst.stride = static_cast(output_->GetStride()); + dst.uvStart = dst.yStart + uvOffsetForOutput_; + dst.yStride = static_cast(uvOffsetForOutput_ / static_cast(dst.stride)); + RawYuvCopyInfo src; + src.yStart = buffer->GetAddr(); + src.stride = static_cast(buffer->GetStride()); + src.uvStart = src.yStart + srcUvOffset; + src.yStride = static_cast(srcUvOffset / static_cast(src.stride)); + src.yOffset = 0; + src.uvOffset = 0; + + uint32_t decodedRows = outputIndex / gridInfo_.cols; + uint32_t decodedCols = outputIndex % gridInfo_.cols; + uint32_t dirtyWidth = CalculateDirtyLen(dst.stride, src.stride, gridInfo_.cols, decodedCols); + uint32_t dirtyHeight = CalculateDirtyLen(dst.yStride, src.yStride, gridInfo_.rows, decodedRows); + dst.yOffset = decodedRows * dst.stride * gridInfo_.tileHeight + decodedCols * src.stride; + dst.uvOffset = decodedRows * dst.stride * gridInfo_.tileHeight / SAMPLE_RATIO_FOR_YUV420_SP + + decodedCols * src.stride; + + if (!CopyRawYuvData(src, dst, dirtyWidth, dirtyHeight)) { + LOGE("failed to assemble output(grid=%{public}d))", outputIndex); + SignalError(); + } +} + +void HeifHardwareDecoder::ReceiveOutputBufferLoop() +{ + LOGI("in"); + uint32_t outputIndex = 0; + while (!HasError()) { + uint32_t bufferId; + shared_ptr buffer; + if (!WaitForOmxToReturnOutputBuffer(bufferId, buffer)) { + LOGE("output time out"); + continue; + } + if (buffer == nullptr) { + LOGE("null output buffer"); + break; + } + uint32_t flag = buffer->GetBufferFlag(); + if (flag & OMX_BUFFERFLAG_EOS) { + LOGI("output eos, quit loop"); + break; + } + if (gridInfo_.enableGrid) { + AssembleOutput(outputIndex, buffer); + } + ++outputIndex; + int32_t ret = heifDecoderImpl_->ReleaseOutputBuffer(bufferId); + if (ret != IC_ERR_OK) { + LOGE("failed to release output buffer"); + } + } + LOGI("out"); +} + +void HeifHardwareDecoder::SignalError() +{ + std::lock_guard lk(errMtx_); + hasErr_ = true; +} + +bool HeifHardwareDecoder::HasError() +{ + std::lock_guard lk(errMtx_); + return hasErr_; +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/codec_state.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/codec_state.cpp new file mode 100644 index 000000000..b3288cdd1 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/codec_state.cpp @@ -0,0 +1,710 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/image_codec.h" +#include "hardware/imagecodec/image_codec_list.h" +#include "hardware/imagecodec/image_codec_log.h" + +namespace OHOS::ImagePlugin { +using namespace std; +using namespace HdiCodecNamespace; + +/**************************** BaseState Start ****************************/ +void ImageCodec::BaseState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::CODEC_EVENT: { + OnCodecEvent(info); + return; + } + case MsgWhat::OMX_EMPTY_BUFFER_DONE: { + uint32_t bufferId; + (void)info.param->GetValue(BUFFER_ID, bufferId); + codec_->OnOMXEmptyBufferDone(bufferId, inputMode_); + return; + } + case MsgWhat::OMX_FILL_BUFFER_DONE: { + OmxCodecBuffer omxBuffer; + (void)info.param->GetValue("omxBuffer", omxBuffer); + codec_->OnOMXFillBufferDone(omxBuffer, outputMode_); + return; + } + case MsgWhat::GET_INPUT_FORMAT: + case MsgWhat::GET_OUTPUT_FORMAT: { + OnGetFormat(info); + return; + } + case MsgWhat::RELEASE: { + OnShutDown(info); + return; + } + default: { + const char* msgWhat = ImageCodec::ToString(static_cast(info.type)); + if (info.id == ASYNC_MSG_ID) { + SLOGI("ignore msg %{public}s in current state", msgWhat); + } else { // Make sure that all sync message are replied + SLOGE("%{public}s cannot be called at this state", msgWhat); + ReplyErrorCode(info.id, IC_ERR_INVALID_STATE); + } + return; + } + } +} + +void ImageCodec::BaseState::ReplyErrorCode(MsgId id, int32_t err) +{ + if (id == ASYNC_MSG_ID) { + return; + } + ParamSP reply = ParamBundle::Create(); + reply->SetValue("err", err); + codec_->PostReply(id, reply); +} + +void ImageCodec::BaseState::OnCodecEvent(const MsgInfo &info) +{ + CodecEventType event; + uint32_t data1; + uint32_t data2; + (void)info.param->GetValue("event", event); + (void)info.param->GetValue("data1", data1); + (void)info.param->GetValue("data2", data2); + if (event == CODEC_EVENT_CMD_COMPLETE && + data1 == static_cast(CODEC_COMMAND_FLUSH) && + data2 == static_cast(OMX_ALL)) { + SLOGD("ignore flush all complete event"); + } else { + OnCodecEvent(event, data1, data2); + } +} + +void ImageCodec::BaseState::OnCodecEvent(CodecEventType event, uint32_t data1, uint32_t data2) +{ + if (event == CODEC_EVENT_ERROR) { + SLOGE("omx report error event, data1 = %{public}u, data2 = %{public}u", data1, data2); + codec_->SignalError(IC_ERR_SERVICE_DIED); + } else { + SLOGW("ignore event %{public}d, data1 = %{public}u, data2 = %{public}u", event, data1, data2); + } +} + +void ImageCodec::BaseState::OnGetFormat(const MsgInfo &info) +{ + shared_ptr fmt = (info.type == MsgWhat::GET_INPUT_FORMAT) ? + codec_->inputFormat_ : codec_->outputFormat_; + ParamSP reply = ParamBundle::Create(); + if (fmt) { + reply->SetValue("err", IC_ERR_OK); + reply->SetValue("format", *fmt); + codec_->PostReply(info.id, reply); + } else { + ReplyErrorCode(info.id, IC_ERR_UNKNOWN); + } +} + +void ImageCodec::BaseState::OnCheckIfStuck(const MsgInfo &info) +{ + int32_t generation; + (void)info.param->GetValue("generation", generation); + if (generation == codec_->stateGeneration_) { + SLOGE("stucked"); + codec_->PrintAllBufferInfo(); + codec_->SignalError(IC_ERR_UNKNOWN); + } +} + +void ImageCodec::BaseState::OnForceShutDown(const MsgInfo &info) +{ + int32_t generation; + (void)info.param->GetValue("generation", generation); + codec_->ForceShutdown(generation); +} +/**************************** BaseState End ******************************/ + +/**************************** UninitializedState start ****************************/ +void ImageCodec::UninitializedState::OnStateEntered() +{ + codec_->ReleaseComponent(); +} + +void ImageCodec::UninitializedState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::INIT: { + string name; + (void)info.param->GetValue("name", name); + int32_t err = OnAllocateComponent(name); + ReplyErrorCode(info.id, err); + if (err == IC_ERR_OK) { + codec_->ChangeStateTo(codec_->initializedState_); + } + break; + } + default: { + BaseState::OnMsgReceived(info); + } + } +} + +int32_t ImageCodec::UninitializedState::OnAllocateComponent(const std::string &name) +{ + codec_->compMgr_ = GetManager(); + if (codec_->compMgr_ == nullptr) { + SLOGE("GetCodecComponentManager failed"); + return IC_ERR_UNKNOWN; + } + codec_->compCb_ = new HdiCallback(codec_); + int32_t ret = codec_->compMgr_->CreateComponent(codec_->compNode_, codec_->componentId_, name, + 0, codec_->compCb_); + if (ret != HDF_SUCCESS || codec_->compNode_ == nullptr) { + codec_->compCb_ = nullptr; + codec_->compMgr_ = nullptr; + SLOGE("CreateComponent failed, ret=%{public}d", ret); + return IC_ERR_UNKNOWN; + } + codec_->componentName_ = name; + codec_->compUniqueStr_ = "[" + to_string(codec_->componentId_) + "][" + name + "]"; + SLOGI("create omx node succ"); + return IC_ERR_OK; +} + +void ImageCodec::UninitializedState::OnShutDown(const MsgInfo &info) +{ + ReplyErrorCode(info.id, IC_ERR_OK); +} + +/**************************** UninitializedState End ******************************/ + +/**************************** InitializedState Start **********************************/ +void ImageCodec::InitializedState::OnStateEntered() +{ + codec_->inputPortEos_ = false; + codec_->outputPortEos_ = false; + codec_->outputFormat_.reset(); + + ProcessShutDownFromRunning(); + codec_->notifyCallerAfterShutdownComplete_ = false; + codec_->ProcessDeferredMessages(); +} + +void ImageCodec::InitializedState::ProcessShutDownFromRunning() +{ + if (!codec_->isShutDownFromRunning_) { + return; + } + SLOGI("we are doing shutdown from running/portchange/flush -> stopping -> initialized"); + codec_->ChangeStateTo(codec_->uninitializedState_); + if (codec_->notifyCallerAfterShutdownComplete_) { + SLOGI("reply to release msg"); + MsgInfo msg { MsgWhat::RELEASE, 0, nullptr }; + if (codec_->GetFirstSyncMsgToReply(msg)) { + ReplyErrorCode(msg.id, IC_ERR_OK); + } + codec_->notifyCallerAfterShutdownComplete_ = false; + } + codec_->isShutDownFromRunning_ = false; +} + +void ImageCodec::InitializedState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::SET_CALLBACK: { + OnSetCallBack(info); + return; + } + case MsgWhat::CONFIGURE: { + OnConfigure(info); + return; + } + case MsgWhat::GET_OUTPUT_BUFFER_USAGE: { + OnGetOutputBufferUsage(info); + break; + } + case MsgWhat::SET_OUTPUT_BUFFER: { + OnSetOutputBuffer(info); + break; + } + case MsgWhat::START: { + OnStart(info); + return; + } + default: { + BaseState::OnMsgReceived(info); + } + } +} + +void ImageCodec::InitializedState::OnSetCallBack(const MsgInfo &info) +{ + int32_t err; + shared_ptr cb; + (void)info.param->GetValue("callback", cb); + if (cb == nullptr) { + err = IC_ERR_INVALID_VAL; + SLOGE("invalid param"); + } else { + codec_->callback_ = cb; + err = IC_ERR_OK; + } + ReplyErrorCode(info.id, err); +} + +void ImageCodec::InitializedState::OnConfigure(const MsgInfo &info) +{ + Format fmt; + (void)info.param->GetValue("format", fmt); + ReplyErrorCode(info.id, codec_->OnConfigure(fmt)); +} + +void ImageCodec::InitializedState::OnGetOutputBufferUsage(const MsgInfo &info) +{ + ParamSP reply = ParamBundle::Create(); + reply->SetValue("err", IC_ERR_OK); + reply->SetValue("usage", codec_->OnGetOutputBufferUsage()); + codec_->PostReply(info.id, reply); +} + +void ImageCodec::InitializedState::OnSetOutputBuffer(const MsgInfo &info) +{ + sptr output; + (void)info.param->GetValue("output", output); + ReplyErrorCode(info.id, codec_->OnSetOutputBuffer(output)); +} + +void ImageCodec::InitializedState::OnStart(const MsgInfo &info) +{ + if (!codec_->ReadyToStart()) { + ReplyErrorCode(info.id, IC_ERR_INVALID_OPERATION); + return; + } + SLOGI("begin to set omx to idle"); + int32_t ret = codec_->compNode_->SendCommand(CODEC_COMMAND_STATE_SET, CODEC_STATE_IDLE, {}); + if (ret == HDF_SUCCESS) { + codec_->ReplyToSyncMsgLater(info); + codec_->ChangeStateTo(codec_->startingState_); + } else { + SLOGE("set omx to idle failed, ret=%{public}d", ret); + ReplyErrorCode(info.id, IC_ERR_UNKNOWN); + } +} + +void ImageCodec::InitializedState::OnShutDown(const MsgInfo &info) +{ + SLOGI("receive RELEASE"); + codec_->ChangeStateTo(codec_->uninitializedState_); + codec_->notifyCallerAfterShutdownComplete_ = false; + ReplyErrorCode(info.id, IC_ERR_OK); +} +/**************************** InitializedState End ******************************/ + +/**************************** StartingState Start ******************************/ +void ImageCodec::StartingState::OnStateEntered() +{ + hasError_ = false; + + ParamSP msg = ParamBundle::Create(); + msg->SetValue("generation", codec_->stateGeneration_); + codec_->SendAsyncMsg(MsgWhat::CHECK_IF_STUCK, msg, THREE_SECONDS_IN_US); + + int32_t ret = AllocateBuffers(); + if (ret != IC_ERR_OK) { + SLOGE("AllocateBuffers failed, back to init state"); + hasError_ = true; + ReplyStartMsg(ret); + codec_->ChangeStateTo(codec_->initializedState_); + } +} + +int32_t ImageCodec::StartingState::AllocateBuffers() +{ + int32_t ret = codec_->AllocateBuffersOnPort(OMX_DirInput); + if (ret != IC_ERR_OK) { + return ret; + } + ret = codec_->AllocateBuffersOnPort(OMX_DirOutput); + if (ret != IC_ERR_OK) { + return ret; + } + return IC_ERR_OK; +} + +void ImageCodec::StartingState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::GET_INPUT_FORMAT: + case MsgWhat::GET_OUTPUT_FORMAT: { + codec_->DeferMessage(info); + return; + } + case MsgWhat::START: { + ReplyErrorCode(info.id, IC_ERR_OK); + return; + } + case MsgWhat::CHECK_IF_STUCK: { + int32_t generation; + if (info.param->GetValue("generation", generation) && + generation == codec_->stateGeneration_) { + SLOGE("stucked, force state transition"); + hasError_ = true; + ReplyStartMsg(IC_ERR_UNKNOWN); + codec_->ChangeStateTo(codec_->initializedState_); + } + return; + } + default: { + BaseState::OnMsgReceived(info); + } + } +} + +void ImageCodec::StartingState::OnCodecEvent(CodecEventType event, uint32_t data1, uint32_t data2) +{ + if (event != CODEC_EVENT_CMD_COMPLETE) { + return BaseState::OnCodecEvent(event, data1, data2); + } + if (data1 != static_cast(CODEC_COMMAND_STATE_SET)) { + SLOGW("ignore event: data1=%{public}u, data2=%{public}u", data1, data2); + return; + } + if (data2 == static_cast(CODEC_STATE_IDLE)) { + SLOGI("omx now idle, begin to set omx to executing"); + int32_t ret = codec_->compNode_->SendCommand(CODEC_COMMAND_STATE_SET, CODEC_STATE_EXECUTING, {}); + if (ret != HDF_SUCCESS) { + SLOGE("set omx to executing failed, ret=%{public}d", ret); + hasError_ = true; + ReplyStartMsg(IC_ERR_UNKNOWN); + codec_->ChangeStateTo(codec_->initializedState_); + } + } else if (data2 == static_cast(CODEC_STATE_EXECUTING)) { + SLOGI("omx now executing"); + ReplyStartMsg(IC_ERR_OK); + codec_->SubmitAllBuffersOwnedByUs(); + codec_->ChangeStateTo(codec_->runningState_); + } +} + +void ImageCodec::StartingState::OnShutDown(const MsgInfo &info) +{ + codec_->DeferMessage(info); +} + +void ImageCodec::StartingState::ReplyStartMsg(int32_t errCode) +{ + MsgInfo msg {MsgWhat::START, 0, nullptr}; + if (codec_->GetFirstSyncMsgToReply(msg)) { + SLOGI("start %{public}s", (errCode == 0) ? "succ" : "failed"); + ReplyErrorCode(msg.id, errCode); + } else { + SLOGE("there should be a start msg to reply"); + } +} + +void ImageCodec::StartingState::OnStateExited() +{ + if (hasError_) { + SLOGW("error occured, roll omx back to loaded and free allocated buffers"); + if (codec_->RollOmxBackToLoaded()) { + codec_->ClearBufferPool(OMX_DirInput); + codec_->ClearBufferPool(OMX_DirOutput); + } + } + BaseState::OnStateExited(); +} + +/**************************** StartingState End ******************************/ + +/**************************** RunningState Start ********************************/ +void ImageCodec::RunningState::OnStateEntered() +{ + codec_->ProcessDeferredMessages(); +} + +void ImageCodec::RunningState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::START: + ReplyErrorCode(info.id, codec_->SubmitAllBuffersOwnedByUs()); + break; + case MsgWhat::QUEUE_INPUT_BUFFER: + codec_->OnQueueInputBuffer(info, inputMode_); + break; + case MsgWhat::RELEASE_OUTPUT_BUFFER: + codec_->OnReleaseOutputBuffer(info, outputMode_); + break; + default: + BaseState::OnMsgReceived(info); + break; + } +} + +void ImageCodec::RunningState::OnCodecEvent(CodecEventType event, uint32_t data1, uint32_t data2) +{ + switch (event) { + case CODEC_EVENT_PORT_SETTINGS_CHANGED: { + if (data1 != OMX_DirOutput) { + SLOGI("ignore input port changed"); + return; + } + if (data2 == 0 || data2 == OMX_IndexParamPortDefinition) { + SLOGI("output port settings changed, begin to ask omx to disable out port"); + codec_->UpdateOutPortFormat(); + int32_t ret = codec_->compNode_->SendCommand( + CODEC_COMMAND_PORT_DISABLE, OMX_DirOutput, {}); + if (ret == HDF_SUCCESS) { + codec_->EraseOutBuffersOwnedByUs(); + codec_->ChangeStateTo(codec_->outputPortChangedState_); + } else { + SLOGE("ask omx to disable out port failed"); + codec_->SignalError(IC_ERR_UNKNOWN); + } + } else if (data2 == OMX_IndexColorAspects) { + codec_->UpdateColorAspects(); + } else { + SLOGI("unknown data2 0x%{public}x for CODEC_EVENT_PORT_SETTINGS_CHANGED", data2); + } + return; + } + default: { + BaseState::OnCodecEvent(event, data1, data2); + } + } +} + +void ImageCodec::RunningState::OnShutDown(const MsgInfo &info) +{ + codec_->isShutDownFromRunning_ = true; + codec_->notifyCallerAfterShutdownComplete_ = true; + codec_->isBufferCirculating_ = false; + + SLOGI("receive release msg, begin to set omx to idle"); + int32_t ret = codec_->compNode_->SendCommand(CODEC_COMMAND_STATE_SET, CODEC_STATE_IDLE, {}); + if (ret == HDF_SUCCESS) { + codec_->ReplyToSyncMsgLater(info); + codec_->ChangeStateTo(codec_->stoppingState_); + } else { + SLOGE("set omx to idle failed, ret=%{public}d", ret); + ReplyErrorCode(info.id, IC_ERR_UNKNOWN); + } +} +/**************************** RunningState End ********************************/ + + +/**************************** OutputPortChangedState Start ********************************/ +void ImageCodec::OutputPortChangedState::OnStateEntered() +{ + ParamSP msg = ParamBundle::Create(); + msg->SetValue("generation", codec_->stateGeneration_); + codec_->SendAsyncMsg(MsgWhat::CHECK_IF_STUCK, msg, THREE_SECONDS_IN_US); +} + +void ImageCodec::OutputPortChangedState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::START: + case MsgWhat::GET_INPUT_FORMAT: + case MsgWhat::GET_OUTPUT_FORMAT: { + codec_->DeferMessage(info); + return; + } + case MsgWhat::QUEUE_INPUT_BUFFER: { + codec_->OnQueueInputBuffer(info, inputMode_); + return; + } + case MsgWhat::RELEASE_OUTPUT_BUFFER: { + codec_->OnReleaseOutputBuffer(info, outputMode_); + return; + } + case MsgWhat::FORCE_SHUTDOWN: { + OnForceShutDown(info); + return; + } + case MsgWhat::CHECK_IF_STUCK: { + OnCheckIfStuck(info); + return; + } + default: { + BaseState::OnMsgReceived(info); + } + } +} + +void ImageCodec::OutputPortChangedState::OnShutDown(const MsgInfo &info) +{ + if (codec_->hasFatalError_) { + ParamSP stopMsg = ParamBundle::Create(); + stopMsg->SetValue("generation", codec_->stateGeneration_); + codec_->SendAsyncMsg(MsgWhat::FORCE_SHUTDOWN, stopMsg, THREE_SECONDS_IN_US); + } + codec_->ReclaimBuffer(OMX_DirOutput, BufferOwner::OWNED_BY_USER, true); + codec_->DeferMessage(info); +} + +void ImageCodec::OutputPortChangedState::OnCodecEvent(CodecEventType event, uint32_t data1, uint32_t data2) +{ + switch (event) { + case CODEC_EVENT_CMD_COMPLETE: { + if (data1 == CODEC_COMMAND_PORT_DISABLE) { + if (data2 != OMX_DirOutput) { + SLOGW("ignore input port disable complete"); + return; + } + SLOGI("output port is disabled"); + HandleOutputPortDisabled(); + } else if (data1 == CODEC_COMMAND_PORT_ENABLE) { + if (data2 != OMX_DirOutput) { + SLOGW("ignore input port enable complete"); + return; + } + SLOGI("output port is enabled"); + HandleOutputPortEnabled(); + } + return; + } + default: { + BaseState::OnCodecEvent(event, data1, data2); + } + } +} + +void ImageCodec::OutputPortChangedState::HandleOutputPortDisabled() +{ + int32_t ret = IC_ERR_OK; + if (!codec_->outputBufferPool_.empty()) { + SLOGE("output port is disabled but not empty: %{public}zu", codec_->outputBufferPool_.size()); + ret = IC_ERR_UNKNOWN; + } + if (ret == IC_ERR_OK) { + ret = codec_->ReConfigureOutputBufferCnt(); + } + if (ret == IC_ERR_OK) { + SLOGI("begin to ask omx to enable out port"); + int32_t err = codec_->compNode_->SendCommand(CODEC_COMMAND_PORT_ENABLE, OMX_DirOutput, {}); + if (err == HDF_SUCCESS) { + ret = codec_->AllocateBuffersOnPort(OMX_DirOutput); + } else { + SLOGE("ask omx to enable out port failed, ret=%{public}d", ret); + ret = IC_ERR_UNKNOWN; + } + } + if (ret != IC_ERR_OK) { + codec_->SignalError(IC_ERR_INVALID_VAL); + } +} + +void ImageCodec::OutputPortChangedState::HandleOutputPortEnabled() +{ + if (codec_->isBufferCirculating_) { + codec_->SubmitOutputBuffersToOmxNode(); + } + codec_->callback_->OnOutputFormatChanged(*(codec_->outputFormat_.get())); + codec_->ChangeStateTo(codec_->runningState_); +} +/**************************** OutputPortChangedState End ********************************/ + +/**************************** StoppingState Start ********************************/ +void ImageCodec::StoppingState::OnStateEntered() +{ + omxNodeInIdleState_ = false; + omxNodeIsChangingToLoadedState_ = false; + codec_->ReclaimBuffer(OMX_DirInput, BufferOwner::OWNED_BY_USER); + codec_->ReclaimBuffer(OMX_DirOutput, BufferOwner::OWNED_BY_USER); + SLOGI("all buffer owned by user are now owned by us"); + + ParamSP msg = ParamBundle::Create(); + msg->SetValue("generation", codec_->stateGeneration_); + codec_->SendAsyncMsg(MsgWhat::CHECK_IF_STUCK, msg, THREE_SECONDS_IN_US); +} + +void ImageCodec::StoppingState::OnMsgReceived(const MsgInfo &info) +{ + switch (info.type) { + case MsgWhat::CHECK_IF_STUCK: { + int32_t generation; + (void)info.param->GetValue("generation", generation); + if (generation == codec_->stateGeneration_) { + SLOGE("stucked, force state transition"); + codec_->ReclaimBuffer(OMX_DirInput, BufferOwner::OWNED_BY_OMX); + codec_->ReclaimBuffer(OMX_DirOutput, BufferOwner::OWNED_BY_OMX); + SLOGI("all buffer owned by omx are now owned by us"); + ChangeOmxNodeToLoadedState(true); + codec_->ChangeStateTo(codec_->initializedState_); + } + return; + } + default: { + BaseState::OnMsgReceived(info); + } + } +} + +void ImageCodec::StoppingState::OnCodecEvent(CodecEventType event, uint32_t data1, uint32_t data2) +{ + switch (event) { + case CODEC_EVENT_CMD_COMPLETE: { + if (data1 != static_cast(CODEC_COMMAND_STATE_SET)) { + SLOGW("unexpected CODEC_EVENT_CMD_COMPLETE: %{public}u %{public}u", data1, data2); + return; + } + if (data2 == static_cast(CODEC_STATE_IDLE)) { + SLOGI("omx now idle"); + omxNodeInIdleState_ = true; + ChangeStateIfWeOwnAllBuffers(); + } else if (data2 == static_cast(CODEC_STATE_LOADED)) { + SLOGI("omx now loaded"); + codec_->ChangeStateTo(codec_->initializedState_); + } + return; + } + default: { + BaseState::OnCodecEvent(event, data1, data2); + } + } +} + +void ImageCodec::StoppingState::ChangeStateIfWeOwnAllBuffers() +{ + if (omxNodeInIdleState_ && codec_->IsAllBufferOwnedByUs()) { + ChangeOmxNodeToLoadedState(false); + } else { + SLOGD("cannot change state yet"); + } +} + +void ImageCodec::StoppingState::ChangeOmxNodeToLoadedState(bool forceToFreeBuffer) +{ + if (!omxNodeIsChangingToLoadedState_) { + SLOGI("begin to set omx to loaded"); + int32_t ret = codec_->compNode_->SendCommand(CODEC_COMMAND_STATE_SET, CODEC_STATE_LOADED, {}); + if (ret == HDF_SUCCESS) { + omxNodeIsChangingToLoadedState_ = true; + } else { + SLOGE("set omx to loaded failed, ret=%{public}d", ret); + } + } + if (forceToFreeBuffer || omxNodeIsChangingToLoadedState_) { + codec_->ClearBufferPool(OMX_DirInput); + codec_->ClearBufferPool(OMX_DirOutput); + return; + } + codec_->SignalError(IC_ERR_UNKNOWN); +} + +void ImageCodec::StoppingState::OnShutDown(const MsgInfo &info) +{ + codec_->DeferMessage(info); +} +/**************************** StoppingState End ********************************/ + +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/format.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/format.cpp new file mode 100644 index 000000000..3178bd9e4 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/format.cpp @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/format.h" +#include + +namespace OHOS::ImagePlugin { +using namespace std; + +Format::Format(const Format& src) +{ + m_items = src.m_items; +} + +Format& Format::operator=(const Format& src) +{ + m_items = src.m_items; + return *this; +} + +bool Format::ContainKey(const std::string &key) const +{ + return (m_items.find(key) != m_items.end()); +} +} // namespace OHOS::ImagePlugin diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec.cpp new file mode 100644 index 000000000..dbeadbcec --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec.cpp @@ -0,0 +1,988 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/image_codec.h" +#include "hardware/imagecodec/image_decoder.h" +#include "hardware/imagecodec/image_codec_list.h" +#include "hardware/imagecodec/image_codec_log.h" +#include "syspara/parameters.h" // base/startup/init/interfaces/innerkits/include/ +#include "qos.h" + +namespace OHOS::ImagePlugin { +using namespace std; +using namespace HdiCodecNamespace; + +static bool IsSecureMode(const string &name) +{ + string prefix = ".secure"; + if (name.length() <= prefix.length()) { + return false; + } + return (name.rfind(prefix) == (name.length() - prefix.length())); +} + +shared_ptr ImageCodec::Create() +{ + vector capList = GetCapList(); + shared_ptr codec; + string name; + for (const auto& cap : capList) { + if (cap.role == MEDIA_ROLETYPE_VIDEO_HEVC && cap.type == VIDEO_DECODER && !IsSecureMode(cap.compName)) { + name = cap.compName; + codec = make_shared(); + break; + } + } + if ((codec != nullptr) && (codec->InitWithName(name) == IC_ERR_OK)) { + return codec; + } + return nullptr; +} + +int32_t ImageCodec::SetCallback(const shared_ptr &callback) +{ + HLOGI(">>"); + function proc = [&](ParamSP msg) { + msg->SetValue("callback", callback); + }; + return DoSyncCall(MsgWhat::SET_CALLBACK, proc); +} + +int32_t ImageCodec::Configure(const Format &format) +{ + function proc = [&](ParamSP msg) { + msg->SetValue("format", format); + }; + return DoSyncCall(MsgWhat::CONFIGURE, proc); +} + +int32_t ImageCodec::QueueInputBuffer(uint32_t index) +{ + function proc = [&](ParamSP msg) { + msg->SetValue(BUFFER_ID, index); + }; + return DoSyncCall(MsgWhat::QUEUE_INPUT_BUFFER, proc); +} + +int32_t ImageCodec::ReleaseOutputBuffer(uint32_t index) +{ + function proc = [&](ParamSP msg) { + msg->SetValue(BUFFER_ID, index); + }; + return DoSyncCall(MsgWhat::RELEASE_OUTPUT_BUFFER, proc); +} + +int32_t ImageCodec::GetInputFormat(Format& format) +{ + HLOGI(">>"); + ParamSP reply; + int32_t ret = DoSyncCallAndGetReply(MsgWhat::GET_INPUT_FORMAT, nullptr, reply); + if (ret != IC_ERR_OK) { + HLOGE("failed to get input format"); + return ret; + } + IF_TRUE_RETURN_VAL_WITH_MSG(!reply->GetValue("format", format), + IC_ERR_UNKNOWN, "input format not replied"); + return IC_ERR_OK; +} + +int32_t ImageCodec::GetOutputFormat(Format& format) +{ + ParamSP reply; + int32_t ret = DoSyncCallAndGetReply(MsgWhat::GET_OUTPUT_FORMAT, nullptr, reply); + if (ret != IC_ERR_OK) { + HLOGE("failed to get output format"); + return ret; + } + IF_TRUE_RETURN_VAL_WITH_MSG(!reply->GetValue("format", format), + IC_ERR_UNKNOWN, "output format not replied"); + return IC_ERR_OK; +} + +int32_t ImageCodec::Start() +{ + HLOGI(">>"); + return DoSyncCall(MsgWhat::START, nullptr); +} + +int32_t ImageCodec::Release() +{ + HLOGI(">>"); + return DoSyncCall(MsgWhat::RELEASE, nullptr); +} + +int32_t ImageCodec::GetOutputBufferUsage(uint64_t& usage) +{ + ParamSP reply; + int32_t ret = DoSyncCallAndGetReply(MsgWhat::GET_OUTPUT_BUFFER_USAGE, nullptr, reply); + if (ret != IC_ERR_OK) { + HLOGE("failed to get output buffer usage"); + return ret; + } + IF_TRUE_RETURN_VAL_WITH_MSG(!reply->GetValue("usage", usage), + IC_ERR_UNKNOWN, "output buffer usage not replied"); + return IC_ERR_OK; +} + +int32_t ImageCodec::SetOutputBuffer(sptr output) +{ + HLOGI(">>"); + std::function proc = [&](ParamSP msg) { + msg->SetValue("output", output); + }; + return DoSyncCall(MsgWhat::SET_OUTPUT_BUFFER, proc); +} +/**************************** public functions end ****************************/ + +ImageCodec::ImageCodec(OMX_VIDEO_CODINGTYPE codingType, bool isEncoder) + : isEncoder_(isEncoder), codingType_(codingType) +{ + debugMode_ = OHOS::system::GetBoolParameter("image.codec.debug", false); + dumpMode_ = OHOS::system::GetBoolParameter("image.codec.dump", false); + LOGI(">> debug mode = %{public}d, dump mode = %{public}d", debugMode_, dumpMode_); + + uninitializedState_ = make_shared(this); + initializedState_ = make_shared(this); + startingState_ = make_shared(this); + runningState_ = make_shared(this); + outputPortChangedState_ = make_shared(this); + stoppingState_ = make_shared(this); + StateMachine::ChangeStateTo(uninitializedState_); +} + +ImageCodec::~ImageCodec() +{ + HLOGI(">>"); + MsgHandleLoop::Stop(); + ReleaseComponent(); +} + +int32_t ImageCodec::InitWithName(const string &name) +{ + function proc = [&](ParamSP msg) { + msg->SetValue("name", name); + }; + return DoSyncCall(MsgWhat::INIT, proc); +} + +const char* ImageCodec::ToString(BufferOwner owner) +{ + switch (owner) { + case BufferOwner::OWNED_BY_US: + return "us"; + case BufferOwner::OWNED_BY_USER: + return "user"; + case BufferOwner::OWNED_BY_OMX: + return "omx"; + default: + return ""; + } +} + +const char* ImageCodec::ToString(MsgWhat what) +{ + static const map m = { + { INIT, "INIT" }, + { SET_CALLBACK, "SET_CALLBACK" }, + { CONFIGURE, "CONFIGURE" }, + { START, "START" }, + { GET_INPUT_FORMAT, "GET_INPUT_FORMAT" }, + { GET_OUTPUT_FORMAT, "GET_OUTPUT_FORMAT" }, + { QUEUE_INPUT_BUFFER, "QUEUE_INPUT_BUFFER" }, + { RELEASE_OUTPUT_BUFFER, "RELEASE_OUTPUT_BUFFER" }, + { RELEASE, "RELEASE" }, + { GET_OUTPUT_BUFFER_USAGE, "GET_OUTPUT_BUFFER_USAGE" }, + { SET_OUTPUT_BUFFER, "SET_OUTPUT_BUFFER" }, + { CODEC_EVENT, "CODEC_EVENT" }, + { OMX_EMPTY_BUFFER_DONE, "OMX_EMPTY_BUFFER_DONE" }, + { OMX_FILL_BUFFER_DONE, "OMX_FILL_BUFFER_DONE" }, + { CHECK_IF_STUCK, "CHECK_IF_STUCK" }, + { FORCE_SHUTDOWN, "FORCE_SHUTDOWN" }, + }; + auto it = m.find(what); + if (it != m.end()) { + return it->second; + } + return "UNKNOWN"; +} + +void ImageCodec::ReplyErrorCode(MsgId id, int32_t err) +{ + if (id == ASYNC_MSG_ID) { + return; + } + ParamSP reply = ParamBundle::Create(); + reply->SetValue("err", err); + PostReply(id, reply); +} + +bool ImageCodec::GetPixelFmtFromUser(const Format &format) +{ + optional fmt; + int32_t graphicFmt; + if (format.GetValue(ImageCodecDescriptionKey::PIXEL_FORMAT, graphicFmt)) { + fmt = TypeConverter::GraphicFmtToFmt(static_cast(graphicFmt)); + } else { + HLOGE("pixel format unspecified"); + return false; + } + configuredFmt_ = fmt.value(); + HLOGI("configured pixel format is %{public}s", configuredFmt_.strFmt.c_str()); + return true; +} + +optional ImageCodec::GetFrameRateFromUser(const Format &format) +{ + double frameRate; + if (format.GetValue(ImageCodecDescriptionKey::FRAME_RATE, frameRate) && frameRate > 0) { + LOGI("user set frame rate %{public}.2f", frameRate); + return frameRate; + } + return nullopt; +} + +int32_t ImageCodec::SetFrameRateAdaptiveMode(const Format &format) +{ + if (!format.ContainKey(ImageCodecDescriptionKey::VIDEO_FRAME_RATE_ADAPTIVE_MODE)) { + return IC_ERR_UNKNOWN; + } + + WorkingFrequencyParam param {}; + InitOMXParamExt(param); + if (!GetParameter(OMX_IndexParamWorkingFrequency, param)) { + HLOGW("get working freq param failed"); + return IC_ERR_UNKNOWN; + } + HLOGI("level cnt is %{public}d, set level to %{public}d", param.level, param.level - 1); + param.level = param.level - 1; + + if (!SetParameter(OMX_IndexParamWorkingFrequency, param)) { + HLOGW("set working freq param failed"); + return IC_ERR_UNKNOWN; + } + return IC_ERR_OK; +} + +int32_t ImageCodec::SetProcessName(const Format &format) +{ + string processName; + if (!format.GetValue(ImageCodecDescriptionKey::PROCESS_NAME, processName)) { + return IC_ERR_UNKNOWN; + } + HLOGI("processName name is %{public}s", processName.c_str()); + + ProcessNameParam param {}; + InitOMXParamExt(param); + if (strcpy_s(param.processName, sizeof(param.processName), processName.c_str()) != EOK) { + HLOGW("strcpy failed"); + return IC_ERR_UNKNOWN; + } + if (!SetParameter(OMX_IndexParamProcessName, param)) { + HLOGW("set process name failed"); + return IC_ERR_UNKNOWN; + } + return IC_ERR_OK; +} + +int32_t ImageCodec::SetVideoPortInfo(OMX_DIRTYPE portIndex, const PortInfo& info) +{ + if (info.pixelFmt.has_value()) { + CodecVideoPortFormatParam param; + InitOMXParamExt(param); + param.portIndex = portIndex; + param.codecCompressFormat = info.codingType; + param.codecColorFormat = info.pixelFmt->graphicFmt; + param.framerate = info.frameRate * FRAME_RATE_COEFFICIENT; + if (!SetParameter(OMX_IndexCodecVideoPortFormat, param)) { + HLOGE("set port format failed"); + return IC_ERR_UNKNOWN; + } + } + { + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOMXParam(def); + def.nPortIndex = portIndex; + if (!GetParameter(OMX_IndexParamPortDefinition, def)) { + HLOGE("get port definition failed"); + return IC_ERR_UNKNOWN; + } + def.format.video.nFrameWidth = info.width; + def.format.video.nFrameHeight = info.height; + def.format.video.eCompressionFormat = info.codingType; + // we dont set eColorFormat here because it has been set by CodecVideoPortFormatParam + def.format.video.xFramerate = info.frameRate * FRAME_RATE_COEFFICIENT; + if (portIndex == OMX_DirInput && info.inputBufSize.has_value()) { + def.nBufferSize = info.inputBufSize.value(); + } + if (info.bufferCnt.has_value()) { + def.nBufferCountActual = info.bufferCnt.value(); + } + if (!SetParameter(OMX_IndexParamPortDefinition, def)) { + HLOGE("set port definition failed"); + return IC_ERR_UNKNOWN; + } + if (portIndex == OMX_DirOutput) { + if (outputFormat_ == nullptr) { + outputFormat_ = make_shared(); + } + outputFormat_->SetValue(ImageCodecDescriptionKey::FRAME_RATE, info.frameRate); + } + } + + return (portIndex == OMX_DirInput) ? UpdateInPortFormat() : UpdateOutPortFormat(); +} + +void ImageCodec::PrintPortDefinition(const OMX_PARAM_PORTDEFINITIONTYPE& def) +{ + const OMX_VIDEO_PORTDEFINITIONTYPE& video = def.format.video; + HLOGI("----- %{public}s port definition -----", (def.nPortIndex == OMX_DirInput) ? "INPUT" : "OUTPUT"); + HLOGI("bEnabled %{public}d, bPopulated %{public}d", def.bEnabled, def.bPopulated); + HLOGI("nBufferCountActual %{public}u, nBufferSize %{public}u", def.nBufferCountActual, def.nBufferSize); + HLOGI("nFrameWidth x nFrameHeight (%{public}u x %{public}u), framerate %{public}u(%{public}.2f)", + video.nFrameWidth, video.nFrameHeight, video.xFramerate, video.xFramerate / FRAME_RATE_COEFFICIENT); + HLOGI(" nStride x nSliceHeight (%{public}u x %{public}u)", video.nStride, video.nSliceHeight); + HLOGI("eCompressionFormat %{public}d(%{public}#x), eColorFormat %{public}d(%{public}#x)", + video.eCompressionFormat, video.eCompressionFormat, video.eColorFormat, video.eColorFormat); + HLOGI("----------------------------------"); +} + +int32_t ImageCodec::GetPortDefinition(OMX_DIRTYPE portIndex, OMX_PARAM_PORTDEFINITIONTYPE& def) +{ + InitOMXParam(def); + def.nPortIndex = portIndex; + if (!GetParameter(OMX_IndexParamPortDefinition, def)) { + HLOGE("get %{public}s port definition failed", (portIndex == OMX_DirInput ? "input" : "output")); + return IC_ERR_INVALID_VAL; + } + if (def.nBufferSize == 0 || def.nBufferSize > MAX_IMAGE_CODEC_BUFFER_SIZE) { + HLOGE("invalid nBufferSize %{public}u", def.nBufferSize); + return IC_ERR_INVALID_VAL; + } + PrintPortDefinition(def); + return IC_ERR_OK; +} + +int32_t ImageCodec::AllocateHardwareBuffers(OMX_DIRTYPE portIndex) +{ + HeifPerfTracker tracker(__FUNCTION__); + OMX_PARAM_PORTDEFINITIONTYPE def; + int32_t ret = GetPortDefinition(portIndex, def); + IF_TRUE_RETURN_VAL(ret != IC_ERR_OK, ret); + + vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + pool.clear(); + for (uint32_t i = 0; i < def.nBufferCountActual; ++i) { + shared_ptr omxBuffer = make_shared(); + omxBuffer->size = sizeof(OmxCodecBuffer); + omxBuffer->version.version.majorVersion = 1; + omxBuffer->bufferType = CODEC_BUFFER_TYPE_DMA_MEM_FD; + omxBuffer->fd = -1; + omxBuffer->allocLen = def.nBufferSize; + omxBuffer->fenceFd = -1; + shared_ptr outBuffer = make_shared(); + ret = compNode_->AllocateBuffer(portIndex, *omxBuffer, *outBuffer); + if (ret != HDF_SUCCESS) { + HLOGE("Failed to AllocateBuffer on %{public}s port", (portIndex == OMX_DirInput ? "input" : "output")); + return IC_ERR_INVALID_VAL; + } + shared_ptr imgCodecBuffer = ImageCodecBuffer::CreateDmaBuffer(outBuffer->fd, + static_cast(def.nBufferSize), static_cast(def.format.video.nStride)); + if (imgCodecBuffer == nullptr || imgCodecBuffer->GetCapacity() != static_cast(def.nBufferSize)) { + HLOGE("AllocateHardwareBuffers failed"); + return IC_ERR_NO_MEMORY; + } + BufferInfo bufInfo; + bufInfo.isInput = (portIndex == OMX_DirInput) ? true : false; + bufInfo.owner = BufferOwner::OWNED_BY_US; + bufInfo.surfaceBuffer = nullptr; + bufInfo.imgCodecBuffer = imgCodecBuffer; + bufInfo.omxBuffer = outBuffer; + bufInfo.bufferId = outBuffer->bufferId; + bufInfo.CleanUpUnusedInfo(); + pool.push_back(bufInfo); + } + return IC_ERR_OK; +} + +int32_t ImageCodec::AllocateSurfaceBuffers(OMX_DIRTYPE portIndex, sptr output) +{ + HeifPerfTracker tracker(__FUNCTION__); + OMX_PARAM_PORTDEFINITIONTYPE def; + int32_t ret = GetPortDefinition(portIndex, def); + if (ret != IC_ERR_OK) { + return ret; + } + vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + pool.clear(); + for (uint32_t i = 0; i < def.nBufferCountActual; ++i) { + shared_ptr imgCodecBuffer = (output != nullptr) ? + ImageCodecBuffer::CreateSurfaceBuffer(output) : ImageCodecBuffer::CreateSurfaceBuffer(requestCfg_); + if (imgCodecBuffer == nullptr) { + HLOGE("AllocateSurfaceBuffers failed"); + return IC_ERR_NO_MEMORY; + } + sptr surfaceBuffer = imgCodecBuffer->GetSurfaceBuffer(); + IF_TRUE_RETURN_VAL_WITH_MSG(surfaceBuffer == nullptr, IC_ERR_INVALID_VAL, "failed to get surfacebuffer"); + shared_ptr omxBuffer = isEncoder_ ? + DynamicSurfaceBufferToOmxBuffer() : SurfaceBufferToOmxBuffer(surfaceBuffer); + IF_TRUE_RETURN_VAL(omxBuffer == nullptr, IC_ERR_INVALID_VAL); + shared_ptr outBuffer = make_shared(); + int32_t hdiRet = compNode_->UseBuffer(portIndex, *omxBuffer, *outBuffer); + if (hdiRet != HDF_SUCCESS) { + HLOGE("Failed to UseBuffer on %{public}s port", (portIndex == OMX_DirInput ? "input" : "output")); + return IC_ERR_INVALID_VAL; + } + BufferInfo bufInfo; + bufInfo.isInput = (portIndex == OMX_DirInput) ? true : false; + bufInfo.owner = BufferOwner::OWNED_BY_US; + bufInfo.surfaceBuffer = surfaceBuffer; + bufInfo.imgCodecBuffer = imgCodecBuffer; + bufInfo.omxBuffer = outBuffer; + bufInfo.bufferId = outBuffer->bufferId; + pool.push_back(bufInfo); + } + + return IC_ERR_OK; +} + +shared_ptr ImageCodec::SurfaceBufferToOmxBuffer(const sptr& surfaceBuffer) +{ + BufferHandle* bufferHandle = surfaceBuffer->GetBufferHandle(); + IF_TRUE_RETURN_VAL_WITH_MSG(bufferHandle == nullptr, nullptr, "surfacebuffer has null bufferhandle"); + auto omxBuffer = make_shared(); + omxBuffer->size = sizeof(OmxCodecBuffer); + omxBuffer->version.version.majorVersion = 1; + omxBuffer->bufferType = CODEC_BUFFER_TYPE_HANDLE; + omxBuffer->bufferhandle = new NativeBuffer(bufferHandle); + omxBuffer->fd = -1; + omxBuffer->allocLen = surfaceBuffer->GetSize(); + omxBuffer->fenceFd = -1; + return omxBuffer; +} + +shared_ptr ImageCodec::DynamicSurfaceBufferToOmxBuffer() +{ + auto omxBuffer = make_shared(); + omxBuffer->size = sizeof(OmxCodecBuffer); + omxBuffer->version.version.majorVersion = 1; + omxBuffer->bufferType = CODEC_BUFFER_TYPE_DYNAMIC_HANDLE; + omxBuffer->fd = -1; + omxBuffer->allocLen = 0; + omxBuffer->fenceFd = -1; + return omxBuffer; +} + +ImageCodec::BufferInfo* ImageCodec::FindBufferInfoByID(OMX_DIRTYPE portIndex, uint32_t bufferId) +{ + vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + for (BufferInfo &info : pool) { + if (info.bufferId == bufferId) { + return &info; + } + } + HLOGE("unknown buffer id %{public}u", bufferId); + return nullptr; +} + +optional ImageCodec::FindBufferIndexByID(OMX_DIRTYPE portIndex, uint32_t bufferId) +{ + const vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + for (size_t i = 0; i < pool.size(); i++) { + if (pool[i].bufferId == bufferId) { + return i; + } + } + HLOGE("unknown buffer id %{public}u", bufferId); + return nullopt; +} + +void ImageCodec::NotifyUserToFillThisInBuffer(BufferInfo &info) +{ + callback_->OnInputBufferAvailable(info.bufferId, info.imgCodecBuffer); + ChangeOwner(info, BufferOwner::OWNED_BY_USER); +} + +void ImageCodec::OnQueueInputBuffer(const MsgInfo &msg, BufferOperationMode mode) +{ + uint32_t bufferId; + (void)msg.param->GetValue(BUFFER_ID, bufferId); + BufferInfo* bufferInfo = FindBufferInfoByID(OMX_DirInput, bufferId); + if (bufferInfo == nullptr) { + ReplyErrorCode(msg.id, IC_ERR_INVALID_VAL); + return; + } + if (bufferInfo->owner != BufferOwner::OWNED_BY_USER) { + HLOGE("wrong ownership: buffer id=%{public}d, owner=%{public}s", bufferId, ToString(bufferInfo->owner)); + ReplyErrorCode(msg.id, IC_ERR_INVALID_VAL); + return; + } + bufferInfo->imgCodecBuffer->GetBufferCirculateInfo(bufferInfo->omxBuffer->pts, + bufferInfo->omxBuffer->flag, + bufferInfo->omxBuffer->filledLen, + bufferInfo->omxBuffer->offset); + ChangeOwner(*bufferInfo, BufferOwner::OWNED_BY_US); + ReplyErrorCode(msg.id, IC_ERR_OK); + OnQueueInputBuffer(mode, bufferInfo); +} + +void ImageCodec::OnQueueInputBuffer(BufferOperationMode mode, BufferInfo* info) +{ + switch (mode) { + case KEEP_BUFFER: { + return; + } + case RESUBMIT_BUFFER: { + if (inputPortEos_) { + HLOGI("input already eos, keep this buffer"); + return; + } + bool eos = (info->omxBuffer->flag & OMX_BUFFERFLAG_EOS); + if (!eos && info->omxBuffer->filledLen == 0) { + HLOGI("this is not a eos buffer but not filled, ask user to re-fill it"); + NotifyUserToFillThisInBuffer(*info); + return; + } + if (eos) { + inputPortEos_ = true; + } + int32_t ret = NotifyOmxToEmptyThisInBuffer(*info); + if (ret != IC_ERR_OK) { + SignalError(IC_ERR_UNKNOWN); + } + return; + } + default: { + HLOGE("SHOULD NEVER BE HERE"); + return; + } + } +} + +int32_t ImageCodec::NotifyOmxToEmptyThisInBuffer(BufferInfo& info) +{ + info.Dump(compUniqueStr_, dumpMode_); + info.EndCpuAccess(); + int32_t ret = compNode_->EmptyThisBuffer(*(info.omxBuffer)); + if (ret != HDF_SUCCESS) { + HLOGE("EmptyThisBuffer failed"); + return IC_ERR_UNKNOWN; + } + ChangeOwner(info, BufferOwner::OWNED_BY_OMX); + return IC_ERR_OK; +} + +int32_t ImageCodec::NotifyOmxToFillThisOutBuffer(BufferInfo& info) +{ + info.omxBuffer->flag = 0; + int32_t ret = compNode_->FillThisBuffer(*(info.omxBuffer)); + if (ret != HDF_SUCCESS) { + HLOGE("outBufId = %{public}u failed", info.bufferId); + return IC_ERR_UNKNOWN; + } + ChangeOwner(info, BufferOwner::OWNED_BY_OMX); + return IC_ERR_OK; +} + +void ImageCodec::OnOMXFillBufferDone(const OmxCodecBuffer& omxBuffer, BufferOperationMode mode) +{ + optional idx = FindBufferIndexByID(OMX_DirOutput, omxBuffer.bufferId); + if (!idx.has_value()) { + return; + } + BufferInfo& info = outputBufferPool_[idx.value()]; + if (info.owner != BufferOwner::OWNED_BY_OMX) { + HLOGE("wrong ownership: buffer id=%{public}d, owner=%{public}s", info.bufferId, ToString(info.owner)); + return; + } + info.omxBuffer->offset = omxBuffer.offset; + info.omxBuffer->filledLen = omxBuffer.filledLen; + info.omxBuffer->pts = omxBuffer.pts; + info.omxBuffer->flag = omxBuffer.flag; + ChangeOwner(info, BufferOwner::OWNED_BY_US); + OnOMXFillBufferDone(mode, info, idx.value()); +} + +void ImageCodec::OnOMXFillBufferDone(BufferOperationMode mode, BufferInfo& info, size_t bufferIdx) +{ + switch (mode) { + case KEEP_BUFFER: + return; + case RESUBMIT_BUFFER: { + if (outputPortEos_) { + HLOGI("output eos, keep this buffer"); + return; + } + bool eos = (info.omxBuffer->flag & OMX_BUFFERFLAG_EOS); + if (!eos && info.omxBuffer->filledLen == 0) { + HLOGI("it's not a eos buffer but not filled, ask omx to re-fill it"); + NotifyOmxToFillThisOutBuffer(info); + return; + } + NotifyUserOutBufferAvaliable(info); + if (eos) { + outputPortEos_ = true; + } + return; + } + case FREE_BUFFER: + EraseBufferFromPool(OMX_DirOutput, bufferIdx); + return; + default: + HLOGE("SHOULD NEVER BE HERE"); + return; + } +} + +void ImageCodec::NotifyUserOutBufferAvaliable(BufferInfo &info) +{ + info.BeginCpuAccess(); + info.Dump(compUniqueStr_, dumpMode_); + shared_ptr omxBuffer = info.omxBuffer; + info.imgCodecBuffer->SetBufferCirculateInfo(omxBuffer->pts, omxBuffer->flag, + omxBuffer->filledLen, omxBuffer->offset); + callback_->OnOutputBufferAvailable(info.bufferId, info.imgCodecBuffer); + ChangeOwner(info, BufferOwner::OWNED_BY_USER); +} + +void ImageCodec::OnReleaseOutputBuffer(const MsgInfo &msg, BufferOperationMode mode) +{ + uint32_t bufferId; + (void)msg.param->GetValue(BUFFER_ID, bufferId); + optional idx = FindBufferIndexByID(OMX_DirOutput, bufferId); + if (!idx.has_value()) { + ReplyErrorCode(msg.id, IC_ERR_INVALID_VAL); + return; + } + BufferInfo& info = outputBufferPool_[idx.value()]; + if (info.owner != BufferOwner::OWNED_BY_USER) { + HLOGE("wrong ownership: buffer id=%{public}d, owner=%{public}s", bufferId, ToString(info.owner)); + ReplyErrorCode(msg.id, IC_ERR_INVALID_VAL); + return; + } + HLOGD("outBufId = %{public}u", bufferId); + ChangeOwner(info, BufferOwner::OWNED_BY_US); + ReplyErrorCode(msg.id, IC_ERR_OK); + + switch (mode) { + case KEEP_BUFFER: { + return; + } + case RESUBMIT_BUFFER: { + if (outputPortEos_) { + HLOGI("output eos, keep this buffer"); + return; + } + int32_t ret = NotifyOmxToFillThisOutBuffer(info); + if (ret != IC_ERR_OK) { + SignalError(IC_ERR_UNKNOWN); + } + return; + } + case FREE_BUFFER: { + EraseBufferFromPool(OMX_DirOutput, idx.value()); + return; + } + default: { + HLOGE("SHOULD NEVER BE HERE"); + return; + } + } +} + +void ImageCodec::ReclaimBuffer(OMX_DIRTYPE portIndex, BufferOwner owner, bool erase) +{ + vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + for (size_t i = pool.size(); i > 0;) { + i--; + BufferInfo& info = pool[i]; + if (info.owner == owner) { + ChangeOwner(info, BufferOwner::OWNED_BY_US); + if (erase) { + EraseBufferFromPool(portIndex, i); + } + } + } +} + +bool ImageCodec::IsAllBufferOwnedByUs(OMX_DIRTYPE portIndex) +{ + const vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + for (const BufferInfo& info : pool) { + if (info.owner != BufferOwner::OWNED_BY_US) { + return false; + } + } + return true; +} + +bool ImageCodec::IsAllBufferOwnedByUs() +{ + return IsAllBufferOwnedByUs(OMX_DirInput) && IsAllBufferOwnedByUs(OMX_DirOutput); +} + +void ImageCodec::EraseOutBuffersOwnedByUs() +{ + // traverse index in reverse order because we need to erase index from vector + for (size_t i = outputBufferPool_.size(); i > 0;) { + i--; + const BufferInfo& info = outputBufferPool_[i]; + if (info.owner == BufferOwner::OWNED_BY_US) { + EraseBufferFromPool(OMX_DirOutput, i); + } + } +} + +void ImageCodec::ClearBufferPool(OMX_DIRTYPE portIndex) +{ + const vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + for (size_t i = pool.size(); i > 0;) { + i--; + EraseBufferFromPool(portIndex, i); + } +} + +void ImageCodec::FreeOmxBuffer(OMX_DIRTYPE portIndex, const BufferInfo& info) +{ + if (compNode_ && info.omxBuffer) { + int32_t omxRet = compNode_->FreeBuffer(portIndex, *(info.omxBuffer)); + if (omxRet != HDF_SUCCESS) { + HLOGW("notify omx to free buffer failed"); + } + } +} + +int32_t ImageCodec::DoSyncCall(MsgWhat msgType, function oper) +{ + ParamSP reply; + return DoSyncCallAndGetReply(msgType, oper, reply); +} + +int32_t ImageCodec::DoSyncCallAndGetReply(MsgWhat msgType, function oper, ParamSP &reply) +{ + ParamSP msg = ParamBundle::Create(); + IF_TRUE_RETURN_VAL_WITH_MSG(msg == nullptr, IC_ERR_NO_MEMORY, "out of memory"); + if (oper) { + oper(msg); + } + bool ret = MsgHandleLoop::SendSyncMsg(msgType, msg, reply); + IF_TRUE_RETURN_VAL_WITH_MSG(!ret, IC_ERR_UNKNOWN, "wait msg %{public}d time out", msgType); + int32_t err; + IF_TRUE_RETURN_VAL_WITH_MSG(reply == nullptr || !reply->GetValue("err", err), + IC_ERR_UNKNOWN, "error code of msg %{public}d not replied", msgType); + return err; +} + +void ImageCodec::ChangeOmxToTargetState(CodecStateType &state, CodecStateType targetState) +{ + int32_t ret = compNode_->SendCommand(CODEC_COMMAND_STATE_SET, targetState, {}); + if (ret != HDF_SUCCESS) { + HLOGE("failed to change omx state, ret=%{public}d", ret); + return; + } + + int tryCnt = 0; + do { + if (tryCnt++ > 10) { // try up to 10 times + HLOGE("failed to change to state(%{public}d), abort", targetState); + state = CODEC_STATE_INVALID; + break; + } + this_thread::sleep_for(10ms); // wait 10ms + ret = compNode_->GetState(state); + if (ret != HDF_SUCCESS) { + HLOGE("failed to get omx state, ret=%{public}d", ret); + } + } while (ret == HDF_SUCCESS && state != targetState && state != CODEC_STATE_INVALID); +} + +bool ImageCodec::RollOmxBackToLoaded() +{ + CodecStateType state; + int32_t ret = compNode_->GetState(state); + if (ret != HDF_SUCCESS) { + HLOGE("failed to get omx node status(ret=%{public}d), can not perform state rollback", ret); + return false; + } + HLOGI("current omx state (%{public}d)", state); + switch (state) { + case CODEC_STATE_EXECUTING: { + ChangeOmxToTargetState(state, CODEC_STATE_IDLE); + [[fallthrough]]; + } + case CODEC_STATE_IDLE: { + ChangeOmxToTargetState(state, CODEC_STATE_LOADED); + [[fallthrough]]; + } + case CODEC_STATE_LOADED: + case CODEC_STATE_INVALID: { + return true; + } + default: { + HLOGE("invalid omx state: %{public}d", state); + return false; + } + } +} + +void ImageCodec::CleanUpOmxNode() +{ + if (compNode_ == nullptr) { + return; + } + + if (RollOmxBackToLoaded()) { + for (const BufferInfo& info : inputBufferPool_) { + FreeOmxBuffer(OMX_DirInput, info); + } + for (const BufferInfo& info : outputBufferPool_) { + FreeOmxBuffer(OMX_DirOutput, info); + } + } +} + +void ImageCodec::ReleaseComponent() +{ + CleanUpOmxNode(); + if (compMgr_ != nullptr) { + compMgr_->DestroyComponent(componentId_); + } + compNode_ = nullptr; + compCb_ = nullptr; + compMgr_ = nullptr; + componentId_ = 0; + componentName_.clear(); +} + +int32_t ImageCodec::ForceShutdown(int32_t generation) +{ + if (generation != stateGeneration_) { + HLOGE("ignoring stale force shutdown message: #%{public}d (now #%{public}d)", + generation, stateGeneration_); + return IC_ERR_OK; + } + HLOGI("force to shutdown"); + isShutDownFromRunning_ = true; + notifyCallerAfterShutdownComplete_ = false; + auto err = compNode_->SendCommand(CODEC_COMMAND_STATE_SET, CODEC_STATE_IDLE, {}); + if (err == HDF_SUCCESS) { + ChangeStateTo(stoppingState_); + } + return IC_ERR_OK; +} + +void ImageCodec::SignalError(ImageCodecError err) +{ + HLOGE("fatal error happened: errCode=%{public}d", err); + hasFatalError_ = true; + callback_->OnError(err); +} + +void ImageCodec::DeferMessage(const MsgInfo &info) +{ + deferredQueue_.push_back(info); +} + +void ImageCodec::ProcessDeferredMessages() +{ + for (const MsgInfo &info : deferredQueue_) { + StateMachine::OnMsgReceived(info); + } + deferredQueue_.clear(); +} + +void ImageCodec::ReplyToSyncMsgLater(const MsgInfo& msg) +{ + syncMsgToReply_[msg.type].push(make_pair(msg.id, msg.param)); +} + +bool ImageCodec::GetFirstSyncMsgToReply(MsgInfo& msg) +{ + auto iter = syncMsgToReply_.find(msg.type); + if (iter == syncMsgToReply_.end()) { + return false; + } + tie(msg.id, msg.param) = iter->second.front(); + iter->second.pop(); + return true; +} + +/**************************** HdiCallback functions begin ****************************/ +int32_t ImageCodec::HdiCallback::EventHandler(CodecEventType event, const EventInfo &info) +{ + LOGI("event = %{public}d, data1 = %{public}u, data2 = %{public}u", event, info.data1, info.data2); + ParamSP msg = ParamBundle::Create(); + msg->SetValue("event", event); + msg->SetValue("data1", info.data1); + msg->SetValue("data2", info.data2); + codec_->SendAsyncMsg(MsgWhat::CODEC_EVENT, msg); + return HDF_SUCCESS; +} + +int32_t ImageCodec::HdiCallback::EmptyBufferDone(int64_t appData, const OmxCodecBuffer& buffer) +{ + ParamSP msg = ParamBundle::Create(); + msg->SetValue(BUFFER_ID, buffer.bufferId); + codec_->SendAsyncMsg(MsgWhat::OMX_EMPTY_BUFFER_DONE, msg); + return HDF_SUCCESS; +} + +int32_t ImageCodec::HdiCallback::FillBufferDone(int64_t appData, const OmxCodecBuffer& buffer) +{ + ParamSP msg = ParamBundle::Create(); + msg->SetValue("omxBuffer", buffer); + codec_->SendAsyncMsg(MsgWhat::OMX_FILL_BUFFER_DONE, msg); + return HDF_SUCCESS; +} +/**************************** HdiCallback functions begin ****************************/ + +/**************************** BufferInfo functions begin ****************************/ +void ImageCodec::BufferInfo::CleanUpUnusedInfo() +{ + if (omxBuffer == nullptr || omxBuffer->fd < 0) { + return; + } + if (omxBuffer->fd == 0) { + LOGW("fd of omxbuffer should never be 0"); + } + close(omxBuffer->fd); + omxBuffer->fd = -1; +} + +void ImageCodec::BufferInfo::BeginCpuAccess() +{ + if (surfaceBuffer && (surfaceBuffer->GetUsage() & BUFFER_USAGE_MEM_MMZ_CACHE)) { + GSError err = surfaceBuffer->InvalidateCache(); + if (err != GSERROR_OK) { + LOGW("InvalidateCache failed, GSError=%{public}d", err); + } + } +} + +void ImageCodec::BufferInfo::EndCpuAccess() +{ + if (surfaceBuffer && (surfaceBuffer->GetUsage() & BUFFER_USAGE_MEM_MMZ_CACHE)) { + GSError err = surfaceBuffer->Map(); + if (err != GSERROR_OK) { + LOGW("Map failed, GSError=%{public}d", err); + return; + } + err = surfaceBuffer->FlushCache(); + if (err != GSERROR_OK) { + LOGW("FlushCache failed, GSError=%{public}d", err); + } + } +} +/**************************** BufferInfo functions end ****************************/ +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_buffer.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_buffer.cpp new file mode 100644 index 000000000..2f2d59f0d --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_buffer.cpp @@ -0,0 +1,139 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/image_codec_buffer.h" +#include "hardware/imagecodec/image_codec_log.h" +#include "sys/mman.h" +#include "ashmem.h" // commonlibrary/c_utils/base/include/ashmem.h + +namespace OHOS::ImagePlugin { +using namespace std; + +/*============================ ImageCodecBuffer ====================================*/ +std::shared_ptr ImageCodecBuffer::CreateDmaBuffer(int fd, int32_t capacity, int32_t stride) +{ + if (fd < 0 || capacity <= 0) { + LOGE("invalid input param: fd=%{public}d, capacity=%{public}d", fd, capacity); + return nullptr; + } + return make_shared(dup(fd), capacity, stride); +} + +std::shared_ptr ImageCodecBuffer::CreateSurfaceBuffer(const BufferRequestConfig &config) +{ + std::shared_ptr buf = make_shared(config); + if (buf != nullptr && buf->Init()) { + return buf; + } + return nullptr; +} + +std::shared_ptr ImageCodecBuffer::CreateSurfaceBuffer(sptr surface) +{ + std::shared_ptr buf = make_shared(surface); + return buf; +} + +/*============================ ImageDmaBuffer ====================================*/ +ImageDmaBuffer::ImageDmaBuffer(int fd, int32_t capacity, int32_t stride): fd_(fd) +{ + capacity_ = capacity; + stride_ = stride; +} + +ImageDmaBuffer::~ImageDmaBuffer() +{ + if (addr_ != nullptr) { + (void)::munmap(addr_, static_cast(capacity_)); + addr_ = nullptr; + } + + if (fd_ >= 0) { + close(fd_); + fd_ = -1; + } +} + +uint8_t* ImageDmaBuffer::GetAddr() +{ + if (addr_ == nullptr) { + IF_TRUE_RETURN_VAL_WITH_MSG(fd_ < 0, nullptr, "invalid fd=%{public}d", fd_); + unsigned int prot = PROT_READ | PROT_WRITE; + void *addr = ::mmap(nullptr, static_cast(capacity_), static_cast(prot), MAP_SHARED, fd_, 0); + IF_TRUE_RETURN_VAL_WITH_MSG(addr == MAP_FAILED, nullptr, "mmap failed for fd=%{public}d", fd_); + addr_ = reinterpret_cast(addr); + } + return addr_; +} + +/*============================ ImageSurfaceBuffer ====================================*/ +ImageSurfaceBuffer::ImageSurfaceBuffer(const BufferRequestConfig &config): config_(config) +{} + +ImageSurfaceBuffer::ImageSurfaceBuffer(sptr surface) +{ + surfaceBuffer_ = surface; + stride_ = surfaceBuffer_->GetStride(); +} + +ImageSurfaceBuffer::~ImageSurfaceBuffer() +{ + if (addr_ != nullptr) { + GSError ret = surfaceBuffer_->Unmap(); + if (ret != GSERROR_OK) { + LOGE("failed to unmap surface buffer"); + } + addr_ = nullptr; + } +} + +bool ImageSurfaceBuffer::Init() +{ + surfaceBuffer_ = SurfaceBuffer::Create(); + IF_TRUE_RETURN_VAL_WITH_MSG(surfaceBuffer_ == nullptr, false, "failed to create surface buffer"); + GSError ret = surfaceBuffer_->Alloc(config_); + IF_TRUE_RETURN_VAL_WITH_MSG(ret != GSERROR_OK, false, "failed to alloc surface buffer"); + capacity_ = static_cast(surfaceBuffer_->GetSize()); + stride_ = surfaceBuffer_->GetStride(); + return true; +} + +int32_t ImageSurfaceBuffer::GetFileDescriptor() +{ + return surfaceBuffer_->GetFileDescriptor(); +} + +uint8_t* ImageSurfaceBuffer::GetAddr() +{ + if (addr_ == nullptr) { + GSError ret = surfaceBuffer_->Map(); + IF_TRUE_RETURN_VAL_WITH_MSG(ret != GSERROR_OK, nullptr, + "mmap failed, err=%{public}s", GSErrorStr(ret).c_str()); + addr_ = reinterpret_cast(surfaceBuffer_->GetVirAddr()); + if (surfaceBuffer_->GetUsage() & BUFFER_USAGE_MEM_MMZ_CACHE) { + ret = surfaceBuffer_->InvalidateCache(); + if (ret != GSERROR_OK) { + LOGW("InvalidateCache failed, GSError=%{public}d", ret); + } + } + } + return addr_; +} + +sptr ImageSurfaceBuffer::GetSurfaceBuffer() +{ + return surfaceBuffer_; +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_dfx.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_dfx.cpp new file mode 100644 index 000000000..cdd9baa23 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_dfx.cpp @@ -0,0 +1,254 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "hardware/imagecodec/image_codec.h" +#include "hardware/imagecodec/image_codec_log.h" + +namespace OHOS::ImagePlugin { +using namespace std; + +void ImageCodec::PrintAllBufferInfo() +{ + HLOGI("------------INPUT-----------"); + for (const BufferInfo& info : inputBufferPool_) { + HLOGI("inBufId = %{public}u, owner = %{public}s", info.bufferId, ToString(info.owner)); + } + HLOGI("----------------------------"); + HLOGI("------------OUTPUT----------"); + for (const BufferInfo& info : outputBufferPool_) { + HLOGI("outBufId = %{public}u, owner = %{public}s", info.bufferId, ToString(info.owner)); + } + HLOGI("----------------------------"); +} + +std::array ImageCodec::CountOwner(bool isInput) +{ + std::array arr; + arr.fill(0); + const vector& pool = isInput ? inputBufferPool_ : outputBufferPool_; + for (const BufferInfo &info : pool) { + arr[info.owner]++; + } + return arr; +} + +void ImageCodec::ChangeOwner(BufferInfo& info, BufferOwner newOwner) +{ + if (!debugMode_) { + info.owner = newOwner; + return; + } + BufferOwner oldOwner = info.owner; + const char* oldOwnerStr = ToString(oldOwner); + const char* newOwnerStr = ToString(newOwner); + const char* idStr = info.isInput ? "inBufId" : "outBufId"; + + // calculate hold time + auto now = chrono::steady_clock::now(); + uint64_t holdUs = chrono::duration_cast(now - info.lastOwnerChangeTime).count(); + double holdMs = holdUs / US_TO_MS; + TotalCntAndCost& holdRecord = info.isInput ? inputHoldTimeRecord_[oldOwner][newOwner] : + outputHoldTimeRecord_[oldOwner][newOwner]; + holdRecord.totalCnt++; + holdRecord.totalCostUs += holdUs; + double aveHoldMs = holdRecord.totalCostUs / US_TO_MS / holdRecord.totalCnt; + + // now change owner + info.lastOwnerChangeTime = now; + info.owner = newOwner; + std::array arr = CountOwner(info.isInput); + HLOGI("%{public}s = %{public}u, after hold %{public}.1f ms (%{public}.1f ms), %{public}s -> %{public}s, " + "%{public}u/%{public}u/%{public}u", + idStr, info.bufferId, holdMs, aveHoldMs, oldOwnerStr, newOwnerStr, + arr[OWNED_BY_US], arr[OWNED_BY_USER], arr[OWNED_BY_OMX]); + + if (info.isInput && oldOwner == OWNED_BY_US && newOwner == OWNED_BY_OMX) { + UpdateInputRecord(info, now); + } + if (!info.isInput && oldOwner == OWNED_BY_OMX && newOwner == OWNED_BY_US) { + UpdateOutputRecord(info, now); + } +} + +void ImageCodec::UpdateInputRecord(const BufferInfo& info, std::chrono::time_point now) +{ + if (!info.IsValidFrame()) { + return; + } + inTimeMap_[info.omxBuffer->pts] = now; + if (inTotalCnt_ == 0) { + firstInTime_ = now; + } + inTotalCnt_++; + + uint64_t fromFirstInToNow = chrono::duration_cast(now - firstInTime_).count(); + if (fromFirstInToNow == 0) { + HLOGI("pts = %{public}" PRId64 ", len = %{public}u, flags = 0x%{public}x", + info.omxBuffer->pts, info.omxBuffer->filledLen, info.omxBuffer->flag); + } else { + double inFps = inTotalCnt_ * US_TO_S / fromFirstInToNow; + HLOGI("pts = %{public}" PRId64 ", len = %{public}u, flags = 0x%{public}x, in fps %{public}.2f", + info.omxBuffer->pts, info.omxBuffer->filledLen, info.omxBuffer->flag, inFps); + } +} + +void ImageCodec::UpdateOutputRecord(const BufferInfo& info, std::chrono::time_point now) +{ + if (!info.IsValidFrame()) { + return; + } + auto it = inTimeMap_.find(info.omxBuffer->pts); + if (it == inTimeMap_.end()) { + return; + } + if (outRecord_.totalCnt == 0) { + firstOutTime_ = now; + } + outRecord_.totalCnt++; + + uint64_t fromInToOut = chrono::duration_cast(now - it->second).count(); + inTimeMap_.erase(it); + outRecord_.totalCostUs += fromInToOut; + double oneFrameCostMs = fromInToOut / US_TO_MS; + double averageCostMs = outRecord_.totalCostUs / US_TO_MS / outRecord_.totalCnt; + + uint64_t fromFirstOutToNow = chrono::duration_cast(now - firstOutTime_).count(); + if (fromFirstOutToNow == 0) { + HLOGI("pts = %{public}" PRId64 ", len = %{public}u, flags = 0x%{public}x, " + "cost %{public}.2f ms (%{public}.2f ms)", + info.omxBuffer->pts, info.omxBuffer->filledLen, info.omxBuffer->flag, + oneFrameCostMs, averageCostMs); + } else { + double outFps = outRecord_.totalCnt * US_TO_S / fromFirstOutToNow; + HLOGI("pts = %{public}" PRId64 ", len = %{public}u, flags = 0x%{public}x, " + "cost %{public}.2f ms (%{public}.2f ms), out fps %{public}.2f", + info.omxBuffer->pts, info.omxBuffer->filledLen, info.omxBuffer->flag, + oneFrameCostMs, averageCostMs, outFps); + } +} + +bool ImageCodec::BufferInfo::IsValidFrame() const +{ + if (omxBuffer->flag & OMX_BUFFERFLAG_EOS) { + return false; + } + if (omxBuffer->flag & OMX_BUFFERFLAG_CODECCONFIG) { + return false; + } + if (omxBuffer->filledLen == 0) { + return false; + } + return true; +} + +void ImageCodec::BufferInfo::Dump(const string& prefix, bool dumpMode) const +{ + if (!dumpMode) { + return; + } + if (isInput) { + Dump(prefix + "_Input"); + } else { + Dump(prefix + "_Output"); + } +} + +void ImageCodec::BufferInfo::Dump(const string& prefix) const +{ + if (surfaceBuffer) { + DumpSurfaceBuffer(prefix); + } else { + DumpLinearBuffer(prefix); + } +} + +void ImageCodec::BufferInfo::DumpSurfaceBuffer(const std::string& prefix) const +{ + const char* va = reinterpret_cast(surfaceBuffer->GetVirAddr()); + if (va == nullptr) { + LOGW("surface buffer has null va"); + return; + } + bool eos = (omxBuffer->flag & OMX_BUFFERFLAG_EOS); + if (eos || omxBuffer->filledLen == 0) { + return; + } + int w = surfaceBuffer->GetWidth(); + int h = surfaceBuffer->GetHeight(); + int alignedW = surfaceBuffer->GetStride(); + uint32_t totalSize = surfaceBuffer->GetSize(); + if (w <= 0 || h <= 0 || alignedW <= 0 || w > alignedW) { + LOGW("invalid buffer dimension"); + return; + } + std::optional fmt = TypeConverter::GraphicFmtToFmt( + static_cast(surfaceBuffer->GetFormat())); + if (fmt == nullopt) { + LOGW("invalid fmt=%{public}d", surfaceBuffer->GetFormat()); + return; + } + + char name[128]; + int ret = sprintf_s(name, sizeof(name), "%s/%s_%dx%d(%d)_fmt%s_pts%" PRId64 ".yuv", + DUMP_PATH, prefix.c_str(), w, h, alignedW, fmt->strFmt.c_str(), omxBuffer->pts); + if (ret > 0) { + ofstream ofs(name, ios::binary); + if (ofs.is_open()) { + ofs.write(va, totalSize); + } else { + LOGW("cannot open %{public}s", name); + } + } + // if we unmap here, flush cache will fail +} + +void ImageCodec::BufferInfo::DumpLinearBuffer(const string& prefix) const +{ + if (imgCodecBuffer == nullptr) { + LOGW("invalid imgCodecBuffer"); + return; + } + const char* va = reinterpret_cast(imgCodecBuffer->GetAddr()); + if (va == nullptr) { + LOGW("null va"); + return; + } + bool eos = (omxBuffer->flag & OMX_BUFFERFLAG_EOS); + if (eos || omxBuffer->filledLen == 0) { + return; + } + + char name[128]; + int ret = 0; + if (isInput) { + ret = sprintf_s(name, sizeof(name), "%s/%s.bin", DUMP_PATH, prefix.c_str()); + } else { + ret = sprintf_s(name, sizeof(name), "%s/%s_(%d)_pts%" PRId64 ".yuv", + DUMP_PATH, prefix.c_str(), imgCodecBuffer->GetStride(), omxBuffer->pts); + } + if (ret <= 0) { + LOGW("sprintf_s failed"); + return; + } + std::ios_base::openmode mode = isInput ? (ios::binary | ios::app) : ios::binary; + ofstream ofs(name, mode); + if (ofs.is_open()) { + ofs.write(va, omxBuffer->filledLen); + } else { + LOGW("cannot open %{public}s", name); + } +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_list.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_list.cpp new file mode 100644 index 000000000..1e36770b1 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_codec_list.cpp @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/image_codec_list.h" +#include "hardware/imagecodec/image_codec_log.h" +#include "iservmgr_hdi.h" // drivers/hdf_core/interfaces/inner_api/hdi/ +#include "syspara/parameters.h" // base/startup/init/interfaces/innerkits/include/ + +namespace OHOS::ImagePlugin { +using namespace std; +using namespace OHOS::HDI::ServiceManager::V1_0; +using namespace HdiCodecNamespace; + +static mutex g_heifCodecMtx; +static sptr g_compMgrForHeif; + +class Listener : public ServStatListenerStub { +public: + void OnReceive(const ServiceStatus &status) override + { + if (status.serviceName == "codec_component_manager_service" && status.status == SERVIE_STATUS_STOP) { + LOGW("codec_component_manager_service died"); + lock_guard lk(g_heifCodecMtx); + g_compMgrForHeif = nullptr; + } + } +}; + +static bool IsPassthrough() +{ + static bool usePassthrough = OHOS::system::GetBoolParameter("image.codec.usePassthrough", false); + LOGI("%{public}s mode", usePassthrough ? "passthrough" : "ipc"); + return usePassthrough; +} + +sptr GetManager() +{ + lock_guard lk(g_heifCodecMtx); + if (g_compMgrForHeif) { + return g_compMgrForHeif; + } + LOGI("need to get ICodecComponentManager"); + bool isPassthrough = IsPassthrough(); + if (!isPassthrough) { + sptr serviceMng = IServiceManager::Get(); + if (serviceMng) { + serviceMng->RegisterServiceStatusListener(new Listener(), DEVICE_CLASS_DEFAULT); + } + } + g_compMgrForHeif = ICodecComponentManager::Get(isPassthrough); + return g_compMgrForHeif; +} + +vector GetCapList() +{ + sptr mnger = GetManager(); + if (mnger == nullptr) { + LOGE("failed to create codec component manager"); + return {}; + } + int32_t compCnt = 0; + int32_t ret = mnger->GetComponentNum(compCnt); + if (ret != HDF_SUCCESS || compCnt <= 0) { + LOGE("failed to query component number, ret=%{public}d", ret); + return {}; + } + std::vector capList(compCnt); + ret = mnger->GetComponentCapabilityList(capList, compCnt); + if (ret != HDF_SUCCESS) { + LOGE("failed to query component capability list, ret=%{public}d", ret); + return {}; + } + if (capList.empty()) { + LOGE("GetComponentCapabilityList return empty"); + } else { + LOGI("GetComponentCapabilityList return %{public}zu components", capList.size()); + } + return capList; +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_decoder.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_decoder.cpp new file mode 100644 index 000000000..a317c1770 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/image_decoder.cpp @@ -0,0 +1,405 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/image_decoder.h" +#include "hardware/imagecodec/image_codec_log.h" +#include +#include "codec_omx_ext.h" // drivers/peripheral/codec/interfaces/include/codec_omx_ext.h +#include "OMX_VideoExt.h" // third_party/openmax/api/1.1.2/OMX_VideoExt.h +#include "surface_buffer.h" // foundation/graphic/graphic_surface/interfaces/inner_api/surface/surface_buffer.h + +namespace OHOS::ImagePlugin { +using namespace std; + +ImageDecoder::ImageDecoder() + : ImageCodec(static_cast(CODEC_OMX_VIDEO_CodingHEVC), false) +{} + +int32_t ImageDecoder::OnConfigure(const Format &format) +{ + configFormat_ = make_shared(format); + + (void)format.GetValue(ImageCodecDescriptionKey::ENABLE_HEIF_GRID, enableHeifGrid_); + + UseBufferType useBufferTypes; + InitOMXParamExt(useBufferTypes); + useBufferTypes.portIndex = OMX_DirOutput; + useBufferTypes.bufferType = CODEC_BUFFER_TYPE_HANDLE; + if (!SetParameter(OMX_IndexParamUseBufferType, useBufferTypes)) { + HLOGE("component don't support CODEC_BUFFER_TYPE_HANDLE"); + return IC_ERR_INVALID_VAL; + } + + (void)SetProcessName(format); + (void)SetFrameRateAdaptiveMode(format); + return SetupPort(format); +} + +int32_t ImageDecoder::SetupPort(const Format &format) +{ + uint32_t width; + if (!format.GetValue(ImageCodecDescriptionKey::WIDTH, width) || width <= 0) { + HLOGE("format should contain width"); + return IC_ERR_INVALID_VAL; + } + uint32_t height; + if (!format.GetValue(ImageCodecDescriptionKey::HEIGHT, height) || height <= 0) { + HLOGE("format should contain height"); + return IC_ERR_INVALID_VAL; + } + HLOGI("user set width %{public}u, height %{public}u", width, height); + if (!GetPixelFmtFromUser(format)) { + return IC_ERR_INVALID_VAL; + } + uint32_t inputBufferCnt = 0; + (void)format.GetValue(ImageCodecDescriptionKey::INPUT_BUFFER_COUNT, inputBufferCnt); + uint32_t outputBufferCnt = 0; + (void)format.GetValue(ImageCodecDescriptionKey::OUTPUT_BUFFER_COUNT, outputBufferCnt); + + optional frameRate = GetFrameRateFromUser(format); + if (!frameRate.has_value()) { + HLOGI("user don't set valid frame rate, use default 30.0"); + frameRate = 30.0; // default frame rate 30.0 + } + + PortInfo inputPortInfo {width, height, codingType_, std::nullopt, frameRate.value()}; + int32_t maxInputSize = 0; + (void)format.GetValue(ImageCodecDescriptionKey::MAX_INPUT_SIZE, maxInputSize); + if (maxInputSize > 0) { + inputPortInfo.inputBufSize = static_cast(maxInputSize); + } + if (inputBufferCnt > 0) { + inputPortInfo.bufferCnt = inputBufferCnt; + } + int32_t ret = SetVideoPortInfo(OMX_DirInput, inputPortInfo); + if (ret != IC_ERR_OK) { + return ret; + } + + PortInfo outputPortInfo = {width, height, OMX_VIDEO_CodingUnused, configuredFmt_, frameRate.value()}; + if (outputBufferCnt > 0) { + outputPortInfo.bufferCnt = outputBufferCnt; + } + ret = SetVideoPortInfo(OMX_DirOutput, outputPortInfo); + if (ret != IC_ERR_OK) { + return ret; + } + + return IC_ERR_OK; +} + +bool ImageDecoder::UpdateConfiguredFmt(OMX_COLOR_FORMATTYPE portFmt) +{ + auto graphicFmt = static_cast(portFmt); + if (graphicFmt != configuredFmt_.graphicFmt) { + optional fmt = TypeConverter::GraphicFmtToFmt(graphicFmt); + if (!fmt.has_value()) { + return false; + } + HLOGI("GraphicPixelFormat need update: configured(%{public}s) -> portdefinition(%{public}s)", + configuredFmt_.strFmt.c_str(), fmt->strFmt.c_str()); + configuredFmt_ = fmt.value(); + } + return true; +} + +int32_t ImageDecoder::UpdateInPortFormat() +{ + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOMXParam(def); + def.nPortIndex = OMX_DirInput; + if (!GetParameter(OMX_IndexParamPortDefinition, def)) { + HLOGE("get input port definition failed"); + return IC_ERR_UNKNOWN; + } + PrintPortDefinition(def); + if (inputFormat_ == nullptr) { + inputFormat_ = make_shared(); + } + inputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, def.format.video.nFrameWidth); + inputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, def.format.video.nFrameHeight); + return IC_ERR_OK; +} + +int32_t ImageDecoder::UpdateOutPortFormat() +{ + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOMXParam(def); + def.nPortIndex = OMX_DirOutput; + if (!GetParameter(OMX_IndexParamPortDefinition, def)) { + HLOGE("get output port definition failed"); + return IC_ERR_UNKNOWN; + } + PrintPortDefinition(def); + if (def.nBufferCountActual == 0) { + HLOGE("invalid bufferCount"); + return IC_ERR_UNKNOWN; + } + (void)UpdateConfiguredFmt(def.format.video.eColorFormat); + + uint32_t w = def.format.video.nFrameWidth; + uint32_t h = def.format.video.nFrameHeight; + + // save into member variable + requestCfg_.timeout = 0; + requestCfg_.width = static_cast(w); + requestCfg_.height = static_cast(h); + requestCfg_.strideAlignment = STRIDE_ALIGNMENT; + requestCfg_.format = configuredFmt_.graphicFmt; + requestCfg_.usage = GetProducerUsage(); + UpdateDisplaySizeByCrop(); + + // save into format + if (outputFormat_ == nullptr) { + outputFormat_ = make_shared(); + } + if (!outputFormat_->ContainKey(ImageCodecDescriptionKey::WIDTH)) { + outputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, w); + } + if (!outputFormat_->ContainKey(ImageCodecDescriptionKey::HEIGHT)) { + outputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, h); + } + outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_WIDTH, requestCfg_.width); + outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_HEIGHT, requestCfg_.height); + outputFormat_->SetValue(ImageCodecDescriptionKey::PIXEL_FORMAT, + static_cast(configuredFmt_.graphicFmt)); + return IC_ERR_OK; +} + +void ImageDecoder::UpdateColorAspects() +{ + CodecVideoColorspace param; + InitOMXParamExt(param); + param.portIndex = OMX_DirOutput; + if (!GetParameter(OMX_IndexColorAspects, param, true)) { + return; + } + HLOGI("range:%{public}d, primary:%{public}d, transfer:%{public}d, matrix:%{public}d)", + param.aspects.range, param.aspects.primaries, param.aspects.transfer, param.aspects.matrixCoeffs); + if (outputFormat_) { + outputFormat_->SetValue(ImageCodecDescriptionKey::RANGE_FLAG, param.aspects.range); + outputFormat_->SetValue(ImageCodecDescriptionKey::COLOR_PRIMARIES, param.aspects.primaries); + outputFormat_->SetValue(ImageCodecDescriptionKey::TRANSFER_CHARACTERISTICS, param.aspects.transfer); + outputFormat_->SetValue(ImageCodecDescriptionKey::MATRIX_COEFFICIENTS, param.aspects.matrixCoeffs); + callback_->OnOutputFormatChanged(*(outputFormat_.get())); + } +} + +void ImageDecoder::UpdateDisplaySizeByCrop() +{ + OMX_CONFIG_RECTTYPE rect; + InitOMXParam(rect); + rect.nPortIndex = OMX_DirOutput; + if (!GetParameter(OMX_IndexConfigCommonOutputCrop, rect, true)) { + HLOGW("get crop failed, use default"); + return; + } + if (rect.nLeft < 0 || rect.nTop < 0 || rect.nWidth == 0 || rect.nHeight == 0 || + static_cast(rect.nLeft) + static_cast(rect.nWidth) > requestCfg_.width || + static_cast(rect.nTop) + static_cast(rect.nHeight) > requestCfg_.height) { + HLOGW("wrong crop rect (%{public}d, %{public}d, %{public}u, %{public}u), use default", + rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight); + return; + } + HLOGI("crop rect (%{public}d, %{public}d, %{public}u, %{public}u)", + rect.nLeft, rect.nTop, rect.nWidth, rect.nHeight); + requestCfg_.width = static_cast(rect.nWidth); + requestCfg_.height = static_cast(rect.nHeight); +} + +int32_t ImageDecoder::ReConfigureOutputBufferCnt() +{ + IF_TRUE_RETURN_VAL(enableHeifGrid_, IC_ERR_OK); + OMX_PARAM_PORTDEFINITIONTYPE def; + InitOMXParam(def); + def.nPortIndex = OMX_DirOutput; + IF_TRUE_RETURN_VAL_WITH_MSG(!GetParameter(OMX_IndexParamPortDefinition, def), IC_ERR_UNKNOWN, + "failed to get output port def"); + uint32_t outputBufferCnt = 0; + (void)configFormat_->GetValue(ImageCodecDescriptionKey::OUTPUT_BUFFER_COUNT, outputBufferCnt); + IF_TRUE_RETURN_VAL_WITH_MSG(outputBufferCnt == 0, IC_ERR_UNKNOWN, "failed to get output buffer cnt"); + def.nBufferCountActual = outputBufferCnt; + IF_TRUE_RETURN_VAL_WITH_MSG(!SetParameter(OMX_IndexParamPortDefinition, def), IC_ERR_UNKNOWN, + "failed to set output port def"); + return IC_ERR_OK; +} + +uint64_t ImageDecoder::GetProducerUsage() +{ + uint64_t producerUsage = BUFFER_MODE_REQUEST_USAGE; + + GetBufferHandleUsageParams vendorUsage; + InitOMXParamExt(vendorUsage); + vendorUsage.portIndex = static_cast(OMX_DirOutput); + if (GetParameter(OMX_IndexParamGetBufferHandleUsage, vendorUsage)) { + HLOGI("vendor producer usage = 0x%" PRIx64 "", vendorUsage.usage); + producerUsage |= vendorUsage.usage; + } + HLOGI("decoder producer usage = 0x%" PRIx64 "", producerUsage); + return producerUsage; +} + +uint64_t ImageDecoder::OnGetOutputBufferUsage() +{ + uint64_t usage = GetProducerUsage(); + usage |= BUFFER_USAGE_CPU_WRITE; + return usage; +} + +int32_t ImageDecoder::OnSetOutputBuffer(sptr output) +{ + if (output == nullptr) { + HLOGE("invalid output buffer"); + return IC_ERR_INVALID_VAL; + } + outputBuffer_ = output; + return IC_ERR_OK; +} + +bool ImageDecoder::ReadyToStart() +{ + if (callback_ == nullptr || outputFormat_ == nullptr || inputFormat_ == nullptr) { + HLOGE("callback not set or format is not configured, can't start"); + return false; + } + if (enableHeifGrid_ && outputBuffer_ != nullptr) { + HLOGE("can not set output buffer when heif grid is enabled"); + return false; + } + if (!enableHeifGrid_ && outputBuffer_ == nullptr) { + HLOGE("must set output buffer when heif grid is not enabled"); + return false; + } + return true; +} + +int32_t ImageDecoder::AllocateBuffersOnPort(OMX_DIRTYPE portIndex) +{ + if (portIndex == OMX_DirInput) { + return AllocateHardwareBuffers(portIndex); + } + int32_t ret = AllocateSurfaceBuffers(portIndex, outputBuffer_); + if (ret == IC_ERR_OK) { + UpdateFormatFromSurfaceBuffer(); + } + return ret; +} + +void ImageDecoder::UpdateFormatFromSurfaceBuffer() +{ + if (outputBufferPool_.empty()) { + return; + } + sptr surfaceBuffer = outputBufferPool_.front().surfaceBuffer; + if (surfaceBuffer == nullptr) { + return; + } + outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_WIDTH, surfaceBuffer->GetWidth()); + outputFormat_->SetValue(ImageCodecDescriptionKey::VIDEO_DISPLAY_HEIGHT, surfaceBuffer->GetHeight()); + outputFormat_->SetValue(ImageCodecDescriptionKey::WIDTH, surfaceBuffer->GetStride()); + + OMX_PARAM_PORTDEFINITIONTYPE def; + int32_t ret = GetPortDefinition(OMX_DirOutput, def); + int32_t sliceHeight = static_cast(def.format.video.nSliceHeight); + if (ret == IC_ERR_OK && sliceHeight >= surfaceBuffer->GetHeight()) { + outputFormat_->SetValue(ImageCodecDescriptionKey::HEIGHT, sliceHeight); + } +} + +int32_t ImageDecoder::SubmitAllBuffersOwnedByUs() +{ + HLOGI(">>"); + if (isBufferCirculating_) { + HLOGI("buffer is already circulating, no need to do again"); + return IC_ERR_OK; + } + int32_t ret = SubmitOutputBuffersToOmxNode(); + if (ret != IC_ERR_OK) { + return ret; + } + for (BufferInfo& info : inputBufferPool_) { + if (info.owner == BufferOwner::OWNED_BY_US) { + NotifyUserToFillThisInBuffer(info); + } + } + isBufferCirculating_ = true; + return IC_ERR_OK; +} + +int32_t ImageDecoder::SubmitOutputBuffersToOmxNode() +{ + for (BufferInfo& info : outputBufferPool_) { + switch (info.owner) { + case BufferOwner::OWNED_BY_US: { + int32_t ret = NotifyOmxToFillThisOutBuffer(info); + if (ret != IC_ERR_OK) { + return ret; + } + continue; + } + case BufferOwner::OWNED_BY_OMX: { + continue; + } + default: { + HLOGE("buffer id %{public}u has invalid owner %{public}d", info.bufferId, info.owner); + return IC_ERR_UNKNOWN; + } + } + } + return IC_ERR_OK; +} + +void ImageDecoder::OnOMXEmptyBufferDone(uint32_t bufferId, BufferOperationMode mode) +{ + BufferInfo *info = FindBufferInfoByID(OMX_DirInput, bufferId); + if (info == nullptr) { + HLOGE("unknown buffer id %{public}u", bufferId); + return; + } + if (info->owner != BufferOwner::OWNED_BY_OMX) { + HLOGE("wrong ownership: buffer id=%{public}d, owner=%{public}s", bufferId, ToString(info->owner)); + return; + } + ChangeOwner(*info, BufferOwner::OWNED_BY_US); + + switch (mode) { + case KEEP_BUFFER: + return; + case RESUBMIT_BUFFER: { + if (!inputPortEos_) { + NotifyUserToFillThisInBuffer(*info); + } + return; + } + default: { + HLOGE("SHOULD NEVER BE HERE"); + return; + } + } +} + +void ImageDecoder::EraseBufferFromPool(OMX_DIRTYPE portIndex, size_t i) +{ + vector& pool = (portIndex == OMX_DirInput) ? inputBufferPool_ : outputBufferPool_; + if (i >= pool.size()) { + return; + } + BufferInfo& info = pool[i]; + FreeOmxBuffer(portIndex, info); + pool.erase(pool.begin() + i); +} + + +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/msg_handle_loop.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/msg_handle_loop.cpp new file mode 100644 index 000000000..311cc8e12 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/msg_handle_loop.cpp @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/msg_handle_loop.h" +#include +#include +#include "qos.h" +#include "hardware/imagecodec/image_codec_log.h" + +namespace OHOS::ImagePlugin { +using namespace std; + +MsgHandleLoop::MsgHandleLoop() +{ + m_thread = thread(&MsgHandleLoop::MainLoop, this); +} + +MsgHandleLoop::~MsgHandleLoop() +{ + Stop(); +} + +void MsgHandleLoop::Stop() +{ + { + lock_guard lock(m_mtx); + m_threadNeedStop = true; + m_threadCond.notify_all(); + } + + if (m_thread.joinable()) { + m_thread.join(); + } +} + +void MsgHandleLoop::SendAsyncMsg(MsgType type, const ParamSP &msg, uint32_t delayUs) +{ + lock_guard lock(m_mtx); + TimeUs nowUs = GetNowUs(); + TimeUs msgProcessTime = (delayUs > INT64_MAX - nowUs) ? INT64_MAX : (nowUs + delayUs); + if (m_msgQueue.find(msgProcessTime) != m_msgQueue.end()) { + LOGW("DUPLICATIVE MSG TIMESTAMP!!!"); + msgProcessTime++; + } + m_msgQueue[msgProcessTime] = MsgInfo {type, ASYNC_MSG_ID, msg}; + m_threadCond.notify_all(); +} + +bool MsgHandleLoop::SendSyncMsg(MsgType type, const ParamSP &msg, ParamSP &reply, uint32_t waitMs) +{ + MsgId id = GenerateMsgId(); + { + lock_guard lock(m_mtx); + TimeUs time = GetNowUs(); + if (m_msgQueue.find(time) != m_msgQueue.end()) { + LOGW("DUPLICATIVE MSG TIMESTAMP!!!"); + time++; + } + m_msgQueue[time] = MsgInfo {type, id, msg}; + m_threadCond.notify_all(); + } + + unique_lock lock(m_replyMtx); + const auto pred = [this, id]() { + return m_replies.find(id) != m_replies.end(); + }; + if (waitMs == 0) { + m_replyCond.wait(lock, pred); + } else { + if (!m_replyCond.wait_for(lock, chrono::milliseconds(waitMs), pred)) { + LOGE("type=%{public}u wait reply timeout", type); + return false; + } + } + reply = m_replies[id]; + m_replies.erase(id); + return true; +} + +void MsgHandleLoop::PostReply(MsgId id, const ParamSP &reply) +{ + if (id == ASYNC_MSG_ID) { + return; + } + lock_guard lock(m_replyMtx); + m_replies[id] = reply; + m_replyCond.notify_all(); +} + +MsgId MsgHandleLoop::GenerateMsgId() +{ + lock_guard lock(m_mtx); + m_lastMsgId++; + if (m_lastMsgId == ASYNC_MSG_ID) { + m_lastMsgId++; + } + return m_lastMsgId; +} + +void MsgHandleLoop::MainLoop() +{ + LOGI("increase thread priority"); + pthread_setname_np(pthread_self(), "OS_ImageCodecLoop"); + OHOS::QOS::SetThreadQos(OHOS::QOS::QosLevel::QOS_USER_INTERACTIVE); + while (true) { + MsgInfo info; + { + unique_lock lock(m_mtx); + m_threadCond.wait(lock, [this] { + return m_threadNeedStop || !m_msgQueue.empty(); + }); + if (m_threadNeedStop) { + LOGI("stopped, remain %{public}zu msg unprocessed", m_msgQueue.size()); + break; + } + TimeUs processUs = m_msgQueue.begin()->first; + TimeUs nowUs = GetNowUs(); + if (processUs > nowUs) { + m_threadCond.wait_for(lock, chrono::microseconds(processUs - nowUs)); + continue; + } + info = m_msgQueue.begin()->second; + m_msgQueue.erase(m_msgQueue.begin()); + } + OnMsgReceived(info); + } +} + +MsgHandleLoop::TimeUs MsgHandleLoop::GetNowUs() +{ + auto now = chrono::steady_clock::now(); + return chrono::duration_cast(now.time_since_epoch()).count(); +} +} // namespace OHOS::ImagePlugin diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/param_bundle.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/param_bundle.cpp new file mode 100644 index 000000000..671aba8ca --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/param_bundle.cpp @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/param_bundle.h" +#include + +namespace OHOS::ImagePlugin { +using namespace std; + +ParamSP ParamBundle::Create() +{ + return {new(nothrow)ParamBundle(), + [](ParamBundle *param) { + delete param; + }}; +} +} // namespace OHOS::ImagePlugin diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/state_machine.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/state_machine.cpp new file mode 100644 index 000000000..969ca85d9 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/state_machine.cpp @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/state_machine.h" +#include "hardware/imagecodec/image_codec_log.h" + +namespace OHOS::ImagePlugin { +void StateMachine::ChangeStateTo(const std::shared_ptr &targetState) +{ + if (currState_ == targetState) { + LOGI("already %{public}s", currState_->stateName_.c_str()); + return; + } + std::shared_ptr lastState = currState_; + currState_ = targetState; + if (lastState == nullptr) { + LOGI("change to %{public}s", currState_->stateName_.c_str()); + } else { + LOGI("%{public}s -> %{public}s", lastState->stateName_.c_str(), currState_->stateName_.c_str()); + lastState->OnStateExited(); + } + currState_->OnStateEntered(); +} + +void StateMachine::OnMsgReceived(const MsgInfo &info) +{ + currState_->OnMsgReceived(info); +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/type_converter.cpp b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/type_converter.cpp new file mode 100644 index 000000000..de5ee7414 --- /dev/null +++ b/plugins/common/libs/image/libextplugin/src/hardware/imagecodec/type_converter.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "hardware/imagecodec/type_converter.h" +#include "hardware/imagecodec/image_codec_log.h" + +namespace OHOS::ImagePlugin { +using namespace std; + +vector g_pixelFmtTable = { + {GRAPHIC_PIXEL_FMT_YCBCR_420_SP, "NV12"}, + {GRAPHIC_PIXEL_FMT_YCRCB_420_SP, "NV21"}, + {GRAPHIC_PIXEL_FMT_YCBCR_P010, "NV12_10bit"}, + {GRAPHIC_PIXEL_FMT_YCRCB_P010, "NV21_10bit"}, +}; + +std::optional TypeConverter::GraphicFmtToFmt(GraphicPixelFormat format) +{ + auto it = find_if(g_pixelFmtTable.begin(), g_pixelFmtTable.end(), [format](const PixelFmt& p) { + return p.graphicFmt == format; + }); + if (it != g_pixelFmtTable.end()) { + return *it; + } + LOGW("unknown GraphicPixelFormat %{public}d", format); + return nullopt; +} +} // namespace OHOS::ImagePlugin \ No newline at end of file diff --git a/plugins/common/libs/image/libextplugin/src/heif_impl/HeifDecoderImpl.cpp b/plugins/common/libs/image/libextplugin/src/heif_impl/HeifDecoderImpl.cpp index ceae02eda..a6ee6f181 100644 --- a/plugins/common/libs/image/libextplugin/src/heif_impl/HeifDecoderImpl.cpp +++ b/plugins/common/libs/image/libextplugin/src/heif_impl/HeifDecoderImpl.cpp @@ -78,8 +78,8 @@ static bool FillFrameInfoForPixelConvert(AVFrame *frame, PixelFormatConvertParam } const OH_NativeBuffer_Plane &planeY = param.planesInfo->planes[0]; const OH_NativeBuffer_Plane &planeUV = param.planesInfo->planes[param.format == AV_PIX_FMT_NV21 ? 2 : 1]; - IMAGE_LOGI("planeY offset: %{public}ld, columnStride: %{public}d, rowStride: %{public}d," - " planeUV offset: %{public}ld, columnStride: %{public}d, rowStride: %{public}d", + IMAGE_LOGI("planeY offset: %{public}llu, columnStride: %{public}u, rowStride: %{public}u," + " planeUV offset: %{public}llu, columnStride: %{public}u, rowStride: %{public}u", planeY.offset, planeY.columnStride, planeY.rowStride, planeUV.offset, planeUV.columnStride, planeUV.rowStride); frame->data[0] = param.data + planeY.offset; diff --git a/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I0 b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I0 new file mode 100644 index 000000000..ce8433537 Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I0 differ diff --git a/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I1 b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I1 new file mode 100644 index 000000000..9b976116b Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I1 differ diff --git a/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I2 b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I2 new file mode 100644 index 000000000..9609a0867 Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I2 differ diff --git a/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I3 b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I3 new file mode 100644 index 000000000..67b0d2c76 Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_I3 differ diff --git a/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_xps b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_xps new file mode 100644 index 000000000..062eee52f Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_grid_512x512_2x2/19700301_225916_595_hevc_xps differ diff --git a/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_I0 b/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_I0 new file mode 100644 index 000000000..010712c13 Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_I0 differ diff --git a/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_xps b/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_xps new file mode 100644 index 000000000..2afbea167 Binary files /dev/null and b/test/resource/image/images/heif_test/1024x1024_nogrid/19700301_230238_059_hevc_xps differ diff --git a/test/resource/image/ohos_test.xml b/test/resource/image/ohos_test.xml index bee0e6d69..6db222f32 100644 --- a/test/resource/image/ohos_test.xml +++ b/test/resource/image/ohos_test.xml @@ -100,6 +100,20 @@