openharmony/multimedia_video_processing_engine
1
0
Fork 0
mirror of https://github.com/openharmony/multimedia_video_processing_engine.git synced 2026-07-01 06:41:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

merge master into master

Browse Source 
视频动态元数据生成Buffer轮转优化

Created-by: weixin_44700052
Commit-by: weixin_44700052
Merged-by: openharmony_ci
Description: ### 一、内容说明（相关的Issue）
[视频动态元数据生成Buffer轮转优化](https://gitcode.com/openharmony/multimedia_video_processing_engine/issues/20)


### 二、建议测试周期和提测地址  
不涉及  

### 三、变更内容
  * 3.1 关联PR列表
无
  * 3.2 数据库和部署说明  
不涉及

  * 3.4 其他技术优化内容（做了什么，变更了什么）
优化动态元数据生成时的buffer轮转，去除其中的buffer拷贝动作。


  * 3.5 废弃通知（什么字段、方法弃用？）
无


  * 3.6  后向不兼容变更（是否有无法向后兼容的变更？）
无

  
### 四、研发自测点（自测哪些？冒烟用例全部自测？）
  自测动态元数据生成平均时延5.68ms，watt平均功耗3.83mA，unitest、moduletest测试通过。
  自测测试结论：时延功耗满足要求。


### 五、测试关注点（需要提醒QA重点关注的、可能会忽略的地方）
  检查点：

| 需求名称 | 是否影响xx公共模块 | 是否需要xx功能 | 需求升级是否依赖其他子产品 |
|------|------------|----------|---------------|
| xxx  | 否          | 需要       | 不需要           |
|      |            |          |               |

  接口测试：不涉及

  性能测试：涉及，动态元数据生成平均时延小于等于6ms，平均功耗小于等于15mA

  并发测试：不涉及

  其他：不涉及



See merge request: openharmony/multimedia_video_processing_engine!29
This commit is contained in:
openharmony_ci
2025-09-26 11:30:36 +08:00
parent 8560cd46ee eb85359835
commit 3d52bdabd7
2 changed files with 72 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
framework/algorithm/metadata_generator_video/include/metadata_generator_video_impl.h
+8 -6
View File
@@ -67,11 +67,11 @@ private:
};
void InitBuffers();
bool WaitProcessing();
bool AcquireInputOutputBuffers(
std::shared_ptr<SurfaceBufferWrapper> &inputBuffer, std::shared_ptr<SurfaceBufferWrapper> &outputBuffer);
bool AcquireInputBuffers(std::shared_ptr<SurfaceBufferWrapper> &inputBuffer);
void DoTask();
void OnTriggered();
void Process(std::shared_ptr<SurfaceBufferWrapper> inputBuffer, std::shared_ptr<SurfaceBufferWrapper> outputBuffer);
void CheckRequestCfg(sptr<SurfaceBuffer> inputBuffer);
void Process(std::shared_ptr<SurfaceBufferWrapper> inputBuffer);
int32_t AttachToNewSurface(sptr<Surface> newSurface);
int32_t SetOutputSurfaceConfig(sptr<Surface> surface);
int32_t SetOutputSurfaceRunning(sptr<Surface> newSurface);
@@ -106,11 +106,13 @@ private:
std::mutex renderQueMutex_; // outputsruface buffer
std::mutex surfaceChangeMutex_;
std::mutex surfaceChangeMutex2_;
std::mutex outputQueMutex_;
sptr<Surface> inputSurface_{nullptr};
sptr<Surface> outputSurface_{nullptr};
static constexpr size_t MAX_BUFFER_CNT{5};
uint32_t outBufferCnt_{MAX_BUFFER_CNT};
uint32_t inBufferCnt_{MAX_BUFFER_CNT};
static constexpr size_t MAX_INPUT_BUFFER_CNT{5};
static constexpr size_t MAX_OUTPUT_BUFFER_CNT{12};
uint32_t outBufferCnt_{MAX_OUTPUT_BUFFER_CNT};
uint32_t inBufferCnt_{MAX_INPUT_BUFFER_CNT};
static constexpr size_t MAX_SURFACE_SEQUENCE{std::numeric_limits<uint32_t>::max()};
uint32_t lastSurfaceSequence_{MAX_SURFACE_SEQUENCE};
BufferRequestConfig requestCfg_{};
framework/algorithm/metadata_generator_video/metadata_generator_video_impl.cpp
+64 -70
View File
@@ -122,9 +122,6 @@ int32_t MetadataGeneratorVideoImpl::AttachToNewSurface(sptr<Surface> newSurface)
int32_t MetadataGeneratorVideoImpl::GetReleaseOutBuffer()
{
std::lock_guard<std::mutex> mapLock(renderQueMutex_);
for (RenderBufferAvilMapType::iterator it = renderBufferMapBak_.begin(); it != renderBufferMapBak_.end(); ++it) {
outputBufferAvilQue_.push(it->second);
}
renderBufferMapBak_.clear();
return VPE_ALGO_ERR_OK;
}
@@ -145,6 +142,7 @@ int32_t MetadataGeneratorVideoImpl::SetOutputSurfaceConfig(sptr<Surface> surface
int32_t MetadataGeneratorVideoImpl::SetOutputSurfaceRunning(sptr<Surface> newSurface)
{
std::lock_guard<std::mutex> outputLock(outputQueMutex_);
std::lock_guard<std::mutex> lockSurface(surfaceChangeMutex_);
std::lock_guard<std::mutex> lockSurface2(surfaceChangeMutex2_);
uint64_t oldId = outputSurface_->GetUniqueId();
@@ -164,6 +162,7 @@ int32_t MetadataGeneratorVideoImpl::SetOutputSurfaceRunning(sptr<Surface> newSur
newSurface->SetQueueSize(outBufferCnt_);
newSurface->Connect();
newSurface->CleanCache();
newSurface->SetDefaultUsage(outputSurface_->GetDefaultUsage());
GetReleaseOutBuffer();
int32_t ret = AttachToNewSurface(newSurface);
if (ret != VPE_ALGO_ERR_OK) {
@@ -221,8 +220,6 @@ sptr<Surface> MetadataGeneratorVideoImpl::CreateInputSurface()
sptr<IBufferProducer> producer = inputSurface_->GetProducer();
sptr<Surface> producerSurface = Surface::CreateSurfaceAsProducer(producer);
CHECK_AND_RETURN_RET_LOG(producerSurface != nullptr, nullptr, "CreateSurfaceAsProducer fail");
producerSurface->SetDefaultUsage(BUFFER_USAGE_CPU_READ | BUFFER_USAGE_CPU_WRITE | BUFFER_USAGE_HW_RENDER |
BUFFER_USAGE_MEM_DMA);
inputSurface_->SetQueueSize(inBufferCnt_);
state_ = VPEAlgoState::CONFIGURING;
@@ -294,16 +291,7 @@ void MetadataGeneratorVideoImpl::InitBuffers()
flushCfg_.damage.y = 0;
flushCfg_.damage.w = requestCfg_.width;
flushCfg_.damage.h = requestCfg_.height;
for (uint32_t i = 0; i < outBufferCnt_; ++i) {
std::shared_ptr<SurfaceBufferWrapper> buffer = std::make_shared<SurfaceBufferWrapper>();
GSError err = outputSurface_->RequestBuffer(buffer->memory, buffer->fence, requestCfg_);
if (err != GSERROR_OK || buffer->memory == nullptr) {
VPE_LOGW("RequestBuffer %{public}u failed, GSError=%{public}d", i, err);
continue;
}
outputBufferAvilQue_.push(buffer);
outputBufferAvilQueBak_.insert(std::make_pair(buffer->memory->GetSeqNum(), buffer));
}
outputSurface_->CleanCache(true);
}
int32_t MetadataGeneratorVideoImpl::Start()
@@ -405,53 +393,55 @@ int32_t MetadataGeneratorVideoImpl::Flush()
}
std::lock_guard<std::mutex> mapLock(renderQueMutex_);
for (auto &[id, buffer] : renderBufferAvilMap_) {
VPE_LOGD("Reclaim buffer %{public}" PRIu64, id);
outputBufferAvilQue_.push(buffer);
}
renderBufferAvilMap_.clear();
state_ = VPEAlgoState::FLUSHED;
return VPE_ALGO_ERR_OK;
}
void MetadataGeneratorVideoImpl::Process(std::shared_ptr<SurfaceBufferWrapper> inputBuffer,
std::shared_ptr<SurfaceBufferWrapper> outputBuffer)
void MetadataGeneratorVideoImpl::CheckRequestCfg(sptr<SurfaceBuffer> inputBuffer)
{
if (requestCfg_.width != inputBuffer->GetWidth() || requestCfg_.height != inputBuffer->GetHeight() ||
requestCfg_.format != inputBuffer->GetFormat() || requestCfg_.usage != inputBuffer->GetUsage()) {
requestCfg_.width = inputBuffer->GetWidth();
requestCfg_.height = inputBuffer->GetHeight();
requestCfg_.format = inputBuffer->GetFormat();
requestCfg_.usage = inputBuffer->GetUsage();
outputSurface_->CleanCache(true);
outputSurface_->SetDefaultUsage(requestCfg_.usage);
std::lock_guard<std::mutex> lock(outputQueMutex_);
outputBufferAvilQueBak_.clear();
}
}
void MetadataGeneratorVideoImpl::Process(std::shared_ptr<SurfaceBufferWrapper> inputBuffer)
{
VPETrace videoTrace("MetadataGeneratorVideoImpl::Process");
int32_t ret = VPE_ALGO_ERR_EXTENSION_PROCESS_FAILED;
outputBuffer->timestamp = inputBuffer->timestamp;
sptr<SurfaceBuffer> surfaceInputBuffer = inputBuffer->memory;
sptr<SurfaceBuffer> surfaceOutputBuffer = outputBuffer->memory;
surfaceInputBuffer->InvalidateCache();
bool copyRet = AlgorithmUtils::CopySurfaceBufferToSurfaceBuffer(surfaceInputBuffer, surfaceOutputBuffer);
if (!copyRet) {
requestCfg_.width = surfaceInputBuffer->GetWidth();
requestCfg_.height = surfaceInputBuffer->GetHeight();
requestCfg_.format = surfaceInputBuffer->GetFormat();
surfaceOutputBuffer->EraseMetadataKey(ATTRKEY_COLORSPACE_INFO);
surfaceOutputBuffer->EraseMetadataKey(ATTRKEY_HDR_METADATA_TYPE);
if (surfaceOutputBuffer->Alloc(requestCfg_) == GSERROR_OK) {
copyRet = AlgorithmUtils::CopySurfaceBufferToSurfaceBuffer(surfaceInputBuffer, surfaceOutputBuffer);
}
CheckRequestCfg(surfaceInputBuffer);
std::unique_lock<std::mutex> outputLock(outputQueMutex_);
auto it = outputBufferAvilQueBak_.find(surfaceInputBuffer->GetSeqNum());
if (it == outputBufferAvilQueBak_.end()) {
ret = outputSurface_->AttachBufferToQueue(surfaceInputBuffer);
CHECK_AND_RETURN_LOG(ret == GSERROR_OK, "AttachBufferToQueue failed %{public}d", ret);
outputBufferAvilQueBak_.emplace(surfaceInputBuffer->GetSeqNum(), inputBuffer);
}
if (copyRet) {
outputLock.unlock();
{
VPETrace cscTrace("MetadataGeneratorVideoImpl::csc_->Process");
ret = csc_->Process(surfaceOutputBuffer);
ret = csc_->Process(surfaceInputBuffer);
}
if (ret != 0 && cb_) {
cb_->OnError(ret);
inputSurface_->ReleaseBuffer(surfaceInputBuffer, -1);
}
inputSurface_->ReleaseBuffer(surfaceInputBuffer, -1);
if (!ret) {
std::unique_lock<std::mutex> lockOnBq(renderQueMutex_);
renderBufferAvilMap_.emplace(outputBuffer->memory->GetSeqNum(), outputBuffer);
} else {
std::lock_guard<std::mutex> renderLock(renderQueMutex_);
outputBufferAvilQue_.push(outputBuffer);
}
if (!ret && cb_) {
cb_->OnOutputBufferAvailable(surfaceOutputBuffer->GetSeqNum(), outputBuffer->bufferFlag);
renderBufferAvilMap_.emplace(surfaceInputBuffer->GetSeqNum(), inputBuffer);
lockOnBq.unlock();
if (cb_) {
cb_->OnOutputBufferAvailable(surfaceInputBuffer->GetSeqNum(), inputBuffer->bufferFlag);
}
}
}
@@ -470,35 +460,30 @@ bool MetadataGeneratorVideoImpl::WaitProcessing()
InitBuffers();
initBuffer_.store(false);
}
return ((inputBufferAvilQue_.size() > 0 && outputBufferAvilQue_.size() > 0) || !isRunning_.load());
return ((inputBufferAvilQue_.size() > 0) || !isRunning_.load());
});
}
return true;
}
bool MetadataGeneratorVideoImpl::AcquireInputOutputBuffers(std::shared_ptr<SurfaceBufferWrapper>& inputBuffer,
std::shared_ptr<SurfaceBufferWrapper>& outputBuffer)
bool MetadataGeneratorVideoImpl::AcquireInputBuffers(std::shared_ptr<SurfaceBufferWrapper>& inputBuffer)
{
std::lock_guard<std::mutex> lockOnBq(onBqMutex_);
std::lock_guard<std::mutex> mapLock(renderQueMutex_);
if (inputBufferAvilQue_.size() == 0 || outputBufferAvilQue_.size() == 0) {
if (inputBufferAvilQue_.size() == 0) {
if (state_ == VPEAlgoState::STOPPED) {
cb_->OnState(static_cast<int32_t>(state_.load()));
}
return false;
}
inputBuffer = inputBufferAvilQue_.front();
outputBuffer = outputBufferAvilQue_.front();
inputBufferAvilQue_.pop();
outputBufferAvilQue_.pop();
return inputBuffer && outputBuffer;
return inputBuffer != nullptr;
}
void MetadataGeneratorVideoImpl::DoTask()
{
std::shared_ptr<SurfaceBufferWrapper> inputBuffer = nullptr;
std::shared_ptr<SurfaceBufferWrapper> outputBuffer = nullptr;
while (true) {
std::lock_guard<std::mutex> lockTask(mtxTaskDone_);
if (!isRunning_.load()) {
@@ -506,20 +491,27 @@ void MetadataGeneratorVideoImpl::DoTask()
}
isProcessing_.store(true);
if (!AcquireInputOutputBuffers(inputBuffer, outputBuffer)) {
if (!AcquireInputBuffers(inputBuffer) || inputBuffer->memory == nullptr) {
break;
}
if (inputBuffer->bufferFlag == MDG_BUFFER_FLAG_EOS) {
std::unique_lock<std::mutex> outputLock(outputQueMutex_);
auto it = outputBufferAvilQueBak_.find(inputBuffer->memory->GetSeqNum());
if (it == outputBufferAvilQueBak_.end()) {
int32_t ret = outputSurface_->AttachBufferToQueue(inputBuffer->memory);
CHECK_AND_RETURN_LOG(ret == GSERROR_OK, "AttachBufferToQueue failed %{public}d", ret);
}
outputLock.unlock();
{
std::unique_lock<std::mutex> lockOnBq(renderQueMutex_);
renderBufferAvilMap_.emplace(outputBuffer->memory->GetSeqNum(), outputBuffer);
renderBufferAvilMap_.emplace(inputBuffer->memory->GetSeqNum(), inputBuffer);
}
if (cb_) {
cb_->OnOutputBufferAvailable(outputBuffer->memory->GetSeqNum(), MDG_BUFFER_FLAG_EOS);
cb_->OnOutputBufferAvailable(inputBuffer->memory->GetSeqNum(), MDG_BUFFER_FLAG_EOS);
}
break;
}
Process(inputBuffer, outputBuffer);
Process(inputBuffer);
}
isProcessing_.store(false);
cvTaskDone_.notify_all();
@@ -565,8 +557,7 @@ int32_t MetadataGeneratorVideoImpl::ReleaseOutputBuffer(uint32_t index, bool ren
std::lock_guard<std::mutex> renderLock(renderQueMutex_);
renderBufferMapBak_.emplace(buffer->memory->GetSeqNum(), buffer);
} else {
std::lock_guard<std::mutex> renderLock(renderQueMutex_);
outputBufferAvilQue_.push(buffer);
inputSurface_->ReleaseBuffer(buffer->memory, -1);
}
return VPE_ALGO_ERR_OK;
}
@@ -606,15 +597,16 @@ GSError MetadataGeneratorVideoImpl::OnProducerBufferReleased()
lockSurface.unlock();
outputBufferAvilQue_.push(buf);
auto bufSeqNum = buf->memory->GetSeqNum();
lastSurfaceSequence_ = bufSeqNum;
renderBufferMapBak_.erase(bufSeqNum);
auto it = outputBufferAvilQueBak_.find(bufSeqNum);
if (it == outputBufferAvilQueBak_.end()) {
outputBufferAvilQueBak_.insert(std::make_pair(bufSeqNum, buf));
auto firstSeqNum = renderBufferMapBak_.begin();
if (firstSeqNum != renderBufferMapBak_.end()) {
outputBufferAvilQueBak_.erase(firstSeqNum->first);
renderBufferMapBak_.erase(firstSeqNum->first);
auto it = renderBufferMapBak_.find(bufSeqNum);
if (it != renderBufferMapBak_.end()) {
lastSurfaceSequence_ = bufSeqNum;
renderBufferMapBak_.erase(bufSeqNum);
inputSurface_->ReleaseBuffer(buf->memory, -1);
} else {
std::lock_guard<std::mutex> lock(outputQueMutex_);
auto it2 = outputBufferAvilQueBak_.find(buf->memory->GetSeqNum());
if (it2 == outputBufferAvilQueBak_.end()) {
outputSurface_->DetachBufferFromQueue(buf->memory);
}
}
}
@@ -646,15 +638,17 @@ GSError MetadataGeneratorVideoImpl::OnConsumerBufferAvailable()
constexpr uint32_t waitForEver = -1; // wait fence -1
if (buffer->fence != nullptr) {
(void)buffer->fence->Wait(waitForEver);
buffer->memory->InvalidateCache();
}
inputBufferAvilQue_.push(buffer);
if (!getUsage_) {
requestCfg_.usage = (buffer->memory->GetUsage() | requestCfg_.usage);
requestCfg_.usage = buffer->memory->GetUsage();
getUsage_ = true;
requestCfg_.width = buffer->memory->GetWidth();
requestCfg_.height = buffer->memory->GetHeight();
requestCfg_.format = buffer->memory->GetFormat();
outputSurface_->SetDefaultUsage(requestCfg_.usage);
initBuffer_.store(true);
}