!3198 记录关键时间戳

Merge pull request !3198 from FileStream/master
2024-11-27 00:50:36 +00:00 · 2024-08-31 15:29:00 +00:00 · 2024-08-31 15:29:00 +00:00 · 9fe88c4348
commit 9fe88c4348
parent 63e14751ca 550a44d597
5 changed files with 150 additions and 3 deletions
--- a/interfaces/inner_api/native/surface_encoder_filter.h
+++ b/interfaces/inner_api/native/surface_encoder_filter.h
@ -64,6 +64,7 @@ public:
    void OnUnlinkedResult(std::shared_ptr<Meta> &meta);
    void SetCallingInfo(int32_t appUid, int32_t appPid, const std::string &bundleName, uint64_t instanceId);
    void OnError(MediaAVCodec::AVCodecErrorType errorType, int32_t errorCode);
+    void OnReportKeyFramePts(std::string KeyFramePts);

 protected:
    Status OnLinked(StreamType inType, const std::shared_ptr<Meta> &meta,
--- a/services/media_engine/filters/surface_encoder_adapter.cpp
+++ b/services/media_engine/filters/surface_encoder_adapter.cpp
@ -263,6 +263,14 @@ Status SurfaceEncoderAdapter::SetEncoderAdapterCallback(
    }
 }

+Status SurfaceEncoderAdapter::SetEncoderAdapterKeyFramePtsCallback(
+    const std::shared_ptr<EncoderAdapterKeyFramePtsCallback> &encoderAdapterKeyFramePtsCallback)
+{
+    MEDIA_LOG_I("SetEncoderAdapterKeyFramePtsCallback");
+    encoderAdapterKeyFramePtsCallback_ = encoderAdapterKeyFramePtsCallback;
+    return Status::OK;
+}
+
 Status SurfaceEncoderAdapter::SetInputSurface(sptr<Surface> surface)
 {
    MEDIA_LOG_I("GetInputSurface");
@ -307,6 +315,7 @@ Status SurfaceEncoderAdapter::Start()
    }
    ret = codecServer_->Start();
    isStart_ = true;
+    isStartKeyFramePts_= true;
    if (ret == 0) {
        return Status::OK;
    } else {
@ -320,11 +329,14 @@ Status SurfaceEncoderAdapter::Stop()
    MEDIA_LOG_I("Stop");
    MediaAVCodec::AVCodecTrace trace("SurfaceEncoderAdapter::Stop");
    GetCurrentTime(stopTime_);
+    isStopKeyFramePts_ = true;
+    isStartKeyFramePts_ = false;
    MEDIA_LOG_I("Stop time: " PUBLIC_LOG_D64, stopTime_);

    if (isStart_ && !isTransCoderMode) {
        std::unique_lock<std::mutex> lock(stopMutex_);
        stopCondition_.wait_for(lock, std::chrono::milliseconds(TIME_OUT_MS));
+        AddStopPts();
    }
    if (releaseBufferTask_) {
        isThreadExit_ = true;
@ -361,6 +373,8 @@ Status SurfaceEncoderAdapter::Pause()
        (pauseResumeQueue_.back().second == StateCode::RESUME && pauseResumeQueue_.back().first <= pauseTime)) {
        pauseResumeQueue_.push_back({pauseTime, StateCode::PAUSE});
        pauseResumeQueue_.push_back({std::numeric_limits<int64_t>::max(), StateCode::RESUME});
+        pauseResumePts_.push_back({pauseTime, StateCode::PAUSE});
+        pauseResumePts_.push_back({std::numeric_limits<int64_t>::max(), StateCode::RESUME});
    }
    return Status::OK;
 }
@ -383,6 +397,7 @@ Status SurfaceEncoderAdapter::Resume()
    }
    if (pauseResumeQueue_.back().second == StateCode::RESUME) {
        pauseResumeQueue_.back().first = std::min(resumeTime, pauseResumeQueue_.back().first);
+        pauseResumePts_.back().first = std::min(resumeTime, pauseResumePts_.back().first);
    }
    return Status::OK;
 }
@ -415,8 +430,10 @@ Status SurfaceEncoderAdapter::Reset()
    stopTime_ = -1;
    totalPauseTime_ = 0;
    isStart_ = false;
+    isStartKeyFramePts_ = false;
    mappingTimeQueue_.clear();
    pauseResumeQueue_.clear();
+    pauseResumePts_.clear();
    if (ret == 0) {
        return Status::OK;
    } else {
@ -565,6 +582,11 @@ void SurfaceEncoderAdapter::OnOutputBufferAvailable(uint32_t index, std::shared_
        if (startBufferTime_ == -1) {
            startBufferTime_ = buffer->pts_;
        }
+        // cache recent 2 pts
+        preKeyFramePts_ = currentKeyFramePts_;
+        currentKeyFramePts_ = buffer->pts_;
+        AddStartPts(buffer->pts_);
+        AddPauseResumePts(buffer->pts_);
    } else {
        MEDIA_LOG_D("OnOutputBufferAvailable buffer->flag_" PUBLIC_LOG_U32, buffer->flag_);
        mappingTime = startBufferTime_ + buffer->pts_;
@ -603,7 +625,7 @@ void SurfaceEncoderAdapter::ReleaseBuffer()
        {
            std::unique_lock<std::mutex> lock(releaseBufferMutex_);
            releaseBufferCondition_.wait(lock, [this] {
-                return isThreadExit_||!indexs_.empty();
+                return isThreadExit_ || !indexs_.empty();
            });
            indexs = indexs_;
            indexs_.clear();
@ -741,5 +763,63 @@ void SurfaceEncoderAdapter::GetCurrentTime(int64_t &currentTime)
    clock_gettime(CLOCK_MONOTONIC, &timestamp);
    currentTime = static_cast<int64_t>(timestamp.tv_sec) * SEC_TO_NS + static_cast<int64_t>(timestamp.tv_nsec);
 }
+
+void SurfaceEncoderAdapter::AddStartPts(int64_t currentPts)
+{
+    // start time
+    if (isStartKeyFramePts_) {
+        keyFramePts_ += std::to_string(currentPts) + ",";
+        isStartKeyFramePts_ = false;
+        MEDIA_LOG_I("AddStartPts success %{public}s end", keyFramePts_.c_str());
+    }
+}
+
+void SurfaceEncoderAdapter::AddStopPts()
+{
+    // stop time
+    MEDIA_LOG_D("AddStopPts enter");
+    if (isStopKeyFramePts_) {
+        if (currentKeyFramePts_ > stopTime_) {
+            keyFramePts_ += std::to_string(preKeyFramePts_);
+            MEDIA_LOG_I("AddStopPts preKeyFramePts_ %{public}s end", keyFramePts_.c_str());
+        } else {
+            keyFramePts_ += std::to_string(currentKeyFramePts_);
+            MEDIA_LOG_I("AddStopPts currentKeyFramePts_ %{public}s end", keyFramePts_.c_str());
+        }
+        isStopKeyFramePts_ = false;
+        encoderAdapterKeyFramePtsCallback_->OnReportKeyFramePts(keyFramePts_);
+        keyFramePts_.clear();
+    }
+}
+
+bool SurfaceEncoderAdapter::AddPauseResumePts(int64_t currentPts)
+{
+    if (pauseResumePts_.empty()) {
+        return false;
+    }
+    auto stateCode = pauseResumePts_[0].second;
+    MEDIA_LOG_D("CheckFrames stateCode: " PUBLIC_LOG_D32
+        " time:" PUBLIC_LOG_D64, static_cast<int32_t>(stateCode), pauseResumePts_[0].first);
+    // means not dropped frames when less than pause time
+    if (stateCode == StateCode::PAUSE && currentPts < pauseResumePts_[0].first) {
+        return false;
+    }
+    // means dropped frames when less than resume time
+    if (stateCode == StateCode::RESUME && currentPts < pauseResumePts_[0].first) {
+        return true;
+    }
+    if (stateCode == StateCode::PAUSE) {
+        MEDIA_LOG_D("AddPausePts %{public}s start", keyFramePts_.c_str());
+        keyFramePts_ += std::to_string(preKeyFramePts_) + ",";
+        MEDIA_LOG_D("AddPausePts %{public}s end", keyFramePts_.c_str());
+    }
+    if (stateCode == StateCode::RESUME) {
+        MEDIA_LOG_D("AddResumePts %{public}s start", keyFramePts_.c_str());
+        keyFramePts_ += std::to_string(currentKeyFramePts_) + ",";
+        MEDIA_LOG_D("AddResumePts %{public}s end", keyFramePts_.c_str());
+    }
+    pauseResumePts_.pop_front();
+    return AddPauseResumePts(currentPts);
+}
 } // namespace MEDIA
 } // namespace OHOS
--- a/services/media_engine/filters/surface_encoder_adapter.h
+++ b/services/media_engine/filters/surface_encoder_adapter.h
@ -50,6 +50,12 @@ public:
    virtual void OnOutputFormatChanged(const std::shared_ptr<Meta> &format) = 0;
 };

+class EncoderAdapterKeyFramePtsCallback {
+public:
+    virtual ~EncoderAdapterKeyFramePtsCallback() = default;
+    virtual void OnReportKeyFramePts(std::string KeyFramePts) = 0;
+};
+
 class SurfaceEncoderAdapter : public std::enable_shared_from_this<SurfaceEncoderAdapter>  {
 public:
    explicit SurfaceEncoderAdapter();
@ -60,6 +66,8 @@ public:
    Status SetWatermark(std::shared_ptr<AVBuffer> &waterMarkBuffer);
    Status SetOutputBufferQueue(const sptr<AVBufferQueueProducer> &bufferQueueProducer);
    Status SetEncoderAdapterCallback(const std::shared_ptr<EncoderAdapterCallback> &encoderAdapterCallback);
+    Status SetEncoderAdapterKeyFramePtsCallback(
+        const std::shared_ptr<EncoderAdapterKeyFramePtsCallback> &encoderAdapterKeyFramePtsCallback);
    Status SetInputSurface(sptr<Surface> surface);
    Status SetTransCoderMode();
    sptr<Surface> GetInputSurface();
@ -82,6 +90,7 @@ public:
        std::shared_ptr<Format> &parameter);

    std::shared_ptr<EncoderAdapterCallback> encoderAdapterCallback_;
+    std::shared_ptr<EncoderAdapterKeyFramePtsCallback> encoderAdapterKeyFramePtsCallback_;

 private:
    void ReleaseBuffer();
@ -91,6 +100,9 @@ private:
    void ConfigureAboutEnableTemporalScale(MediaAVCodec::Format &format, const std::shared_ptr<Meta> &meta);
    bool CheckFrames(int64_t currentPts, int64_t &checkFramesPauseTime);
    void GetCurrentTime(int64_t &currentTime);
+    void AddStartPts(int64_t currentPts);
+    void AddStopPts();
+    bool AddPauseResumePts(int64_t currentPts);

    std::shared_ptr<MediaAVCodec::AVCodecVideoEncoder> codecServer_;
    sptr<AVBufferQueueProducer> outputBufferQueueProducer_;
@ -122,6 +134,13 @@ private:
    uint64_t instanceId_{0};
    int32_t appUid_ {0};
    int32_t appPid_ {0};
+
+    std::string keyFramePts_;
+    bool isStartKeyFramePts_ = false;
+    bool isStopKeyFramePts_ = false;
+    int64_t currentKeyFramePts_{-1};
+    int64_t preKeyFramePts_{-1};
+    std::deque<std::pair<int64_t, StateCode>> pauseResumePts_;
 };
 } // namespace MediaAVCodec
 } // namespace OHOS
--- a/services/media_engine/filters/surface_encoder_filter.cpp
+++ b/services/media_engine/filters/surface_encoder_filter.cpp
@ -22,6 +22,7 @@
 #include "common/media_core.h"
 #include "filter/filter_factory.h"
 #include "surface_encoder_adapter.h"
+#include "muxer_filter.h"

 namespace {
 constexpr OHOS::HiviewDFX::HiLogLabel LABEL = { LOG_CORE, LOG_DOMAIN_SYSTEM_PLAYER, "HiStreamer" };
@ -42,6 +43,7 @@ public:
        : surfaceEncoderFilter_(std::move(surfaceEncoderFilter))
    {
    }
+
    void OnLinkedResult(const sptr<AVBufferQueueProducer> &queue, std::shared_ptr<Meta> &meta) override
    {
        if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) {
@ -50,6 +52,7 @@ public:
            MEDIA_LOG_I("invalid surfaceEncoderFilter");
        }
    }
+
    void OnUnlinkedResult(std::shared_ptr<Meta> &meta) override
    {
        if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) {
@ -58,6 +61,7 @@ public:
            MEDIA_LOG_I("invalid surfaceEncoderFilter");
        }
    }
+
    void OnUpdatedResult(std::shared_ptr<Meta> &meta) override
    {
        if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) {
@ -66,6 +70,7 @@ public:
            MEDIA_LOG_I("invalid surfaceEncoderFilter");
        }
    }
+
 private:
    std::weak_ptr<SurfaceEncoderFilter> surfaceEncoderFilter_;
 };
@ -94,6 +99,28 @@ private:
    std::weak_ptr<SurfaceEncoderFilter> surfaceEncoderFilter_;
 };

+class SurfaceEncoderAdapterKeyFramePtsCallback : public EncoderAdapterKeyFramePtsCallback {
+public:
+    explicit SurfaceEncoderAdapterKeyFramePtsCallback(std::shared_ptr<SurfaceEncoderFilter> surfaceEncoderFilter)
+        : surfaceEncoderFilter_(std::move(surfaceEncoderFilter))
+    {
+    }
+    
+    void OnReportKeyFramePts(std::string KeyFramePts) override
+    {
+        if (auto surfaceEncoderFilter = surfaceEncoderFilter_.lock()) {
+            MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts start");
+            surfaceEncoderFilter->OnReportKeyFramePts(KeyFramePts);
+            MEDIA_LOG_D("SurfaceEncoderAdapterKeyFramePtsCallback OnReportKeyFramePts end");
+        } else {
+            MEDIA_LOG_I("invalid surfaceEncoderFilter");
+        }
+    }
+
+private:
+    std::weak_ptr<SurfaceEncoderFilter> surfaceEncoderFilter_;
+};
+
 SurfaceEncoderFilter::SurfaceEncoderFilter(std::string name, FilterType type): Filter(name, type)
 {
    MEDIA_LOG_I("encoder filter create");
@ -142,6 +169,9 @@ void SurfaceEncoderFilter::Init(const std::shared_ptr<EventReceiver> &receiver,
            std::shared_ptr<EncoderAdapterCallback> encoderAdapterCallback =
                std::make_shared<SurfaceEncoderAdapterCallback>(shared_from_this());
            mediaCodec_->SetEncoderAdapterCallback(encoderAdapterCallback);
+            std::shared_ptr<EncoderAdapterKeyFramePtsCallback> encoderAdapterKeyFramePtsCallback =
+                std::make_shared<SurfaceEncoderAdapterKeyFramePtsCallback>(shared_from_this());
+            mediaCodec_->SetEncoderAdapterKeyFramePtsCallback(encoderAdapterKeyFramePtsCallback);
        } else {
            MEDIA_LOG_I("Init mediaCodec fail");
            eventReceiver_->OnEvent({"surface_encoder_filter", EventType::EVENT_ERROR, Status::ERROR_UNKNOWN});
@ -378,6 +408,20 @@ void SurfaceEncoderFilter::SetCallingInfo(int32_t appUid, int32_t appPid,
        mediaCodec_->SetCallingInfo(appUid, appPid, bundleName, instanceId);
    }
 }
+
+void SurfaceEncoderFilter::OnReportKeyFramePts(std::string KeyFramePts)
+{
+    MEDIA_LOG_I("OnReportKeyFramePts %{public}s enter", KeyFramePts.c_str());
+    std::shared_ptr<Meta> userMeta = std::make_shared<Meta>();
+    userMeta->SetData("com.openharmony.recorder.timestamp", KeyFramePts.data());
+    std::shared_ptr<MuxerFilter> muxerFilter = std::static_pointer_cast<MuxerFilter>(nextFilter_);
+    if (muxerFilter != nullptr) {
+        muxerFilter->SetUserMeta(userMeta);
+        MEDIA_LOG_I("SetUserMeta %{public}s", KeyFramePts.c_str());
+    } else {
+        MEDIA_LOG_E("muxerFilter is null");
+    }
+}
 } // namespace Pipeline
 } // namespace MEDIA
 } // namespace OHOS
--- a/test/unittest/decoder_filter_test/BUILD.gn
+++ b/test/unittest/decoder_filter_test/BUILD.gn
@ -78,12 +78,15 @@ ohos_unittest("decoder_filter_unit_test") {
    "$av_codec_root_dir/services/dfx:av_codec_service_log_dfx_public_config",
  ]
  sources = decoder_filter_test_sources + [
-              "video_resize_filter_unit_test.cpp",
              "audio_decoder_filter_unit_test.cpp",
-              "video_decoder_adapter_unit_test.cpp",
              "metadata_filter_unit_test.cpp",
              "surface_encoder_filter_unit_test.cpp",
+              "video_decoder_adapter_unit_test.cpp",
+              "video_resize_filter_unit_test.cpp",
            ]
+
+  public_deps = [ "$av_codec_root_dir/services/media_engine/filters:av_codec_media_engine_filters" ]
+
  deps = [
    "$av_codec_root_dir/interfaces/inner_api/native:av_codec_client",
    "$av_codec_root_dir/interfaces/kits/c:native_media_codecbase",