diff --git a/libavformat/rtpdec.c b/libavformat/rtpdec.c
index 77b59a3ff3..700924962e 100644
--- a/libavformat/rtpdec.c
+++ b/libavformat/rtpdec.c
@@ -123,9 +123,13 @@ static int rtcp_parse_packet(RTPDemuxContext *s, const unsigned char *buf, int l
             payload_len = (AV_RB16(buf + 2) + 1) * 4;
 
             s->last_rtcp_ntp_time = AV_RB64(buf + 8);
-            if (s->first_rtcp_ntp_time == AV_NOPTS_VALUE)
-                s->first_rtcp_ntp_time = s->last_rtcp_ntp_time;
             s->last_rtcp_timestamp = AV_RB32(buf + 16);
+            if (s->first_rtcp_ntp_time == AV_NOPTS_VALUE) {
+                s->first_rtcp_ntp_time = s->last_rtcp_ntp_time;
+                if (!s->base_timestamp)
+                    s->base_timestamp = s->last_rtcp_timestamp;
+                s->rtcp_ts_offset = s->last_rtcp_timestamp - s->base_timestamp;
+            }
 
             buf += payload_len;
             len -= payload_len;
@@ -440,8 +444,15 @@ static void finalize_packet(RTPDemuxContext *s, AVPacket *pkt, uint32_t timestam
         delta_timestamp = timestamp - s->last_rtcp_timestamp;
         /* convert to the PTS timebase */
         addend = av_rescale(s->last_rtcp_ntp_time - s->first_rtcp_ntp_time, s->st->time_base.den, (uint64_t)s->st->time_base.num << 32);
-        pkt->pts = s->range_start_offset + addend + delta_timestamp;
+        pkt->pts = s->range_start_offset + s->rtcp_ts_offset + addend +
+                   delta_timestamp;
+        return;
     }
+    if (timestamp == RTP_NOTS_VALUE)
+        return;
+    if (!s->base_timestamp)
+        s->base_timestamp = timestamp;
+    pkt->pts = s->range_start_offset + timestamp - s->base_timestamp;
 }
 
 static int rtp_parse_packet_internal(RTPDemuxContext *s, AVPacket *pkt,
diff --git a/libavformat/rtpdec.h b/libavformat/rtpdec.h
index 778ca92984..c5a521760d 100644
--- a/libavformat/rtpdec.h
+++ b/libavformat/rtpdec.h
@@ -172,6 +172,7 @@ struct RTPDemuxContext {
     int64_t last_rtcp_ntp_time;    // TODO: move into statistics
     int64_t first_rtcp_ntp_time;   // TODO: move into statistics
     uint32_t last_rtcp_timestamp;  // TODO: move into statistics
+    int64_t rtcp_ts_offset;
 
     /* rtcp sender statistics */
     unsigned int packet_count;     // TODO: move into statistics (outgoing)
diff --git a/libavformat/rtsp.c b/libavformat/rtsp.c
index 8c9d79cd8f..ca84e5df2b 100644
--- a/libavformat/rtsp.c
+++ b/libavformat/rtsp.c
@@ -1617,11 +1617,21 @@ int ff_rtsp_fetch_packet(AVFormatContext *s, AVPacket *pkt)
                  * in order to map their timestamp origin to the same ntp time
                  * as this one. */
                 int i;
+                AVStream *st = NULL;
+                if (rtsp_st->stream_index >= 0)
+                    st = s->streams[rtsp_st->stream_index];
                 for (i = 0; i < rt->nb_rtsp_streams; i++) {
                     RTPDemuxContext *rtpctx2 = rt->rtsp_streams[i]->transport_priv;
-                    if (rtpctx2 &&
-                        rtpctx2->first_rtcp_ntp_time == AV_NOPTS_VALUE)
+                    AVStream *st2 = NULL;
+                    if (rt->rtsp_streams[i]->stream_index >= 0)
+                        st2 = s->streams[rt->rtsp_streams[i]->stream_index];
+                    if (rtpctx2 && st && st2 &&
+                        rtpctx2->first_rtcp_ntp_time == AV_NOPTS_VALUE) {
                         rtpctx2->first_rtcp_ntp_time = rtpctx->first_rtcp_ntp_time;
+                        rtpctx2->rtcp_ts_offset = av_rescale_q(
+                            rtpctx->rtcp_ts_offset, st->time_base,
+                            st2->time_base);
+                    }
                 }
             }
             if (ret == -RTCP_BYE) {
diff --git a/libavformat/rtspdec.c b/libavformat/rtspdec.c
index 213f7095fb..ad13b170aa 100644
--- a/libavformat/rtspdec.c
+++ b/libavformat/rtspdec.c
@@ -67,6 +67,8 @@ static int rtsp_read_play(AVFormatContext *s)
                 if (reply->range_start != AV_NOPTS_VALUE) {
                     rtpctx->last_rtcp_ntp_time  = AV_NOPTS_VALUE;
                     rtpctx->first_rtcp_ntp_time = AV_NOPTS_VALUE;
+                    rtpctx->base_timestamp      = 0;
+                    rtpctx->rtcp_ts_offset      = 0;
                     if (st)
                         rtpctx->range_start_offset =
                             av_rescale_q(reply->range_start, AV_TIME_BASE_Q,