Skip to content

Commit 950fc9d

Browse files
theandi666theandi666
committed
In order to recover from video lagging behind audio, drop avc frames
that are not referenced by other frames before feeding them into the decoder. Change-Id: I822190af8f8329567bff8da1ea23136d0a765481
1 parent 44895fe commit 950fc9d

File tree

8 files changed

+155
-44
lines changed

8 files changed

+155
-44
lines changed

‎media/libmediaplayerservice/nuplayer/NuPlayer.cpp‎

Lines changed: 80 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -34,25 +34,34 @@
3434
#include <media/stagefright/foundation/ADebug.h>
3535
#include <media/stagefright/foundation/AMessage.h>
3636
#include <media/stagefright/ACodec.h>
37+
#include <media/stagefright/MediaDefs.h>
3738
#include <media/stagefright/MediaErrors.h>
3839
#include <media/stagefright/MetaData.h>
3940
#include <surfaceflinger/Surface.h>
4041
#include <gui/ISurfaceTexture.h>
4142

43+
#include "avc_utils.h"
44+
4245
namespace android {
4346

4447
////////////////////////////////////////////////////////////////////////////////
4548

4649
NuPlayer::NuPlayer()
4750
: mUIDValid(false),
51+
mVideoIsAVC(false),
4852
mAudioEOS(false),
4953
mVideoEOS(false),
5054
mScanSourcesPending(false),
5155
mScanSourcesGeneration(0),
5256
mFlushingAudio(NONE),
5357
mFlushingVideo(NONE),
5458
mResetInProgress(false),
55-
mResetPostponed(false) {
59+
mResetPostponed(false),
60+
mSkipRenderingAudioUntilMediaTimeUs(-1ll),
61+
mSkipRenderingVideoUntilMediaTimeUs(-1ll),
62+
mVideoLateByUs(0ll),
63+
mNumFramesTotal(0ll),
64+
mNumFramesDropped(0ll) {
5665
}
5766

5867
NuPlayer::~NuPlayer() {
@@ -185,10 +194,14 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
185194
{
186195
LOGV("kWhatStart");
187196

197+
mVideoIsAVC = false;
188198
mAudioEOS = false;
189199
mVideoEOS = false;
190200
mSkipRenderingAudioUntilMediaTimeUs = -1;
191201
mSkipRenderingVideoUntilMediaTimeUs = -1;
202+
mVideoLateByUs = 0;
203+
mNumFramesTotal = 0;
204+
mNumFramesDropped = 0;
192205

193206
mSource->start();
194207

@@ -269,6 +282,8 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
269282
} else {
270283
CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
271284
mFlushingVideo = FLUSHED;
285+
286+
mVideoLateByUs = 0;
272287
}
273288

274289
LOGV("decoder %s flush completed", audio ? "audio" : "video");
@@ -397,13 +412,18 @@ void NuPlayer::onMessageReceived(const sp<AMessage> &msg) {
397412
int64_t positionUs;
398413
CHECK(msg->findInt64("positionUs", &positionUs));
399414

415+
CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
416+
400417
if (mDriver != NULL) {
401418
sp<NuPlayerDriver> driver = mDriver.promote();
402419
if (driver != NULL) {
403420
driver->notifyPosition(positionUs);
421+
422+
driver->notifyFrameStats(
423+
mNumFramesTotal, mNumFramesDropped);
404424
}
405425
}
406-
} else {
426+
} else if (what == Renderer::kWhatFlushComplete) {
407427
CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete);
408428

409429
int32_t audio;
@@ -565,6 +585,12 @@ status_t NuPlayer::instantiateDecoder(bool audio, sp<Decoder> *decoder) {
565585
return -EWOULDBLOCK;
566586
}
567587

588+
if (!audio) {
589+
const char *mime;
590+
CHECK(meta->findCString(kKeyMIMEType, &mime));
591+
mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime);
592+
}
593+
568594
sp<AMessage> notify =
569595
new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
570596
id());
@@ -598,53 +624,70 @@ status_t NuPlayer::feedDecoderInputData(bool audio, const sp<AMessage> &msg) {
598624
}
599625

600626
sp<ABuffer> accessUnit;
601-
status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
602627

603-
if (err == -EWOULDBLOCK) {
604-
return err;
605-
} else if (err != OK) {
606-
if (err == INFO_DISCONTINUITY) {
607-
int32_t type;
608-
CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
628+
bool dropAccessUnit;
629+
do {
630+
status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
609631

610-
bool formatChange =
611-
type == ATSParser::DISCONTINUITY_FORMATCHANGE;
632+
if (err == -EWOULDBLOCK) {
633+
return err;
634+
} else if (err != OK) {
635+
if (err == INFO_DISCONTINUITY) {
636+
int32_t type;
637+
CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
612638

613-
LOGV("%s discontinuity (formatChange=%d)",
614-
audio ? "audio" : "video", formatChange);
639+
bool formatChange =
640+
type == ATSParser::DISCONTINUITY_FORMATCHANGE;
615641

616-
if (audio) {
617-
mSkipRenderingAudioUntilMediaTimeUs = -1;
618-
} else {
619-
mSkipRenderingVideoUntilMediaTimeUs = -1;
620-
}
642+
LOGV("%s discontinuity (formatChange=%d)",
643+
audio ? "audio" : "video", formatChange);
621644

622-
sp<AMessage> extra;
623-
if (accessUnit->meta()->findMessage("extra", &extra)
624-
&& extra != NULL) {
625-
int64_t resumeAtMediaTimeUs;
626-
if (extra->findInt64(
627-
"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
628-
LOGI("suppressing rendering of %s until %lld us",
629-
audio ? "audio" : "video", resumeAtMediaTimeUs);
645+
if (audio) {
646+
mSkipRenderingAudioUntilMediaTimeUs = -1;
647+
} else {
648+
mSkipRenderingVideoUntilMediaTimeUs = -1;
649+
}
630650

631-
if (audio) {
632-
mSkipRenderingAudioUntilMediaTimeUs =
633-
resumeAtMediaTimeUs;
634-
} else {
635-
mSkipRenderingVideoUntilMediaTimeUs =
636-
resumeAtMediaTimeUs;
651+
sp<AMessage> extra;
652+
if (accessUnit->meta()->findMessage("extra", &extra)
653+
&& extra != NULL) {
654+
int64_t resumeAtMediaTimeUs;
655+
if (extra->findInt64(
656+
"resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
657+
LOGI("suppressing rendering of %s until %lld us",
658+
audio ? "audio" : "video", resumeAtMediaTimeUs);
659+
660+
if (audio) {
661+
mSkipRenderingAudioUntilMediaTimeUs =
662+
resumeAtMediaTimeUs;
663+
} else {
664+
mSkipRenderingVideoUntilMediaTimeUs =
665+
resumeAtMediaTimeUs;
666+
}
637667
}
638668
}
669+
670+
flushDecoder(audio, formatChange);
639671
}
640672

641-
flushDecoder(audio, formatChange);
673+
reply->setInt32("err", err);
674+
reply->post();
675+
return OK;
642676
}
643677

644-
reply->setInt32("err", err);
645-
reply->post();
646-
return OK;
647-
}
678+
if (!audio) {
679+
++mNumFramesTotal;
680+
}
681+
682+
dropAccessUnit = false;
683+
if (!audio
684+
&& mVideoLateByUs > 100000ll
685+
&& mVideoIsAVC
686+
&& !IsAVCReferenceFrame(accessUnit)) {
687+
dropAccessUnit = true;
688+
++mNumFramesDropped;
689+
}
690+
} while (dropAccessUnit);
648691

649692
// LOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
650693

‎media/libmediaplayerservice/nuplayer/NuPlayer.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ struct NuPlayer : public AHandler {
9292
sp<NativeWindowWrapper> mNativeWindow;
9393
sp<MediaPlayerBase::AudioSink> mAudioSink;
9494
sp<Decoder> mVideoDecoder;
95+
bool mVideoIsAVC;
9596
sp<Decoder> mAudioDecoder;
9697
sp<Renderer> mRenderer;
9798

@@ -119,6 +120,9 @@ struct NuPlayer : public AHandler {
119120
int64_t mSkipRenderingAudioUntilMediaTimeUs;
120121
int64_t mSkipRenderingVideoUntilMediaTimeUs;
121122

123+
int64_t mVideoLateByUs;
124+
int64_t mNumFramesTotal, mNumFramesDropped;
125+
122126
status_t instantiateDecoder(bool audio, sp<Decoder> *decoder);
123127

124128
status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg);

‎media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ NuPlayerDriver::NuPlayerDriver()
3131
: mResetInProgress(false),
3232
mDurationUs(-1),
3333
mPositionUs(-1),
34+
mNumFramesTotal(0),
35+
mNumFramesDropped(0),
3436
mLooper(new ALooper),
3537
mState(UNINITIALIZED),
3638
mStartupSeekTimeUs(-1) {
@@ -292,4 +294,30 @@ void NuPlayerDriver::notifySeekComplete() {
292294
sendEvent(MEDIA_SEEK_COMPLETE);
293295
}
294296

297+
void NuPlayerDriver::notifyFrameStats(
298+
int64_t numFramesTotal, int64_t numFramesDropped) {
299+
Mutex::Autolock autoLock(mLock);
300+
mNumFramesTotal = numFramesTotal;
301+
mNumFramesDropped = numFramesDropped;
302+
}
303+
304+
status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const {
305+
Mutex::Autolock autoLock(mLock);
306+
307+
FILE *out = fdopen(dup(fd), "w");
308+
309+
fprintf(out, " NuPlayer\n");
310+
fprintf(out, " numFramesTotal(%lld), numFramesDropped(%lld), "
311+
"percentageDropped(%.2f)\n",
312+
mNumFramesTotal,
313+
mNumFramesDropped,
314+
mNumFramesTotal == 0
315+
? 0.0 : (double)mNumFramesDropped / mNumFramesTotal);
316+
317+
fclose(out);
318+
out = NULL;
319+
320+
return OK;
321+
}
322+
295323
} // namespace android

‎media/libmediaplayerservice/nuplayer/NuPlayerDriver.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,23 +60,28 @@ struct NuPlayerDriver : public MediaPlayerInterface {
6060
virtual status_t getMetadata(
6161
const media::Metadata::Filter& ids, Parcel *records);
6262

63+
virtual status_t dump(int fd, const Vector<String16> &args) const;
64+
6365
void notifyResetComplete();
6466
void notifyDuration(int64_t durationUs);
6567
void notifyPosition(int64_t positionUs);
6668
void notifySeekComplete();
69+
void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped);
6770

6871
protected:
6972
virtual ~NuPlayerDriver();
7073

7174
private:
72-
Mutex mLock;
75+
mutable Mutex mLock;
7376
Condition mCondition;
7477

7578
// The following are protected through "mLock"
7679
// >>>
7780
bool mResetInProgress;
7881
int64_t mDurationUs;
7982
int64_t mPositionUs;
83+
int64_t mNumFramesTotal;
84+
int64_t mNumFramesDropped;
8085
// <<<
8186

8287
sp<ALooper> mLooper;

‎media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,8 @@ NuPlayer::Renderer::Renderer(
4747
mHasVideo(false),
4848
mSyncQueues(false),
4949
mPaused(false),
50-
mLastPositionUpdateUs(-1ll) {
50+
mLastPositionUpdateUs(-1ll),
51+
mVideoLateByUs(0ll) {
5152
}
5253

5354
NuPlayer::Renderer::~Renderer() {
@@ -357,22 +358,26 @@ void NuPlayer::Renderer::onDrainVideoQueue() {
357358

358359
mVideoQueue.erase(mVideoQueue.begin());
359360
entry = NULL;
361+
362+
mVideoLateByUs = 0ll;
363+
364+
notifyPosition();
360365
return;
361366
}
362367

363-
#if 0
364368
int64_t mediaTimeUs;
365369
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
366370

367371
int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
368-
int64_t lateByUs = ALooper::GetNowUs() - realTimeUs;
372+
mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
369373

370-
if (lateByUs > 40000) {
371-
LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
374+
bool tooLate = (mVideoLateByUs > 40000);
375+
376+
if (tooLate) {
377+
LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
372378
} else {
373379
LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
374380
}
375-
#endif
376381

377382
entry->mNotifyConsumed->setInt32("render", true);
378383
entry->mNotifyConsumed->post();
@@ -604,6 +609,7 @@ void NuPlayer::Renderer::notifyPosition() {
604609
sp<AMessage> notify = mNotify->dup();
605610
notify->setInt32("what", kWhatPosition);
606611
notify->setInt64("positionUs", positionUs);
612+
notify->setInt64("videoLateByUs", mVideoLateByUs);
607613
notify->post();
608614
}
609615

‎media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ struct NuPlayer::Renderer : public AHandler {
101101
bool mPaused;
102102

103103
int64_t mLastPositionUpdateUs;
104+
int64_t mVideoLateByUs;
104105

105106
bool onDrainAudioQueue();
106107
void postDrainAudioQueue(int64_t delayUs = 0);
@@ -118,6 +119,7 @@ struct NuPlayer::Renderer : public AHandler {
118119
void notifyEOS(bool audio, status_t finalResult);
119120
void notifyFlushComplete(bool audio);
120121
void notifyPosition();
122+
void notifyVideoLateBy(int64_t lateByUs);
121123

122124
void flushQueue(List<QueueEntry> *queue);
123125
bool dropBufferWhileFlushing(bool audio, const sp<AMessage> &msg);

‎media/libstagefright/avc_utils.cpp‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,28 @@ bool IsIDR(const sp<ABuffer> &buffer) {
329329
return foundIDR;
330330
}
331331

332+
bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit) {
333+
const uint8_t *data = accessUnit->data();
334+
size_t size = accessUnit->size();
335+
336+
const uint8_t *nalStart;
337+
size_t nalSize;
338+
while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
339+
CHECK_GT(nalSize, 0u);
340+
341+
unsigned nalType = nalStart[0] & 0x1f;
342+
343+
if (nalType == 5) {
344+
return true;
345+
} else if (nalType == 1) {
346+
unsigned nal_ref_idc = (nalStart[0] >> 5) & 3;
347+
return nal_ref_idc != 0;
348+
}
349+
}
350+
351+
return true;
352+
}
353+
332354
sp<MetaData> MakeAACCodecSpecificData(
333355
unsigned profile, unsigned sampling_freq_index,
334356
unsigned channel_configuration) {

‎media/libstagefright/include/avc_utils.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ struct MetaData;
5050
sp<MetaData> MakeAVCCodecSpecificData(const sp<ABuffer> &accessUnit);
5151

5252
bool IsIDR(const sp<ABuffer> &accessUnit);
53+
bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit);
5354

5455
const char *AVCProfileToString(uint8_t profile);
5556

0 commit comments

Comments
 (0)