欢迎您访问程序员文章站本站旨在为大家提供分享程序员计算机编程知识!
您现在的位置是: 首页

Android8.0 Media系统(一)

程序员文章站 2022-07-13 12:50:31
...

以上四篇对Audio系统的简要分析,由于Audio涉及的范围比较广,以后分析其他子系统时在做详细分析。我们继续Media系统的征程,Media系统仍然是一个庞大的系统,以MediaPlayer为例,贯穿了Java,JNI,C++库,硬件抽象层,OpenMax驱动,涉及到音视频,编解码等内容。

Android8.0 Media系统(一)

为了简化分析,又不脱离主线,我们仍然从一个Demo实例开始,逐点逐层地取剖析,由于贯穿的篇幅比较长,我们尽可能得以一个完整的知识点为一篇。本篇我们就以MediaPlayer为主。

1. 创建MediaPlayer

MediaPlayer mediaPlayer = new MediaPlayer();  //创建多媒体播放器
mediaPlayer.setDataSource(path);  //设置源数据
mediaPlayer.setDisplay(surfaceView.getHolder());  //播放视频
mediaPlayer.prepare();  //播放器准备
mediaPlayer.start(); //开始播放

frameworks\base\media\java\android\media\MediaPlayer.java
静态代码块在构造函数之前创建的

static {
   System.loadLibrary("media_jni");
   native_init(); //初始化
}

frameworks\base\media\jni\android_media_MediaPlayer.cpp

static voidandroid_media_MediaPlayer_native_init(JNIEnv *env)
{
    jclass clazz;
    clazz = env->FindClass("android/media/MediaPlayer"); //查找类
    fields.context = env->GetFieldID(clazz, "mNativeContext", "J"); //绑定上下文
    fields.post_event = env->GetStaticMethodID(clazz, "postEventFromNative", //绑定Java方法
                                               "(Ljava/lang/Object;IIILjava/lang/Object;)V");
    fields.surface_texture = env->GetFieldID(clazz, "mNativeSurfaceTexture", "J");
    env->DeleteLocalRef(clazz);
    clazz = env->FindClass("android/net/ProxyInfo"); //查找代理信息
    fields.proxyConfigGetHost =
        env->GetMethodID(clazz, "getHost", "()Ljava/lang/String;");
    fields.proxyConfigGetPort =
        env->GetMethodID(clazz, "getPort", "()I");
    fields.proxyConfigGetExclusionList =
        env->GetMethodID(clazz, "getExclusionListAsString", "()Ljava/lang/String;");
    env->DeleteLocalRef(clazz);
    gBufferingParamsFields.init(env);
    // Modular DRM
    FIND_CLASS(clazz, "android/media/MediaDrm$MediaDrmStateException");
    gPlaybackParamsFields.init(env);
    gSyncParamsFields.init(env);
    gVolumeShaperFields.init(env);
}

frameworks\base\media\java\android\media\MediaPlayer.java

public MediaPlayer() {
   ......
   native_setup(new WeakReference<MediaPlayer>(this)); //调用JNI层
   ......
}

frameworks\base\media\jni\android_media_MediaPlayer.cpp

static void android_media_MediaPlayer_native_setup(JNIEnv *env, jobject thiz, jobject weak_this)
{
    sp<MediaPlayer> mp = new MediaPlayer(); //创建native对象
    // create new listener and give it to MediaPlayer
    sp<JNIMediaPlayerListener> listener = new JNIMediaPlayerListener(env, thiz, weak_this);
    mp->setListener(listener); //设置Java层的回调监听

    // Stow our new C++ MediaPlayer in an opaque field in the Java object.
    setMediaPlayer(env, thiz, mp); //存个副本
}

2. 设置播放数据源

static void android_media_MediaPlayer_setDataSourceFD(JNIEnv *env, jobject thiz, jobject fileDescriptor, jlong offset, jlong length)
{
    sp<MediaPlayer> mp = getMediaPlayer(env, thiz); //取先前保存的mp
    ......
    int fd = jniGetFDFromFileDescriptor(env, fileDescriptor); //取文件描述
    ALOGV("setDataSourceFD: fd %d", fd);
    //setDataSource
    process_media_player_call( env, thiz, mp->setDataSource(fd, offset, length), "java/io/IOException", "setDataSourceFD failed." );
}

frameworks\av\media\libmedia\mediaplayer.cpp

status_t MediaPlayer::setDataSource(int fd, int64_t offset, int64_t length)
{
    ALOGV("setDataSource(%d, %" PRId64 ", %" PRId64 ")", fd, offset, length);
    status_t err = UNKNOWN_ERROR;
    const sp<IMediaPlayerService> service(getMediaPlayerService()); //服务代理
    if (service != 0) {
        sp<IMediaPlayer> player(service->create(this, mAudioSessionId)); //创建Player
        if ((NO_ERROR != doSetRetransmitEndpoint(player)) ||
            (NO_ERROR != player->setDataSource(fd, offset, length))) {
            player.clear();
        }
        err = attachNewPlayer(player); //更新备份
    }
    return err;
}

frameworks\av\media\libmediaplayerservice\MediaPlayerService.cpp

sp<IMediaPlayer> MediaPlayerService::create(const sp<IMediaPlayerClient>& client,
        audio_session_t audioSessionId)
{
    pid_t pid = IPCThreadState::self()->getCallingPid();
    int32_t connId = android_atomic_inc(&mNextConnId);
    //创建一个Client
    sp<Client> c = new Client(
            this, pid, connId, client, audioSessionId,
            IPCThreadState::self()->getCallingUid());

    wp<Client> w = c;
    {
        Mutex::Autolock lock(mLock);
        mClients.add(w);
    }
    return c;
}
 class Client : public BnMediaPlayer {
   ......
 }
status_t MediaPlayerService::Client::setDataSource(int fd, int64_t offset, int64_t length)
{
    //获取Player的Type
    player_type playerType = MediaPlayerFactory::getPlayerType(this,fd, offset,length);
    //按类型创建播放器
    sp<MediaPlayerBase> p = setDataSource_pre(playerType);
    // now set data source
    setDataSource_post(p, p->setDataSource(fd, offset, length)); //设置数据源
    return mStatus;
}

frameworks\av\media\libmediaplayerservice\MediaPlayerFactory.cpp

player_type MediaPlayerFactory::getPlayerType(const sp<IMediaPlayer>& client,
                                              int fd,
                                              int64_t offset,
                                              int64_t length) {
    GET_PLAYER_TYPE_IMPL(client, fd, offset, length); //调用下面的宏函数
}

泛型宏定义,按score值创建播放器

#define GET_PLAYER_TYPE_IMPL(a...)                      \
    Mutex::Autolock lock_(&sLock);                      \
                                                        \
    player_type ret = STAGEFRIGHT_PLAYER;               \
    float bestScore = 0.0;                              \
                                                        \
    for (size_t i = 0; i < sFactoryMap.size(); ++i) {   \
                                                        \
        IFactory* v = sFactoryMap.valueAt(i);           \
        float thisScore;                                \
        CHECK(v != NULL);                               \
        thisScore = v->scoreFactory(a, bestScore);      \
        if (thisScore > bestScore) {                    \
            ret = sFactoryMap.keyAt(i);                 \
            bestScore = thisScore;                      \
        }                                               \
    }                                                   \
                                                        \
    if (0.0 == bestScore) {                             \
        ret = getDefaultPlayerType();                   \
    }                                                   \
                                                        \
    return ret;

默认返回的播放器类型

static player_type getDefaultPlayerType() {
    return NU_PLAYER; //默认返回NuPlayer
}

执行 setDataSource_pre,先去创建播放器

sp<MediaPlayerBase> MediaPlayerService::Client::setDataSource_pre(
        player_type playerType)
{
    // create the right type of player
    sp<MediaPlayerBase> p = createPlayer(playerType); //按类型创建播放器
    sp<IServiceManager> sm = defaultServiceManager();
    sp<IBinder> binder = sm->getService(String16("media.extractor")); //获取提取器
    if (property_get_bool("persist.media.treble_omx", true)) { //获取 treble 架构下openmax 属性
        // Treble IOmx
        sp<IOmx> omx = IOmx::getService(); //获取openmax服务
        ......
    } else {
        // Legacy IOMX
        binder = sm->getService(String16("media.codec")); //获取解码器服务
        ......
    }

    if (!p->hardwareOutput()) {
        Mutex::Autolock l(mLock);
        //按Session创建音频输出
        mAudioOutput = new AudioOutput(mAudioSessionId, IPCThreadState::self()->getCallingUid(),
                mPid, mAudioAttributes);
        static_cast<MediaPlayerInterface*>(p.get())->setAudioSink(mAudioOutput);
    }

    return p;
}

取创建播放器

sp<MediaPlayerBase> MediaPlayerService::Client::createPlayer(player_type playerType)
{
    // determine if we have the right player type
    sp<MediaPlayerBase> p = mPlayer;
    ......
    if (p == NULL) {
        p = MediaPlayerFactory::createPlayer(playerType, this, notify, mPid); //创建Player
    }

    if (p != NULL) {
        p->setUID(mUid);
    }

    return p;
}

调用工厂的createPlayer方法

sp<MediaPlayerBase> MediaPlayerFactory::createPlayer(
        player_type playerType,
        void* cookie,
        notify_callback_f notifyFunc,
        pid_t pid) {
    sp<MediaPlayerBase> p;
    IFactory* factory;
    status_t init_result;
    Mutex::Autolock lock_(&sLock);
    factory = sFactoryMap.valueFor(playerType);
    CHECK(NULL != factory);
    p = factory->createPlayer(pid); //调用工厂子类吃醋昂见Player
    ......
    return p;
}

NuPlayerFactory 继承自 IFactory

class NuPlayerFactory : public MediaPlayerFactory::IFactory {
  public:
    ........
    virtual sp<MediaPlayerBase> createPlayer(pid_t pid) {
        ALOGV(" create NuPlayer");
        return new NuPlayerDriver(pid); //返回 NuPlayerDriver
    }
};

frameworks\av\media\libmediaplayerservice\nuplayer\NuPlayerDriver.cpp
去看看NuPlayerDriver做了什么

NuPlayerDriver::NuPlayerDriver(pid_t pid)
    : mState(STATE_IDLE),
      mIsAsyncPrepare(false),
      mAsyncResult(UNKNOWN_ERROR),
      mSetSurfaceInProgress(false),
      mDurationUs(-1),
      mPositionUs(-1),
      mSeekInProgress(false),
      mPlayingTimeUs(0),
      mLooper(new ALooper),
      mPlayer(new NuPlayer(pid)), //新建NuPlayer
      mPlayerFlags(0),
      mAnalyticsItem(NULL),
      mAtEOS(false),
      mLooping(false),
      mAutoLoop(false) {
    mLooper->setName("NuPlayerDriver Looper");
    // set up an analytics record
    mAnalyticsItem = new MediaAnalyticsItem(kKeyPlayer);
    mAnalyticsItem->generateSessionID();
    //注册启动消息循环
    mLooper->start(false, /* runOnCallingThread */
            true,  /* canCallJava */
            PRIORITY_AUDIO);
    mLooper->registerHandler(mPlayer);
    mPlayer->setDriver(this);
}

继续上面,setDataSource调用

status_t NuPlayerDriver::setDataSource(int fd, int64_t offset, int64_t length) {
    ALOGV("setDataSource(%p) file(%d)", this, fd);
    Mutex::Autolock autoLock(mLock);

    if (mState != STATE_IDLE) {
        return INVALID_OPERATION;
    }

    mState = STATE_SET_DATASOURCE_PENDING;

    mPlayer->setDataSourceAsync(fd, offset, length);

    while (mState == STATE_SET_DATASOURCE_PENDING) {
        mCondition.wait(mLock);
    }

    return mAsyncResult;
}
void NuPlayer::setDataSourceAsync(int fd, int64_t offset, int64_t length) {
    sp<AMessage> msg = new AMessage(kWhatSetDataSource, this);

    sp<AMessage> notify = new AMessage(kWhatSourceNotify, this);

    sp<GenericSource> source = new GenericSource(notify, mUIDValid, mUID);

    status_t err = source->setDataSource(fd, offset, length);

    msg->setObject("source", source);
    msg->post();
    mDataSourceType = DATA_SOURCE_TYPE_GENERIC_FD;
}
status_t NuPlayer::GenericSource::setDataSource(
        int fd, int64_t offset, int64_t length) {
    resetDataSource(); //重设参数
    mFd = dup(fd);
    mOffset = offset;
    mLength = length;

    // delay data source creation to prepareAsync() to avoid blocking
    // the calling thread in setDataSource for any significant time.
    return OK;
}

3. 播放器准备

void NuPlayer::prepareAsync() {
    ALOGV("prepareAsync");

    (new AMessage(kWhatPrepare, this))->post();
}
  case kWhatPrepare:
        {
            ALOGV("onMessageReceived kWhatPrepare");

            mSource->prepareAsync();
            break;
        }

frameworks\av\media\libmediaplayerservice\nuplayer\GenericSource.cpp

由AHandler消息处理器调用onPrepareAsync()函数

void NuPlayer::GenericSource::onPrepareAsync() {
    // delayed data source creation
    if (mDataSource == NULL) {
        // set to false first, if the extractor
        // comes back as secure, set it to true then.
        mIsSecure = false;

        if (!mUri.empty()) {
            const char* uri = mUri.c_str();
            String8 contentType;

            if (!strncasecmp("http://", uri, 7) || !strncasecmp("https://", uri, 8)) {
                mHttpSource = DataSource::CreateMediaHTTP(mHTTPService);
                if (mHttpSource == NULL) {
                    ALOGE("Failed to create http source!");
                    notifyPreparedAndCleanup(UNKNOWN_ERROR);
                    return;
                }
            }

            mDataSource = DataSource::CreateFromURI(
                   mHTTPService, uri, &mUriHeaders, &contentType,
                   static_cast<HTTPBase *>(mHttpSource.get()));
        } else {
            if (property_get_bool("media.stagefright.extractremote", true) &&
                    !FileSource::requiresDrm(mFd, mOffset, mLength, nullptr /* mime */)) {
                sp<IBinder> binder =
                        defaultServiceManager()->getService(String16("media.extractor")); //获取提取器
                if (binder != nullptr) {
                    ALOGD("FileSource remote");
                    sp<IMediaExtractorService> mediaExService(
                            interface_cast<IMediaExtractorService>(binder)); //提取器服务代理
                    sp<IDataSource> source =
                            mediaExService->makeIDataSource(mFd, mOffset, mLength); //数据源代理
                    ALOGV("IDataSource(FileSource): %p %d %lld %lld",
                            source.get(), mFd, (long long)mOffset, (long long)mLength);
                    if (source.get() != nullptr) {
                        mDataSource = DataSource::CreateFromIDataSource(source);
                        if (mDataSource != nullptr) {
                            // Close the local file descriptor as it is not needed anymore.
                            close(mFd);
                            mFd = -1;
                        }
                    } else {
                        ALOGW("extractor service cannot make data source");
                    }
                } else {
                    ALOGW("extractor service not running");
                }
            }
            if (mDataSource == nullptr) {
                ALOGD("FileSource local");
                mDataSource = new FileSource(mFd, mOffset, mLength);
            }
            // TODO: close should always be done on mFd, see the lines following
            // DataSource::CreateFromIDataSource above,
            // and the FileSource constructor should dup the mFd argument as needed.
            mFd = -1;
        }

        if (mDataSource == NULL) {
            ALOGE("Failed to create data source!");
            notifyPreparedAndCleanup(UNKNOWN_ERROR);
            return;
        }
    }

    if (mDataSource->flags() & DataSource::kIsCachingDataSource) {
        mCachedSource = static_cast<NuCachedSource2 *>(mDataSource.get());
    }

    // For cached streaming cases, we need to wait for enough
    // buffering before reporting prepared.
    mIsStreaming = (mCachedSource != NULL);

    // init extractor from data source
    status_t err = initFromDataSource(); //初始化数据源提取器

    if (err != OK) {
        ALOGE("Failed to init from data source!");
        notifyPreparedAndCleanup(err);
        return;
    }

    if (mVideoTrack.mSource != NULL) {
        sp<MetaData> meta = doGetFormatMeta(false /* audio */);
        sp<AMessage> msg = new AMessage;
        err = convertMetaDataToMessage(meta, &msg);
        if(err != OK) {
            notifyPreparedAndCleanup(err);
            return;
        }
        notifyVideoSizeChanged(msg); //通知视频大小改变
    }

    notifyFlagsChanged(
            // FLAG_SECURE will be known if/when prepareDrm is called by the app
            // FLAG_PROTECTED will be known if/when prepareDrm is called by the app
            FLAG_CAN_PAUSE |
            FLAG_CAN_SEEK_BACKWARD |
            FLAG_CAN_SEEK_FORWARD |
            FLAG_CAN_SEEK);

    finishPrepareAsync();

    ALOGV("onPrepareAsync: Done");
}

从数据源初始化提取器

status_t NuPlayer::GenericSource::initFromDataSource() {
    sp<IMediaExtractor> extractor;
    CHECK(mDataSource != NULL);

    extractor = MediaExtractor::Create(mDataSource, NULL); //从StageFright创建提取器

    if (extractor == NULL) {
        ALOGE("initFromDataSource, cannot create extractor!");
        return UNKNOWN_ERROR;
    }

    mFileMeta = extractor->getMetaData(); //提取元数据
    if (mFileMeta != NULL) {
        int64_t duration;
        if (mFileMeta->findInt64(kKeyDuration, &duration)) {
            mDurationUs = duration;
        }
    }

    int32_t totalBitrate = 0;

    size_t numtracks = extractor->countTracks(); //获取总Track
    if (numtracks == 0) {
        ALOGE("initFromDataSource, source has no track!");
        return UNKNOWN_ERROR;
    }

    mMimes.clear();

    for (size_t i = 0; i < numtracks; ++i) {
        sp<IMediaSource> track = extractor->getTrack(i);
        if (track == NULL) {
            continue;
        }

        sp<MetaData> meta = extractor->getTrackMetaData(i); //获取每一Track的元数据
        if (meta == NULL) {
            ALOGE("no metadata for track %zu", i);
            return UNKNOWN_ERROR;
        }

        const char *mime;
        CHECK(meta->findCString(kKeyMIMEType, &mime));

        ALOGV("initFromDataSource track[%zu]: %s", i, mime);

        // Do the string compare immediately with "mime",
        // we can't assume "mime" would stay valid after another
        // extractor operation, some extractors might modify meta
        // during getTrack() and make it invalid.
        if (!strncasecmp(mime, "audio/", 6)) { //音频
            if (mAudioTrack.mSource == NULL) {
                mAudioTrack.mIndex = i;
                mAudioTrack.mSource = track;
                mAudioTrack.mPackets =
                    new AnotherPacketSource(mAudioTrack.mSource->getFormat());

                if (!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_VORBIS)) {
                    mAudioIsVorbis = true;
                } else {
                    mAudioIsVorbis = false;
                }

                mMimes.add(String8(mime));
            }
        } else if (!strncasecmp(mime, "video/", 6)) {//视频
            if (mVideoTrack.mSource == NULL) {
                mVideoTrack.mIndex = i;
                mVideoTrack.mSource = track;
                mVideoTrack.mPackets =
                    new AnotherPacketSource(mVideoTrack.mSource->getFormat());

                // video always at the beginning
                mMimes.insertAt(String8(mime), 0);
            }
        }

        mSources.push(track);
        int64_t durationUs;
        if (meta->findInt64(kKeyDuration, &durationUs)) {//时长
            if (durationUs > mDurationUs) {
                mDurationUs = durationUs;
            }
        }

        int32_t bitrate;
        if (totalBitrate >= 0 && meta->findInt32(kKeyBitRate, &bitrate)) {//比特率
            totalBitrate += bitrate;
        } else {
            totalBitrate = -1;
        }
    }

    ALOGV("initFromDataSource mSources.size(): %zu  mIsSecure: %d  mime[0]: %s", mSources.size(),
            mIsSecure, (mMimes.isEmpty() ? "NONE" : mMimes[0].string()));

    if (mSources.size() == 0) {
        ALOGE("b/23705695");
        return UNKNOWN_ERROR;
    }

    // Modular DRM: The return value doesn't affect source initialization.
    (void)checkDrmInfo();

    mBitrate = totalBitrate;

    return OK;
}

篇幅过长,暂且到此为止,下一篇接着从StageFright的数据提取,解码来分析,可以猜测从多种数据源提取多媒体元数据后经由openmax解码后输出;预知后事如何请听下回分解。