AI: WebRTC: Report video/audio codec info and frame stats in HTTP API. v7.0.118 (#4554)

Author: ossrs-ai

.vscode/launch.json

@@ -57,6 +57,34 @@
             "engineLogging": true
           }
         },
+        {
+          "name": "Launch SRS with console.conf",
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${workspaceFolder}/trunk/cmake/build/srs",
+          "args": ["-c", "console.conf"],
+          "stopAtEntry": false,
+          "cwd": "${workspaceFolder}/trunk",
+          "environment": [],
+          "externalConsole": false,
+          "linux": {
+            "MIMode": "gdb"
+          },
+          "osx": {
+            "MIMode": "lldb"
+          },
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": "build",
+          "logging": {
+            "engineLogging": true
+          }
+        },
         {
           "name": "Launch srs-proxy",
           "type": "go",

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2025-11-05, AI: WebRTC: Report video/audio codec info and frame stats in HTTP API. v7.0.118 (#4554)
 * v7.0, 2025-11-04, AI: SRT: Report video/audio codec info and frame stats in HTTP API. v7.0.117 (#4554)
 * v7.0, 2025-11-03, Merge [#4556](https://github.com/ossrs/srs/pull/4556): Fill missing defs for H264/AVC video levels. v7.0.116 (#4556)
 * v7.0, 2025-10-31, Merge [#4547](https://github.com/ossrs/srs/pull/4547): Add ignore configuration for cursor. v7.0.115 (#4547)

trunk/src/app/srs_app_rtc_conn.cpp

@@ -1208,6 +1208,8 @@ SrsRtcPublishStream::SrsRtcPublishStream(ISrsExecRtcAsyncTask *exec, ISrsExpire
     pt_to_drop_ = 0;
 
     nn_audio_frames_ = 0;
+    nn_rtp_pkts_ = 0;
+    format_ = new SrsRtcFormat();
     twcc_enabled_ = false;
     twcc_id_ = 0;
     twcc_fb_count_ = 0;
@@ -1260,6 +1262,7 @@ SrsRtcPublishStream::~SrsRtcPublishStream()
     srs_freep(pli_worker_);
     srs_freep(twcc_epp_);
     srs_freep(pli_epp_);
+    srs_freep(format_);
     srs_freep(req_);
 
     // update the statistic when client coveried.
@@ -1284,6 +1287,10 @@ srs_error_t SrsRtcPublishStream::initialize(ISrsRequest *r, SrsRtcSourceDescript
 
     req_ = r->copy();
 
+    if ((err = format_->initialize(req_)) != srs_success) {
+        return srs_error_wrap(err, "initialize format");
+    }
+
     if ((err = timer_rtcp_->initialize()) != srs_success) {
         return srs_error_wrap(err, "initialize timer rtcp");
     }
@@ -1670,6 +1677,15 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
         return srs_error_new(ERROR_RTC_RTP, "unknown ssrc=%u", ssrc);
     }
 
+    // Report codec information to statistics on first RTP packet.
+    if ((err = format_->on_rtp_packet(track, is_audio)) != srs_success) {
+        srs_warn("RTC: format packet err %s", srs_error_desc(err).c_str());
+        srs_freep(err);
+    }
+
+    // Update RTP packet statistics.
+    update_rtp_packet_stats(is_audio);
+
     // Consume packet by track.
     if ((err = track->on_rtp(source_, pkt)) != srs_success) {
         return srs_error_wrap(err, "audio track, SSRC=%u, SEQ=%u", ssrc, pkt->header_.get_sequence());
@@ -1690,6 +1706,26 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
     return err;
 }
 
+void SrsRtcPublishStream::update_rtp_packet_stats(bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    // Count RTP packets for statistics.
+    ++nn_rtp_pkts_;
+    if (is_audio) {
+        ++nn_audio_frames_;
+    }
+
+    // Update the stat for frames, counting RTP packets as frames.
+    if (nn_rtp_pkts_ > 288) {
+        if ((err = stat_->on_video_frames(req_, (int)nn_rtp_pkts_)) != srs_success) {
+            srs_warn("RTC: stat frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_rtp_pkts_ = 0;
+    }
+}
+
 srs_error_t SrsRtcPublishStream::check_send_nacks()
 {
     srs_error_t err = srs_success;

trunk/src/app/srs_app_rtc_conn.hpp

@@ -74,6 +74,7 @@ class ISrsCoroutine;
 class ISrsDtlsCertificate;
 class SrsRtcRecvTrack;
 class ISrsRtcPlayStream;
+class ISrsRtcFormat;
 
 const uint8_t kSR = 200;
 const uint8_t kRR = 201;
@@ -542,6 +543,8 @@ class SrsRtcPublishStream : public ISrsRtcPublishStream
 SRS_DECLARE_PRIVATE: // clang-format on
     SrsContextId cid_;
     uint64_t nn_audio_frames_;
+    int nn_rtp_pkts_;
+    ISrsRtcFormat *format_;
     ISrsRtcPliWorker *pli_worker_;
     SrsErrorPithyPrint *twcc_epp_;
 
@@ -619,6 +622,7 @@ class SrsRtcPublishStream : public ISrsRtcPublishStream
 // clang-format off
 SRS_DECLARE_PRIVATE: // clang-format on
     srs_error_t do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuffer *buf);
+    void update_rtp_packet_stats(bool is_audio);
 
 public:
     srs_error_t check_send_nacks();

trunk/src/app/srs_app_rtc_source.cpp

@@ -3279,6 +3279,11 @@ std::string SrsRtcRecvTrack::get_track_id()
     return track_desc_->id_;
 }
 
+SrsRtcTrackDescription *SrsRtcRecvTrack::get_track_desc()
+{
+    return track_desc_;
+}
+
 srs_error_t SrsRtcRecvTrack::on_nack(SrsRtpPacket **ppkt)
 {
     srs_error_t err = srs_success;
@@ -3774,3 +3779,147 @@ uint32_t SrsRtcSSRCGenerator::generate_ssrc()
 
     return ++ssrc_num_;
 }
+
+ISrsRtcFormat::ISrsRtcFormat()
+{
+}
+
+ISrsRtcFormat::~ISrsRtcFormat()
+{
+}
+
+SrsRtcFormat::SrsRtcFormat()
+{
+    req_ = NULL;
+    video_codec_reported_ = false;
+    audio_codec_reported_ = false;
+
+    stat_ = _srs_stat;
+}
+
+SrsRtcFormat::~SrsRtcFormat()
+{
+    req_ = NULL;
+
+    stat_ = NULL;
+}
+
+srs_error_t SrsRtcFormat::initialize(ISrsRequest *req)
+{
+    req_ = req;
+    return srs_success;
+}
+
+srs_error_t SrsRtcFormat::on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    SrsRtcTrackDescription *track_desc = track ? track->get_track_desc() : NULL;
+    if (!req_ || !track_desc || !track_desc->media_) {
+        return err;
+    }
+
+    if (is_audio) {
+        // Only report once
+        if (audio_codec_reported_) {
+            return err;
+        }
+        audio_codec_reported_ = true;
+
+        SrsCodecPayload *media = track_desc->media_;
+        SrsAudioCodecId codec_id = (SrsAudioCodecId)media->codec(false);
+
+        // Parse channels and sample rate from track description
+        if (codec_id == SrsAudioCodecIdOpus) {
+            SrsAudioPayload *audio_media = dynamic_cast<SrsAudioPayload *>(media);
+            if (!audio_media) {
+                return err;
+            }
+
+            // Get sample rate from media payload
+            SrsAudioSampleRate sample_rate = (SrsAudioSampleRate)audio_media->sample_;
+
+            // Get channels from audio payload
+            SrsAudioChannels channels = (SrsAudioChannels)audio_media->channel_;
+
+            if ((err = stat_->on_audio_info(req_, codec_id, sample_rate, channels,
+                                             SrsAacObjectTypeReserved)) != srs_success) {
+                return srs_error_wrap(err, "stat audio info");
+            }
+            srs_trace("RTC: parsed %s codec, sample_rate=%dHz, channels=%d",
+                      srs_audio_codec_id2str(codec_id).c_str(), sample_rate, channels);
+        }
+    } else {
+        // Only report once
+        if (video_codec_reported_) {
+            return err;
+        }
+        video_codec_reported_ = true;
+
+        SrsCodecPayload *media = track_desc->media_;
+        SrsVideoCodecId codec_id = (SrsVideoCodecId)media->codec(true);
+
+        // Parse profile and level from track description
+        if (codec_id == SrsVideoCodecIdAVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h264_param_.profile_level_id_.empty()) {
+                return err;
+            }
+
+            // Parse profile_level_id hex string (e.g., "42e01f")
+            // Format: PPCCLL where PP=profile_idc, CC=constraint_set, LL=level_idc
+            std::string profile_level_id = video_media->h264_param_.profile_level_id_;
+            if (profile_level_id.length() != 6) {
+                return err;
+            }
+
+            // Decode hex string to bytes
+            // srs_hex_decode_string expects size to be the hex string length (6 chars = 3 bytes)
+            uint8_t bytes[3];
+            int hex_len = (int)profile_level_id.length();
+            int r0 = srs_hex_decode_string(bytes, profile_level_id.c_str(), hex_len);
+            if (r0 != (int)sizeof(bytes)) {
+                srs_trace("RTC: failed to decode profile_level_id hex string: %s, r0=%d", profile_level_id.c_str(), r0);
+                video_codec_reported_ = true;
+                return err;
+            }
+
+            // Extract profile and level from the decoded bytes
+            SrsAvcProfile profile = (SrsAvcProfile)bytes[0];
+            SrsAvcLevel level = (SrsAvcLevel)bytes[2];
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%s",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_avc_profile2str(profile).c_str(),
+                      srs_avc_level2str(level).c_str());
+        } else if (codec_id == SrsVideoCodecIdHEVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h265_param_.profile_id_.empty() ||
+                video_media->h265_param_.level_id_.empty()) {
+                return err;
+            }
+
+            // Parse HEVC profile_id and level_id from SDP parameters
+            // profile_id is a decimal string (e.g., "1" for Main profile)
+            // level_id is a decimal string (e.g., "93" for Level 3.1)
+            int profile_id = atoi(video_media->h265_param_.profile_id_.c_str());
+            int level_id = atoi(video_media->h265_param_.level_id_.c_str());
+
+            SrsHevcProfile profile = (SrsHevcProfile)profile_id;
+            SrsHevcLevel level = (SrsHevcLevel)level_id;
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%d",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_hevc_profile2str(profile).c_str(),
+                      level_id);
+        }
+    }
+
+    return err;
+}

trunk/src/app/srs_app_rtc_source.hpp

@@ -45,6 +45,7 @@ class ISrsRtcConsumer;
 class ISrsCircuitBreaker;
 class ISrsRtcPublishStream;
 class ISrsAppFactory;
+class ISrsStatistic;
 
 // Firefox defaults as 109, Chrome is 111.
 const int kAudioPayloadType = 111;
@@ -899,6 +900,7 @@ class SrsRtcRecvTrack
     bool set_track_status(bool active);
     bool get_track_status();
     std::string get_track_id();
+    SrsRtcTrackDescription *get_track_desc();
 
 public:
     // Note that we can set the pkt to NULL to avoid copy, for example, if the NACK cache the pkt and
@@ -1144,4 +1146,37 @@ class SrsRtcSSRCGenerator
     uint32_t generate_ssrc();
 };
 
+// The interface for RTC format.
+class ISrsRtcFormat
+{
+public:
+    ISrsRtcFormat();
+    virtual ~ISrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req) = 0;
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio) = 0;
+};
+
+// Lightweight format parser for RTC streams to extract codec information
+// from RTP packets and update statistics.
+class SrsRtcFormat : public ISrsRtcFormat
+{
+public:
+    SrsRtcFormat();
+    virtual ~SrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req);
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio);
+
+// clang-format off
+SRS_DECLARE_PRIVATE: // clang-format on
+    ISrsRequest *req_;
+    ISrsStatistic *stat_;
+    // Track whether we've already reported codec info to avoid duplicate updates
+    bool video_codec_reported_;
+    bool audio_codec_reported_;
+};
+
 #endif

trunk/src/app/srs_app_srt_conn.cpp

@@ -554,6 +554,7 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
     SrsUniquePtr<SrsPithyPrint> pprint(SrsPithyPrint::create_srt_publish());
 
     int nb_packets = 0;
+    int nb_frames = 0;
 
     // Max udp packet size equal to 1500.
     char buf[1500];
@@ -585,6 +586,15 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
         }
 
         ++nb_packets;
+        ++nb_frames;
+
+        // Update the stat for frames every 100 packets, counting SRT packets as frames.
+        if (nb_frames > 288) {
+            if ((err = stat_->on_video_frames(req_, nb_frames)) != srs_success) {
+                return srs_error_wrap(err, "srt: stat frames");
+            }
+            nb_frames = 0;
+        }
 
         if ((err = on_srt_packet(buf, nb)) != srs_success) {
             return srs_error_wrap(err, "srt: process packet");

trunk/src/core/srs_core_version7.hpp

@@ -9,6 +9,6 @@
 
 #define VERSION_MAJOR 7
 #define VERSION_MINOR 0
-#define VERSION_REVISION 117
+#define VERSION_REVISION 118
 
 #endif