AI: WebRTC: Report video/audio codec info and frame stats in HTTP API. v7.0.118 (#4554)

Author: ossrs-ai

.vscode/launch.json

@@ -57,6 +57,34 @@
             "engineLogging": true
           }
         },
+        {
+          "name": "Launch SRS with console.conf",
+          "type": "cppdbg",
+          "request": "launch",
+          "program": "${workspaceFolder}/trunk/cmake/build/srs",
+          "args": ["-c", "console.conf"],
+          "stopAtEntry": false,
+          "cwd": "${workspaceFolder}/trunk",
+          "environment": [],
+          "externalConsole": false,
+          "linux": {
+            "MIMode": "gdb"
+          },
+          "osx": {
+            "MIMode": "lldb"
+          },
+          "setupCommands": [
+            {
+              "description": "Enable pretty-printing for gdb",
+              "text": "-enable-pretty-printing",
+              "ignoreFailures": true
+            }
+          ],
+          "preLaunchTask": "build",
+          "logging": {
+            "engineLogging": true
+          }
+        },
         {
           "name": "Launch srs-proxy",
           "type": "go",

trunk/src/app/srs_app_rtc_conn.cpp

@@ -1208,6 +1208,8 @@ SrsRtcPublishStream::SrsRtcPublishStream(ISrsExecRtcAsyncTask *exec, ISrsExpire
     pt_to_drop_ = 0;
 
     nn_audio_frames_ = 0;
+    nn_rtp_pkts_ = 0;
+    format_ = new SrsRtcFormat();
     twcc_enabled_ = false;
     twcc_id_ = 0;
     twcc_fb_count_ = 0;
@@ -1260,6 +1262,7 @@ SrsRtcPublishStream::~SrsRtcPublishStream()
     srs_freep(pli_worker_);
     srs_freep(twcc_epp_);
     srs_freep(pli_epp_);
+    srs_freep(format_);
     srs_freep(req_);
 
     // update the statistic when client coveried.
@@ -1284,6 +1287,10 @@ srs_error_t SrsRtcPublishStream::initialize(ISrsRequest *r, SrsRtcSourceDescript
 
     req_ = r->copy();
 
+    if ((err = format_->initialize(req_)) != srs_success) {
+        return srs_error_wrap(err, "initialize format");
+    }
+
     if ((err = timer_rtcp_->initialize()) != srs_success) {
         return srs_error_wrap(err, "initialize timer rtcp");
     }
@@ -1670,6 +1677,15 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
         return srs_error_new(ERROR_RTC_RTP, "unknown ssrc=%u", ssrc);
     }
 
+    // Report codec information to statistics on first RTP packet.
+    if ((err = format_->on_rtp_packet(track, is_audio)) != srs_success) {
+        srs_warn("RTC: format packet err %s", srs_error_desc(err).c_str());
+        srs_freep(err);
+    }
+
+    // Update RTP packet statistics.
+    update_rtp_packet_stats(is_audio);
+
     // Consume packet by track.
     if ((err = track->on_rtp(source_, pkt)) != srs_success) {
         return srs_error_wrap(err, "audio track, SSRC=%u, SEQ=%u", ssrc, pkt->header_.get_sequence());
@@ -1690,6 +1706,26 @@ srs_error_t SrsRtcPublishStream::do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuff
     return err;
 }
 
+void SrsRtcPublishStream::update_rtp_packet_stats(bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    // Count RTP packets for statistics.
+    ++nn_rtp_pkts_;
+    if (is_audio) {
+        ++nn_audio_frames_;
+    }
+
+    // Update the stat for frames, counting RTP packets as frames.
+    if (nn_rtp_pkts_ > 288) {
+        if ((err = stat_->on_video_frames(req_, (int)nn_rtp_pkts_)) != srs_success) {
+            srs_warn("RTC: stat frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_rtp_pkts_ = 0;
+    }
+}
+
 srs_error_t SrsRtcPublishStream::check_send_nacks()
 {
     srs_error_t err = srs_success;

trunk/src/app/srs_app_rtc_conn.hpp

@@ -74,6 +74,7 @@ class ISrsCoroutine;
 class ISrsDtlsCertificate;
 class SrsRtcRecvTrack;
 class ISrsRtcPlayStream;
+class ISrsRtcFormat;
 
 const uint8_t kSR = 200;
 const uint8_t kRR = 201;
@@ -542,6 +543,8 @@ class SrsRtcPublishStream : public ISrsRtcPublishStream
 SRS_DECLARE_PRIVATE: // clang-format on
     SrsContextId cid_;
     uint64_t nn_audio_frames_;
+    int nn_rtp_pkts_;
+    ISrsRtcFormat *format_;
     ISrsRtcPliWorker *pli_worker_;
     SrsErrorPithyPrint *twcc_epp_;
 
@@ -619,6 +622,7 @@ class SrsRtcPublishStream : public ISrsRtcPublishStream
 // clang-format off
 SRS_DECLARE_PRIVATE: // clang-format on
     srs_error_t do_on_rtp_plaintext(SrsRtpPacket *&pkt, SrsBuffer *buf);
+    void update_rtp_packet_stats(bool is_audio);
 
 public:
     srs_error_t check_send_nacks();

trunk/src/app/srs_app_rtc_source.cpp

@@ -3774,3 +3774,146 @@ uint32_t SrsRtcSSRCGenerator::generate_ssrc()
 
     return ++ssrc_num_;
 }
+
+ISrsRtcFormat::ISrsRtcFormat()
+{
+}
+
+ISrsRtcFormat::~ISrsRtcFormat()
+{
+}
+
+SrsRtcFormat::SrsRtcFormat()
+{
+    req_ = NULL;
+    video_codec_reported_ = false;
+    audio_codec_reported_ = false;
+
+    stat_ = _srs_stat;
+}
+
+SrsRtcFormat::~SrsRtcFormat()
+{
+    req_ = NULL;
+
+    stat_ = NULL;
+}
+
+srs_error_t SrsRtcFormat::initialize(ISrsRequest *req)
+{
+    req_ = req;
+    return srs_success;
+}
+
+srs_error_t SrsRtcFormat::on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    if (!req_ || !track || !track->track_desc_ || !track->track_desc_->media_) {
+        return err;
+    }
+
+    if (is_audio) {
+        // Only report once
+        if (audio_codec_reported_) {
+            return err;
+        }
+        audio_codec_reported_ = true;
+
+        SrsCodecPayload *media = track->track_desc_->media_;
+        SrsAudioCodecId codec_id = (SrsAudioCodecId)media->codec(false);
+
+        // Parse channels and sample rate from track description
+        if (codec_id == SrsAudioCodecIdOpus) {
+            SrsAudioPayload *audio_media = dynamic_cast<SrsAudioPayload *>(media);
+            if (!audio_media) {
+                return err;
+            }
+
+            // Get sample rate from media payload
+            SrsAudioSampleRate sample_rate = (SrsAudioSampleRate)audio_media->sample_;
+
+            // Get channels from audio payload
+            SrsAudioChannels channels = (SrsAudioChannels)audio_media->channel_;
+
+            if ((err = stat_->on_audio_info(req_, codec_id, sample_rate, channels,
+                                             SrsAacObjectTypeReserved)) != srs_success) {
+                return srs_error_wrap(err, "stat audio info");
+            }
+            srs_trace("RTC: parsed %s codec, sample_rate=%dHz, channels=%d",
+                      srs_audio_codec_id2str(codec_id).c_str(), sample_rate, channels);
+        }
+    } else {
+        // Only report once
+        if (video_codec_reported_) {
+            return err;
+        }
+        video_codec_reported_ = true;
+
+        SrsCodecPayload *media = track->track_desc_->media_;
+        SrsVideoCodecId codec_id = (SrsVideoCodecId)media->codec(true);
+
+        // Parse profile and level from track description
+        if (codec_id == SrsVideoCodecIdAVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h264_param_.profile_level_id_.empty()) {
+                return err;
+            }
+
+            // Parse profile_level_id hex string (e.g., "42e01f")
+            // Format: PPCCLL where PP=profile_idc, CC=constraint_set, LL=level_idc
+            std::string profile_level_id = video_media->h264_param_.profile_level_id_;
+            if (profile_level_id.length() != 6) {
+                return err;
+            }
+
+            // Decode hex string to bytes
+            // srs_hex_decode_string expects size to be the hex string length (6 chars = 3 bytes)
+            uint8_t bytes[3];
+            int hex_len = (int)profile_level_id.length();
+            int r0 = srs_hex_decode_string(bytes, profile_level_id.c_str(), hex_len);
+            if (r0 != (int)sizeof(bytes)) {
+                srs_trace("RTC: failed to decode profile_level_id hex string: %s, r0=%d", profile_level_id.c_str(), r0);
+                video_codec_reported_ = true;
+                return err;
+            }
+
+            // Extract profile and level from the decoded bytes
+            SrsAvcProfile profile = (SrsAvcProfile)bytes[0];
+            SrsAvcLevel level = (SrsAvcLevel)bytes[2];
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%s",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_avc_profile2str(profile).c_str(),
+                      srs_avc_level2str(level).c_str());
+        } else if (codec_id == SrsVideoCodecIdHEVC) {
+            SrsVideoPayload *video_media = dynamic_cast<SrsVideoPayload *>(media);
+            if (!video_media || video_media->h265_param_.profile_id_.empty() ||
+                video_media->h265_param_.level_id_.empty()) {
+                return err;
+            }
+
+            // Parse HEVC profile_id and level_id from SDP parameters
+            // profile_id is a decimal string (e.g., "1" for Main profile)
+            // level_id is a decimal string (e.g., "93" for Level 3.1)
+            int profile_id = atoi(video_media->h265_param_.profile_id_.c_str());
+            int level_id = atoi(video_media->h265_param_.level_id_.c_str());
+
+            SrsHevcProfile profile = (SrsHevcProfile)profile_id;
+            SrsHevcLevel level = (SrsHevcLevel)level_id;
+
+            if ((err = stat_->on_video_info(req_, codec_id, profile, level, 0, 0)) != srs_success) {
+                return srs_error_wrap(err, "stat video info");
+            }
+            srs_trace("RTC: parsed %s codec, profile=%s, level=%d",
+                      srs_video_codec_id2str(codec_id).c_str(),
+                      srs_hevc_profile2str(profile).c_str(),
+                      level_id);
+        }
+    }
+
+    return err;
+}

trunk/src/app/srs_app_rtc_source.hpp

@@ -45,6 +45,7 @@ class ISrsRtcConsumer;
 class ISrsCircuitBreaker;
 class ISrsRtcPublishStream;
 class ISrsAppFactory;
+class ISrsStatistic;
 
 // Firefox defaults as 109, Chrome is 111.
 const int kAudioPayloadType = 111;
@@ -1144,4 +1145,37 @@ class SrsRtcSSRCGenerator
     uint32_t generate_ssrc();
 };
 
+// The interface for RTC format.
+class ISrsRtcFormat
+{
+public:
+    ISrsRtcFormat();
+    virtual ~ISrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req) = 0;
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio) = 0;
+};
+
+// Lightweight format parser for RTC streams to extract codec information
+// from RTP packets and update statistics.
+class SrsRtcFormat : public ISrsRtcFormat
+{
+public:
+    SrsRtcFormat();
+    virtual ~SrsRtcFormat();
+
+public:
+    virtual srs_error_t initialize(ISrsRequest *req);
+    virtual srs_error_t on_rtp_packet(SrsRtcRecvTrack *track, bool is_audio);
+
+// clang-format off
+SRS_DECLARE_PRIVATE: // clang-format on
+    ISrsRequest *req_;
+    ISrsStatistic *stat_;
+    // Track whether we've already reported codec info to avoid duplicate updates
+    bool video_codec_reported_;
+    bool audio_codec_reported_;
+};
+
 #endif

trunk/src/app/srs_app_srt_conn.cpp

@@ -554,6 +554,7 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
     SrsUniquePtr<SrsPithyPrint> pprint(SrsPithyPrint::create_srt_publish());
 
     int nb_packets = 0;
+    int nb_frames = 0;
 
     // Max udp packet size equal to 1500.
     char buf[1500];
@@ -585,6 +586,15 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
         }
 
         ++nb_packets;
+        ++nb_frames;
+
+        // Update the stat for frames every 100 packets, counting SRT packets as frames.
+        if (nb_frames > 288) {
+            if ((err = stat_->on_video_frames(req_, nb_frames)) != srs_success) {
+                return srs_error_wrap(err, "srt: stat frames");
+            }
+            nb_frames = 0;
+        }
 
         if ((err = on_srt_packet(buf, nb)) != srs_success) {
             return srs_error_wrap(err, "srt: process packet");

trunk/src/kernel/srs_kernel_utility.cpp

@@ -653,16 +653,19 @@ srs_error_t SrsPath::mkdir_all(string dir)
 }
 
 // fromHexChar converts a hex character into its value and a success flag.
-uint8_t srs_from_hex_char(uint8_t c)
+int srs_from_hex_char(uint8_t c, uint8_t *out)
 {
     if ('0' <= c && c <= '9') {
-        return c - '0';
+        *out = c - '0';
+        return 0;
     }
     if ('a' <= c && c <= 'f') {
-        return c - 'a' + 10;
+        *out = c - 'a' + 10;
+        return 0;
     }
     if ('A' <= c && c <= 'F') {
-        return c - 'A' + 10;
+        *out = c - 'A' + 10;
+        return 0;
     }
 
     return -1;
@@ -702,18 +705,18 @@ char *srs_hex_encode_to_string_lowercase(char *des, const u_int8_t *src, int len
 
 int srs_hex_decode_string(uint8_t *data, const char *p, int size)
 {
-    if (size <= 0 || (size % 2) == 1) {
+    if (!p || size <= 0 || (size % 2) == 1) {
         return -1;
     }
 
     for (int i = 0; i < (int)size / 2; i++) {
-        uint8_t a = srs_from_hex_char(p[i * 2]);
-        if (a == (uint8_t)-1) {
+        uint8_t a = 0;
+        if (srs_from_hex_char(p[i * 2], &a) == -1) {
             return -1;
         }
 
-        uint8_t b = srs_from_hex_char(p[i * 2 + 1]);
-        if (b == (uint8_t)-1) {
+        uint8_t b = 0;
+        if (srs_from_hex_char(p[i * 2 + 1], &b) == -1) {
             return -1;
         }