AI: API: Add audio_frames and video_frames to HTTP API. v7.0.122 (#4559) (#4564)

This PR adds separate audio and video frame counting to the HTTP API
(`/api/v1/streams/`) for better stream observability. The API now
reports three frame fields:
- `frames` - Total frames (video + audio)
- `video_frames` - Video frames/packets only
- `audio_frames` - Audio frames/packets only

This enhancement provides better visibility into stream composition and
helps detect issues with CBR/VBR streams, audio/video sync problems, and
codec-specific behavior.

**Before:**
```json
{
  "streams": [
    {
      "frames": 0, // video frames.
    }
  ]
}
```

**After:**
```json
{
  "streams": [
    {
      "frames": 6912, // video frames.
      "audio_frames": 5678, // audio frames.
      "video_frames": 1234, // video frames.
    }
  ]
}
```

Frame Counting Strategy
- All protocols report frames every N frames to balance accuracy and
performance
- Frames are counted at the protocol-specific message/packet level:
  - RTMP: Counts RTMP messages (video/audio)
  - WebRTC: Counts RTP packets (video/audio)
  - SRT: Counts MPEG-TS messages (H.264/HEVC/AAC)

Author: ossrs-ai

trunk/doc/CHANGELOG.md

@@ -7,6 +7,7 @@ The changelog for SRS.
 <a name="v7-changes"></a>
 
 ## SRS 7.0 Changelog
+* v7.0, 2025-11-08, AI: API: Add audio_frames and video_frames to HTTP API. v7.0.122 (#4559)
 * v7.0, 2025-11-07, AI: WHIP: Return detailed HTTP error responses with proper status codes. v7.0.121 (#4502)
 * v7.0, 2025-11-07, AI: HLS: Support query string in hls_key_url for JWT tokens. v7.0.120 (#4426)
 * v7.0, 2025-11-07, AI: RTC: Support keep_original_ssrc to preserve SSRC and timestamps. v7.0.119 (#3850)

trunk/src/app/srs_app_recv_thread.cpp

@@ -299,6 +299,7 @@ SrsPublishRecvThread::SrsPublishRecvThread(ISrsRtmpServer *rtmp_sdk, ISrsRequest
     recv_error_ = srs_success;
     nb_msgs_ = 0;
     video_frames_ = 0;
+    audio_frames_ = 0;
     error_ = srs_cond_new();
 
     req_ = _req;
@@ -349,6 +350,11 @@ uint64_t SrsPublishRecvThread::nb_video_frames()
     return video_frames_;
 }
 
+uint64_t SrsPublishRecvThread::nb_audio_frames()
+{
+    return audio_frames_;
+}
+
 srs_error_t SrsPublishRecvThread::error_code()
 {
     return srs_error_copy(recv_error_);
@@ -396,6 +402,8 @@ srs_error_t SrsPublishRecvThread::consume(SrsRtmpCommonMessage *msg)
 
     if (msg->header_.is_video()) {
         video_frames_++;
+    } else if (msg->header_.is_audio()) {
+        audio_frames_++;
     }
 
     // log to show the time of recv thread.

trunk/src/app/srs_app_recv_thread.hpp

@@ -190,6 +190,8 @@ class SrsPublishRecvThread : public ISrsPublishRecvThread
     int64_t nb_msgs_;
     // The video frames we got.
     uint64_t video_frames_;
+    // The audio frames we got.
+    uint64_t audio_frames_;
     // For mr(merged read),
     bool mr_;
     int mr_fd_;
@@ -219,6 +221,7 @@ class SrsPublishRecvThread : public ISrsPublishRecvThread
     virtual srs_error_t wait(srs_utime_t tm);
     virtual int64_t nb_msgs();
     virtual uint64_t nb_video_frames();
+    virtual uint64_t nb_audio_frames();
     virtual srs_error_t error_code();
     virtual void set_cid(SrsContextId v);
     virtual SrsContextId get_cid();

trunk/src/app/srs_app_rtc_conn.cpp

@@ -1211,7 +1211,7 @@ SrsRtcPublishStream::SrsRtcPublishStream(ISrsExecRtcAsyncTask *exec, ISrsExpire
     pt_to_drop_ = 0;
 
     nn_audio_frames_ = 0;
-    nn_rtp_pkts_ = 0;
+    nn_video_frames_ = 0;
     format_ = new SrsRtcFormat();
     twcc_enabled_ = false;
     twcc_id_ = 0;
@@ -1714,18 +1714,28 @@ void SrsRtcPublishStream::update_rtp_packet_stats(bool is_audio)
     srs_error_t err = srs_success;
 
     // Count RTP packets for statistics.
-    ++nn_rtp_pkts_;
     if (is_audio) {
         ++nn_audio_frames_;
+    } else {
+        ++nn_video_frames_;
+    }
+
+    // Update the stat for video frames, counting RTP packets as frames.
+    if (nn_video_frames_ > 288) {
+        if ((err = stat_->on_video_frames(req_, nn_video_frames_)) != srs_success) {
+            srs_warn("RTC: stat video frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_video_frames_ = 0;
     }
 
-    // Update the stat for frames, counting RTP packets as frames.
-    if (nn_rtp_pkts_ > 288) {
-        if ((err = stat_->on_video_frames(req_, (int)nn_rtp_pkts_)) != srs_success) {
-            srs_warn("RTC: stat frames err %s", srs_error_desc(err).c_str());
+    // Update the stat for audio frames periodically.
+    if (nn_audio_frames_ > 288) {
+        if ((err = stat_->on_audio_frames(req_, nn_audio_frames_)) != srs_success) {
+            srs_warn("RTC: stat audio frames err %s", srs_error_desc(err).c_str());
             srs_freep(err);
         }
-        nn_rtp_pkts_ = 0;
+        nn_audio_frames_ = 0;
     }
 }
 

trunk/src/app/srs_app_rtc_conn.hpp

@@ -542,8 +542,8 @@ class SrsRtcPublishStream : public ISrsRtcPublishStream
 // clang-format off
 SRS_DECLARE_PRIVATE: // clang-format on
     SrsContextId cid_;
-    uint64_t nn_audio_frames_;
-    int nn_rtp_pkts_;
+    int nn_audio_frames_;
+    int nn_video_frames_;
     ISrsRtcFormat *format_;
     ISrsRtcPliWorker *pli_worker_;
     SrsErrorPithyPrint *twcc_epp_;

trunk/src/app/srs_app_rtmp_conn.cpp

@@ -988,6 +988,7 @@ srs_error_t SrsRtmpConn::do_publishing(SrsSharedPtr<SrsLiveSource> source, SrsPu
 
     int64_t nb_msgs = 0;
     uint64_t nb_frames = 0;
+    uint64_t nb_audio_frames = 0;
     while (true) {
         if ((err = trd_->pull()) != srs_success) {
             return srs_error_wrap(err, "rtmp: thread quit");
@@ -1029,6 +1030,12 @@ srs_error_t SrsRtmpConn::do_publishing(SrsSharedPtr<SrsLiveSource> source, SrsPu
         }
         nb_frames = rtrd->nb_video_frames();
 
+        // Update the stat for audio frames.
+        if ((err = stat_->on_audio_frames(req, (int)(rtrd->nb_audio_frames() - nb_audio_frames))) != srs_success) {
+            return srs_error_wrap(err, "rtmp: stat audio frames");
+        }
+        nb_audio_frames = rtrd->nb_audio_frames();
+
         // reportable
         if (pprint->can_print()) {
             kbps_->sample();

trunk/src/app/srs_app_srt_conn.cpp

@@ -554,7 +554,6 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
     SrsUniquePtr<SrsPithyPrint> pprint(SrsPithyPrint::create_srt_publish());
 
     int nb_packets = 0;
-    int nb_frames = 0;
 
     // Max udp packet size equal to 1500.
     char buf[1500];
@@ -586,15 +585,6 @@ srs_error_t SrsMpegtsSrtConn::do_publishing()
         }
 
         ++nb_packets;
-        ++nb_frames;
-
-        // Update the stat for frames every 100 packets, counting SRT packets as frames.
-        if (nb_frames > 288) {
-            if ((err = stat_->on_video_frames(req_, nb_frames)) != srs_success) {
-                return srs_error_wrap(err, "srt: stat frames");
-            }
-            nb_frames = 0;
-        }
 
         if ((err = on_srt_packet(buf, nb)) != srs_success) {
             return srs_error_wrap(err, "srt: process packet");

trunk/src/app/srs_app_srt_source.cpp

@@ -941,6 +941,8 @@ SrsSrtFormat::SrsSrtFormat()
     format_ = new SrsRtmpFormat();
     video_codec_reported_ = false;
     audio_codec_reported_ = false;
+    nn_video_frames_ = 0;
+    nn_audio_frames_ = 0;
 
     stat_ = _srs_stat;
 }
@@ -988,12 +990,45 @@ srs_error_t SrsSrtFormat::on_srt_packet(SrsSrtPacket *pkt)
     return err;
 }
 
+void SrsSrtFormat::update_ts_message_stats(bool is_audio)
+{
+    srs_error_t err = srs_success;
+
+    // Count TS messages for statistics.
+    if (is_audio) {
+        ++nn_audio_frames_;
+    } else {
+        ++nn_video_frames_;
+    }
+
+    // Update the stat for video frames, counting TS messages as frames.
+    if (nn_video_frames_ > 288) {
+        if ((err = stat_->on_video_frames(req_, nn_video_frames_)) != srs_success) {
+            srs_warn("SRT: stat video frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_video_frames_ = 0;
+    }
+
+    // Update the stat for audio frames periodically.
+    if (nn_audio_frames_ > 288) {
+        if ((err = stat_->on_audio_frames(req_, nn_audio_frames_)) != srs_success) {
+            srs_warn("SRT: stat audio frames err %s", srs_error_desc(err).c_str());
+            srs_freep(err);
+        }
+        nn_audio_frames_ = 0;
+    }
+}
+
 srs_error_t SrsSrtFormat::on_ts_message(SrsTsMessage *msg)
 {
     srs_error_t err = srs_success;
 
     // Only parse video and audio messages
     if (msg->channel_->stream_ == SrsTsStreamVideoH264 || msg->channel_->stream_ == SrsTsStreamVideoHEVC) {
+        // Update statistics for video frames
+        update_ts_message_stats(false);
+
         if (video_codec_reported_) {
             return err;
         }
@@ -1016,6 +1051,9 @@ srs_error_t SrsSrtFormat::on_ts_message(SrsTsMessage *msg)
                       c->width_, c->height_);
         }
     } else if (msg->channel_->stream_ == SrsTsStreamAudioAAC) {
+        // Update statistics for audio frames
+        update_ts_message_stats(true);
+
         if (audio_codec_reported_) {
             return err;
         }

trunk/src/app/srs_app_srt_source.hpp

@@ -179,6 +179,7 @@ class SrsSrtFormat : public ISrsSrtFormat, public ISrsTsHandler
 
 // clang-format off
 SRS_DECLARE_PRIVATE: // clang-format on
+    void update_ts_message_stats(bool is_audio);
     srs_error_t parse_video_codec(SrsTsMessage *msg);
     srs_error_t parse_audio_codec(SrsTsMessage *msg);
 
@@ -191,6 +192,9 @@ class SrsSrtFormat : public ISrsSrtFormat, public ISrsTsHandler
     // Track whether we've already reported codec info to avoid duplicate updates
     bool video_codec_reported_;
     bool audio_codec_reported_;
+    // Frame counters for statistics reporting
+    int nn_video_frames_;
+    int nn_audio_frames_;
 };
 
 // Collect and build SRT TS packet to AV frames.

trunk/src/app/srs_app_statistic.cpp

@@ -97,13 +97,15 @@ SrsStatisticStream::SrsStatisticStream()
     kbps_ = new SrsKbps();
 
     nb_clients_ = 0;
-    frames_ = new SrsPps();
+    video_frames_ = new SrsPps();
+    audio_frames_ = new SrsPps();
 }
 
 SrsStatisticStream::~SrsStatisticStream()
 {
     srs_freep(kbps_);
-    srs_freep(frames_);
+    srs_freep(video_frames_);
+    srs_freep(audio_frames_);
 }
 
 srs_error_t SrsStatisticStream::dumps(SrsJsonObject *obj)
@@ -118,7 +120,9 @@ srs_error_t SrsStatisticStream::dumps(SrsJsonObject *obj)
     obj->set("url", SrsJsonAny::str(url_.c_str()));
     obj->set("live_ms", SrsJsonAny::integer(srsu2ms(srs_time_now_cached())));
     obj->set("clients", SrsJsonAny::integer(nb_clients_));
-    obj->set("frames", SrsJsonAny::integer(frames_->sugar_));
+    obj->set("frames", SrsJsonAny::integer(video_frames_->sugar_ + audio_frames_->sugar_));
+    obj->set("audio_frames", SrsJsonAny::integer(audio_frames_->sugar_));
+    obj->set("video_frames", SrsJsonAny::integer(video_frames_->sugar_));
     obj->set("send_bytes", SrsJsonAny::integer(kbps_->get_send_bytes()));
     obj->set("recv_bytes", SrsJsonAny::integer(kbps_->get_recv_bytes()));
 
@@ -395,7 +399,19 @@ srs_error_t SrsStatistic::on_video_frames(ISrsRequest *req, int nb_frames)
     SrsStatisticVhost *vhost = create_vhost(req);
     SrsStatisticStream *stream = create_stream(vhost, req);
 
-    stream->frames_->sugar_ += nb_frames;
+    stream->video_frames_->sugar_ += nb_frames;
+
+    return err;
+}
+
+srs_error_t SrsStatistic::on_audio_frames(ISrsRequest *req, int nb_frames)
+{
+    srs_error_t err = srs_success;
+
+    SrsStatisticVhost *vhost = create_vhost(req);
+    SrsStatisticStream *stream = create_stream(vhost, req);
+
+    stream->audio_frames_->sugar_ += nb_frames;
 
     return err;
 }
@@ -542,7 +558,8 @@ void SrsStatistic::kbps_sample()
         for (it = streams_.begin(); it != streams_.end(); it++) {
             SrsStatisticStream *stream = it->second;
             stream->kbps_->sample();
-            stream->frames_->update();
+            stream->video_frames_->update();
+            stream->audio_frames_->update();
         }
     }
     if (true) {