Line | Count | Source (jump to first uncovered line) |
1 | | /* SPDX-License-Identifier: GPL-3.0-or-later |
2 | | * Copyright © 2016-2025 The TokTok team. |
3 | | * Copyright © 2013-2015 Tox project. |
4 | | */ |
5 | | #include "video.h" |
6 | | |
7 | | #include <assert.h> |
8 | | #include <stdlib.h> |
9 | | #include <string.h> |
10 | | |
11 | | #include "ring_buffer.h" |
12 | | #include "rtp.h" |
13 | | |
14 | | #include "../toxcore/ccompat.h" |
15 | | #include "../toxcore/logger.h" |
16 | | #include "../toxcore/mono_time.h" |
17 | | |
18 | | /** |
19 | | * Codec control function to set encoder internal speed settings. Changes in |
20 | | * this value influences, among others, the encoder's selection of motion |
21 | | * estimation methods. Values greater than 0 will increase encoder speed at the |
22 | | * expense of quality. |
23 | | * |
24 | | * Note Valid range for VP8: `-16..16` |
25 | | */ |
26 | 18 | #define VP8E_SET_CPUUSED_VALUE 16 |
27 | | |
28 | | /** |
29 | | * Initialize encoder with this value. |
30 | | * |
31 | | * Target bandwidth to use for this stream, in kilobits per second. |
32 | | */ |
33 | 18 | #define VIDEO_BITRATE_INITIAL_VALUE 5000 |
34 | 18 | #define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry |
35 | | |
36 | | static vpx_codec_iface_t *video_codec_decoder_interface(void) |
37 | 18 | { |
38 | 18 | return vpx_codec_vp8_dx(); |
39 | 18 | } |
40 | | static vpx_codec_iface_t *video_codec_encoder_interface(void) |
41 | 36 | { |
42 | 36 | return vpx_codec_vp8_cx(); |
43 | 36 | } |
44 | | |
45 | 36 | #define VIDEO_CODEC_DECODER_MAX_WIDTH 800 // its a dummy value, because the struct needs a value there |
46 | 36 | #define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there |
47 | | |
48 | 18 | #define VPX_MAX_DIST_START 40 |
49 | | |
50 | 18 | #define VPX_MAX_ENCODER_THREADS 4 |
51 | 18 | #define VPX_MAX_DECODER_THREADS 4 |
52 | 18 | #define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0 |
53 | | |
54 | | static void vc_init_encoder_cfg(const Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist) |
55 | 18 | { |
56 | 18 | const vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0); |
57 | | |
58 | 18 | if (rc != VPX_CODEC_OK) { |
59 | 0 | LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc)); |
60 | 0 | } |
61 | | |
62 | | /* Target bandwidth to use for this stream, in kilobits per second */ |
63 | 18 | cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE; |
64 | 18 | cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH; |
65 | 18 | cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT; |
66 | 18 | cfg->g_pass = VPX_RC_ONE_PASS; |
67 | 18 | cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; |
68 | 18 | cfg->g_lag_in_frames = 0; |
69 | | |
70 | | /* Allow lagged encoding |
71 | | * |
72 | | * If set, this value allows the encoder to consume a number of input |
73 | | * frames before producing output frames. This allows the encoder to |
74 | | * base decisions for the current frame on future frames. This does |
75 | | * increase the latency of the encoding pipeline, so it is not appropriate |
76 | | * in all situations (ex: realtime encoding). |
77 | | * |
78 | | * Note that this is a maximum value -- the encoder may produce frames |
79 | | * sooner than the given limit. Set this value to 0 to disable this |
80 | | * feature. |
81 | | */ |
82 | 18 | cfg->kf_min_dist = 0; |
83 | 18 | cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically |
84 | 18 | cfg->rc_end_usage = VPX_VBR; // what quality mode? |
85 | | |
86 | | /* |
87 | | * VPX_VBR Variable Bit Rate (VBR) mode |
88 | | * VPX_CBR Constant Bit Rate (CBR) mode |
89 | | * VPX_CQ Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection |
90 | | * VPX_Q Constant Quality (Q) mode |
91 | | */ |
92 | 18 | if (kf_max_dist > 1) { |
93 | 0 | cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically) |
94 | 0 | LOGGER_DEBUG(log, "kf_max_dist=%u (1)", cfg->kf_max_dist); |
95 | 18 | } else { |
96 | 18 | cfg->kf_max_dist = VPX_MAX_DIST_START; |
97 | 18 | LOGGER_DEBUG(log, "kf_max_dist=%u (2)", cfg->kf_max_dist); |
98 | 18 | } |
99 | | |
100 | 18 | cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use |
101 | | /* TODO: set these to something reasonable */ |
102 | | // cfg->g_timebase.num = 1; |
103 | | // cfg->g_timebase.den = 60; // 60 fps |
104 | 18 | cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution |
105 | 18 | cfg->rc_resize_up_thresh = 40; |
106 | 18 | cfg->rc_resize_down_thresh = 5; |
107 | | |
108 | | /* TODO: make quality setting an API call, but start with normal quality */ |
109 | | #if 0 |
110 | | /* Highest-resolution encoder settings */ |
111 | | cfg->rc_dropframe_thresh = 0; |
112 | | cfg->rc_resize_allowed = 0; |
113 | | cfg->rc_min_quantizer = 2; |
114 | | cfg->rc_max_quantizer = 56; |
115 | | cfg->rc_undershoot_pct = 100; |
116 | | cfg->rc_overshoot_pct = 15; |
117 | | cfg->rc_buf_initial_sz = 500; |
118 | | cfg->rc_buf_optimal_sz = 600; |
119 | | cfg->rc_buf_sz = 1000; |
120 | | #endif /* 0 */ |
121 | 18 | } |
122 | | |
123 | | VCSession *vc_new(const Logger *log, const Mono_Time *mono_time, ToxAV *av, uint32_t friend_number, |
124 | | toxav_video_receive_frame_cb *cb, void *cb_data) |
125 | 18 | { |
126 | 18 | VCSession *vc = (VCSession *)calloc(1, sizeof(VCSession)); |
127 | 18 | vpx_codec_err_t rc; |
128 | | |
129 | 18 | if (vc == nullptr) { |
130 | 0 | LOGGER_WARNING(log, "Allocation failed! Application might misbehave!"); |
131 | 0 | return nullptr; |
132 | 0 | } |
133 | | |
134 | 18 | if (create_recursive_mutex(vc->queue_mutex) != 0) { |
135 | 0 | LOGGER_WARNING(log, "Failed to create recursive mutex!"); |
136 | 0 | free(vc); |
137 | 0 | return nullptr; |
138 | 0 | } |
139 | | |
140 | 18 | const int cpu_used_value = VP8E_SET_CPUUSED_VALUE; |
141 | | |
142 | 18 | vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE); |
143 | | |
144 | 18 | if (vc->vbuf_raw == nullptr) { |
145 | 0 | goto BASE_CLEANUP; |
146 | 0 | } |
147 | | |
148 | | /* |
149 | | * VPX_CODEC_USE_FRAME_THREADING |
150 | | * Enable frame-based multi-threading |
151 | | * |
152 | | * VPX_CODEC_USE_ERROR_CONCEALMENT |
153 | | * Conceal errors in decoded frames |
154 | | */ |
155 | 18 | vpx_codec_dec_cfg_t dec_cfg; |
156 | 18 | dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use |
157 | 18 | dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH; |
158 | 18 | dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT; |
159 | | |
160 | 18 | LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)"); |
161 | 18 | rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, |
162 | 18 | VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC); |
163 | | |
164 | 18 | if (rc == VPX_CODEC_INCAPABLE) { |
165 | 0 | LOGGER_WARNING(log, "Postproc not supported by this decoder (0)"); |
166 | 0 | rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING); |
167 | 0 | } |
168 | | |
169 | 18 | if (rc != VPX_CODEC_OK) { |
170 | 0 | LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc)); |
171 | 0 | goto BASE_CLEANUP; |
172 | 0 | } |
173 | | |
174 | 18 | if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) { |
175 | 0 | vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0}; |
176 | 0 | const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); |
177 | |
|
178 | 0 | if (cc_res != VPX_CODEC_OK) { |
179 | 0 | LOGGER_WARNING(log, "Failed to turn on postproc"); |
180 | 0 | } else { |
181 | 0 | LOGGER_DEBUG(log, "turn on postproc: OK"); |
182 | 0 | } |
183 | 18 | } else { |
184 | 18 | vp8_postproc_cfg_t pp = {0, 0, 0}; |
185 | 18 | vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); |
186 | | |
187 | 18 | if (cc_res != VPX_CODEC_OK) { |
188 | 0 | LOGGER_WARNING(log, "Failed to turn OFF postproc"); |
189 | 18 | } else { |
190 | 18 | LOGGER_DEBUG(log, "Disable postproc: OK"); |
191 | 18 | } |
192 | 18 | } |
193 | | |
194 | | /* Set encoder to some initial values |
195 | | */ |
196 | 18 | vpx_codec_enc_cfg_t cfg; |
197 | 18 | vc_init_encoder_cfg(log, &cfg, 1); |
198 | | |
199 | 18 | LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)"); |
200 | 18 | rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING); |
201 | | |
202 | 18 | if (rc != VPX_CODEC_OK) { |
203 | 0 | LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); |
204 | 0 | goto BASE_CLEANUP_1; |
205 | 0 | } |
206 | | |
207 | 18 | rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value); |
208 | | |
209 | 18 | if (rc != VPX_CODEC_OK) { |
210 | 0 | LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
211 | 0 | vpx_codec_destroy(vc->encoder); |
212 | 0 | goto BASE_CLEANUP_1; |
213 | 0 | } |
214 | | |
215 | | /* |
216 | | * VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) |
217 | | * control function to set noise sensitivity |
218 | | * 0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive |
219 | | */ |
220 | | #if 0 |
221 | | rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2); |
222 | | |
223 | | if (rc != VPX_CODEC_OK) { |
224 | | LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
225 | | vpx_codec_destroy(vc->encoder); |
226 | | goto BASE_CLEANUP_1; |
227 | | } |
228 | | |
229 | | #endif /* 0 */ |
230 | | |
231 | 18 | vc->linfts = current_time_monotonic(mono_time); |
232 | 18 | vc->lcfd = 60; |
233 | 18 | vc->vcb = cb; |
234 | 18 | vc->vcb_user_data = cb_data; |
235 | 18 | vc->friend_number = friend_number; |
236 | 18 | vc->av = av; |
237 | 18 | vc->log = log; |
238 | 18 | return vc; |
239 | | |
240 | 0 | BASE_CLEANUP_1: |
241 | 0 | vpx_codec_destroy(vc->decoder); |
242 | 0 | BASE_CLEANUP: |
243 | 0 | pthread_mutex_destroy(vc->queue_mutex); |
244 | 0 | rb_kill(vc->vbuf_raw); |
245 | 0 | free(vc); |
246 | |
|
247 | 0 | return nullptr; |
248 | 0 | } |
249 | | |
250 | | void vc_kill(VCSession *vc) |
251 | 18 | { |
252 | 18 | if (vc == nullptr) { |
253 | 0 | return; |
254 | 0 | } |
255 | | |
256 | 18 | vpx_codec_destroy(vc->encoder); |
257 | 18 | vpx_codec_destroy(vc->decoder); |
258 | 18 | void *p; |
259 | | |
260 | 22 | while (rb_read(vc->vbuf_raw, &p)) { |
261 | 4 | free(p); |
262 | 4 | } |
263 | | |
264 | 18 | rb_kill(vc->vbuf_raw); |
265 | 18 | pthread_mutex_destroy(vc->queue_mutex); |
266 | 18 | LOGGER_DEBUG(vc->log, "Terminated video handler: %p", (void *)vc); |
267 | 18 | free(vc); |
268 | 18 | } |
269 | | |
270 | | void vc_iterate(VCSession *vc) |
271 | 210 | { |
272 | 210 | if (vc == nullptr) { |
273 | 0 | return; |
274 | 0 | } |
275 | | |
276 | 210 | pthread_mutex_lock(vc->queue_mutex); |
277 | | |
278 | 210 | struct RTPMessage *p; |
279 | | |
280 | 210 | if (!rb_read(vc->vbuf_raw, (void **)&p)) { |
281 | 116 | LOGGER_TRACE(vc->log, "no Video frame data available"); |
282 | 116 | pthread_mutex_unlock(vc->queue_mutex); |
283 | 116 | return; |
284 | 116 | } |
285 | | |
286 | 94 | const uint16_t log_rb_size = rb_size(vc->vbuf_raw); |
287 | 94 | pthread_mutex_unlock(vc->queue_mutex); |
288 | 94 | const struct RTPHeader *const header = &p->header; |
289 | | |
290 | 94 | uint32_t full_data_len; |
291 | | |
292 | 94 | if ((header->flags & RTP_LARGE_FRAME) != 0) { |
293 | 94 | full_data_len = header->data_length_full; |
294 | 94 | LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len); |
295 | 94 | } else { |
296 | 0 | full_data_len = p->len; |
297 | 0 | LOGGER_DEBUG(vc->log, "vc_iterate:002"); |
298 | 0 | } |
299 | | |
300 | 94 | LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%d p->header.xe=%d", (int)full_data_len, p->header.xe); |
301 | 94 | LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)log_rb_size); |
302 | 94 | const vpx_codec_err_t rc = vpx_codec_decode(vc->decoder, p->data, full_data_len, nullptr, 0); |
303 | 94 | free(p); |
304 | | |
305 | 94 | if (rc != VPX_CODEC_OK) { |
306 | 0 | LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc)); |
307 | 0 | return; |
308 | 0 | } |
309 | | |
310 | | /* Play decoded images */ |
311 | 94 | vpx_codec_iter_t iter = nullptr; |
312 | | |
313 | 94 | for (vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter); |
314 | 188 | dest != nullptr; |
315 | 94 | dest = vpx_codec_get_frame(vc->decoder, &iter)) { |
316 | 94 | if (vc->vcb != nullptr) { |
317 | 94 | vc->vcb(vc->av, vc->friend_number, dest->d_w, dest->d_h, |
318 | 94 | dest->planes[0], dest->planes[1], dest->planes[2], |
319 | 94 | dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb_user_data); |
320 | 94 | } |
321 | | |
322 | 94 | vpx_img_free(dest); // is this needed? none of the VPx examples show that |
323 | 94 | } |
324 | 94 | } |
325 | | |
326 | | int vc_queue_message(const Mono_Time *mono_time, void *cs, struct RTPMessage *msg) |
327 | 101 | { |
328 | 101 | VCSession *vc = (VCSession *)cs; |
329 | | |
330 | | /* This function is called with complete messages |
331 | | * they have already been assembled. |
332 | | * this function gets called from handle_rtp_packet() |
333 | | */ |
334 | 101 | if (vc == nullptr || msg == nullptr) { |
335 | 0 | free(msg); |
336 | |
|
337 | 0 | return -1; |
338 | 0 | } |
339 | | |
340 | 101 | const struct RTPHeader *const header = &msg->header; |
341 | | |
342 | 101 | if (msg->header.pt == (RTP_TYPE_VIDEO + 2) % 128) { |
343 | 0 | LOGGER_WARNING(vc->log, "Got dummy!"); |
344 | 0 | free(msg); |
345 | 0 | return 0; |
346 | 0 | } |
347 | | |
348 | 101 | if (msg->header.pt != RTP_TYPE_VIDEO % 128) { |
349 | 0 | LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt); |
350 | 0 | free(msg); |
351 | 0 | return -1; |
352 | 0 | } |
353 | | |
354 | 101 | pthread_mutex_lock(vc->queue_mutex); |
355 | | |
356 | 101 | if ((header->flags & RTP_LARGE_FRAME) != 0 && header->pt == RTP_TYPE_VIDEO % 128) { |
357 | 101 | LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]); |
358 | 101 | } |
359 | | |
360 | 101 | free(rb_write(vc->vbuf_raw, msg)); |
361 | | |
362 | | /* Calculate time it took for peer to send us this frame */ |
363 | 101 | const uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts; |
364 | 101 | vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; |
365 | 101 | vc->linfts = current_time_monotonic(mono_time); |
366 | 101 | pthread_mutex_unlock(vc->queue_mutex); |
367 | 101 | return 0; |
368 | 101 | } |
369 | | |
370 | | int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist) |
371 | 108 | { |
372 | 108 | if (vc == nullptr) { |
373 | 0 | return -1; |
374 | 0 | } |
375 | | |
376 | 108 | vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc; |
377 | | |
378 | 108 | if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { |
379 | 102 | return 0; /* Nothing changed */ |
380 | 102 | } |
381 | | |
382 | 6 | if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { |
383 | | /* Only bit rate changed */ |
384 | 6 | LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate); |
385 | 6 | cfg2.rc_target_bitrate = bit_rate; |
386 | 6 | const vpx_codec_err_t rc = vpx_codec_enc_config_set(vc->encoder, &cfg2); |
387 | | |
388 | 6 | if (rc != VPX_CODEC_OK) { |
389 | 0 | LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
390 | 0 | return -1; |
391 | 0 | } |
392 | 6 | } else { |
393 | | /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support |
394 | | * reconfiguring encoder to use resolutions greater than initially set. |
395 | | */ |
396 | 0 | LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", (void *)vc); |
397 | 0 | vpx_codec_ctx_t new_c; |
398 | 0 | vpx_codec_enc_cfg_t cfg; |
399 | 0 | vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist); |
400 | 0 | cfg.rc_target_bitrate = bit_rate; |
401 | 0 | cfg.g_w = width; |
402 | 0 | cfg.g_h = height; |
403 | |
|
404 | 0 | LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder"); |
405 | 0 | vpx_codec_err_t rc = vpx_codec_enc_init(&new_c, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING); |
406 | |
|
407 | 0 | if (rc != VPX_CODEC_OK) { |
408 | 0 | LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); |
409 | 0 | return -1; |
410 | 0 | } |
411 | | |
412 | 0 | const int cpu_used_value = VP8E_SET_CPUUSED_VALUE; |
413 | |
|
414 | 0 | rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, cpu_used_value); |
415 | |
|
416 | 0 | if (rc != VPX_CODEC_OK) { |
417 | 0 | LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
418 | 0 | vpx_codec_destroy(&new_c); |
419 | 0 | return -1; |
420 | 0 | } |
421 | | |
422 | 0 | vpx_codec_destroy(vc->encoder); |
423 | 0 | memcpy(vc->encoder, &new_c, sizeof(new_c)); |
424 | 0 | } |
425 | | |
426 | 6 | return 0; |
427 | 6 | } |