Line | Count | Source (jump to first uncovered line) |
1 | | /* SPDX-License-Identifier: GPL-3.0-or-later |
2 | | * Copyright © 2016-2026 The TokTok team. |
3 | | * Copyright © 2013-2015 Tox project. |
4 | | */ |
5 | | #include "video.h" |
6 | | |
7 | | #include <vpx/vpx_decoder.h> |
8 | | #include <vpx/vpx_encoder.h> |
9 | | #include <vpx/vpx_image.h> |
10 | | |
11 | | #include <vpx/vp8cx.h> |
12 | | #include <vpx/vp8dx.h> |
13 | | |
14 | | #include <assert.h> |
15 | | #include <stdlib.h> |
16 | | #include <string.h> |
17 | | |
18 | | #include "ring_buffer.h" |
19 | | #include "rtp.h" |
20 | | |
21 | | #include "../toxcore/attributes.h" |
22 | | #include "../toxcore/ccompat.h" |
23 | | #include "../toxcore/logger.h" |
24 | | #include "../toxcore/mono_time.h" |
25 | | #include "../toxcore/util.h" |
26 | | |
27 | | struct VCSession { |
28 | | /* encoding */ |
29 | | vpx_codec_ctx_t encoder[1]; |
30 | | uint32_t frame_counter; |
31 | | |
32 | | vpx_image_t raw_encoder_frame; |
33 | | bool raw_encoder_frame_allocated; |
34 | | |
35 | | /* decoding */ |
36 | | vpx_codec_ctx_t decoder[1]; |
37 | | struct RingBuffer *_Nonnull vbuf_raw; /* Un-decoded data */ |
38 | | |
39 | | uint64_t linfts; /* Last received frame time stamp */ |
40 | | uint32_t lcfd; /* Last calculated frame duration for incoming video payload */ |
41 | | |
42 | | uint32_t friend_number; |
43 | | |
44 | | /* Video frame receive callback */ |
45 | | vc_video_receive_frame_cb *_Nullable vcb; |
46 | | void *_Nullable user_data; |
47 | | |
48 | | pthread_mutex_t *_Nonnull queue_mutex; |
49 | | const Logger *_Nonnull log; |
50 | | const Memory *_Nonnull mem; |
51 | | |
52 | | vpx_codec_iter_t iter; |
53 | | }; |
54 | | |
55 | | /** |
56 | | * Codec control function to set encoder internal speed settings. Changes in |
57 | | * this value influences, among others, the encoder's selection of motion |
58 | | * estimation methods. Values greater than 0 will increase encoder speed at the |
59 | | * expense of quality. |
60 | | * |
61 | | * Note Valid range for VP8: `-16..16` |
62 | | */ |
63 | 39 | #define VP8E_SET_CPUUSED_VALUE 16 |
64 | | |
65 | | /** |
66 | | * Initialize encoder with this value. |
67 | | * |
68 | | * Target bandwidth to use for this stream, in kilobits per second. |
69 | | */ |
70 | 39 | #define VIDEO_BITRATE_INITIAL_VALUE 5000 |
71 | 23 | #define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry |
72 | | |
73 | | /** |
74 | | * Security limits to prevent resource exhaustion. |
75 | | */ |
76 | 116 | #define VIDEO_MAX_FRAME_SIZE (10 * 1024 * 1024) |
77 | 199 | #define VIDEO_MAX_RESOLUTION_LIMIT 4096 |
78 | | |
79 | | static vpx_codec_iface_t *_Nonnull video_codec_decoder_interface(void) |
80 | 23 | { |
81 | 23 | return vpx_codec_vp8_dx(); |
82 | 23 | } |
83 | | static vpx_codec_iface_t *_Nonnull video_codec_encoder_interface(void) |
84 | 78 | { |
85 | 78 | return vpx_codec_vp8_cx(); |
86 | 78 | } |
87 | | |
88 | 62 | #define VIDEO_CODEC_DECODER_MAX_WIDTH 800 // its a dummy value, because the struct needs a value there |
89 | 62 | #define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there |
90 | | |
91 | 38 | #define VPX_MAX_DIST_START 40 |
92 | | |
93 | 39 | #define VPX_MAX_ENCODER_THREADS 4 |
94 | 23 | #define VPX_MAX_DECODER_THREADS 4 |
95 | 23 | #define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0 |
96 | | |
97 | | static vpx_codec_err_t vc_init_encoder_cfg(const Logger *_Nonnull log, vpx_codec_enc_cfg_t *_Nonnull cfg, |
98 | | int16_t kf_max_dist) |
99 | 39 | { |
100 | 39 | const vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0); |
101 | | |
102 | 39 | if (rc != VPX_CODEC_OK) { |
103 | 0 | LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc)); |
104 | 0 | return rc; |
105 | 0 | } |
106 | | |
107 | | /* Target bandwidth to use for this stream, in kilobits per second */ |
108 | 39 | cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE; |
109 | 39 | cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH; |
110 | 39 | cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT; |
111 | 39 | cfg->g_pass = VPX_RC_ONE_PASS; |
112 | 39 | cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS; |
113 | 39 | cfg->g_lag_in_frames = 0; |
114 | | |
115 | | /* Allow lagged encoding |
116 | | * |
117 | | * If set, this value allows the encoder to consume a number of input |
118 | | * frames before producing output frames. This allows the encoder to |
119 | | * base decisions for the current frame on future frames. This does |
120 | | * increase the latency of the encoding pipeline, so it is not appropriate |
121 | | * in all situations (ex: realtime encoding). |
122 | | * |
123 | | * Note that this is a maximum value -- the encoder may produce frames |
124 | | * sooner than the given limit. Set this value to 0 to disable this |
125 | | * feature. |
126 | | */ |
127 | 39 | cfg->kf_min_dist = 0; |
128 | 39 | cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically |
129 | 39 | cfg->rc_end_usage = VPX_VBR; // what quality mode? |
130 | | |
131 | | /* |
132 | | * VPX_VBR Variable Bit Rate (VBR) mode |
133 | | * VPX_CBR Constant Bit Rate (CBR) mode |
134 | | * VPX_CQ Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection |
135 | | * VPX_Q Constant Quality (Q) mode |
136 | | */ |
137 | 39 | if (kf_max_dist > 1) { |
138 | 1 | cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically) |
139 | 1 | LOGGER_DEBUG(log, "kf_max_dist=%u (1)", cfg->kf_max_dist); |
140 | 38 | } else { |
141 | 38 | cfg->kf_max_dist = VPX_MAX_DIST_START; |
142 | 38 | LOGGER_DEBUG(log, "kf_max_dist=%u (2)", cfg->kf_max_dist); |
143 | 38 | } |
144 | | |
145 | 39 | cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use |
146 | | /* TODO: set these to something reasonable */ |
147 | | // cfg->g_timebase.num = 1; |
148 | | // cfg->g_timebase.den = 60; // 60 fps |
149 | 39 | cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution |
150 | 39 | cfg->rc_resize_up_thresh = 40; |
151 | 39 | cfg->rc_resize_down_thresh = 5; |
152 | | |
153 | | /* TODO: make quality setting an API call, but start with normal quality */ |
154 | | #if 0 |
155 | | /* Highest-resolution encoder settings */ |
156 | | cfg->rc_dropframe_thresh = 0; |
157 | | cfg->rc_resize_allowed = 0; |
158 | | cfg->rc_min_quantizer = 2; |
159 | | cfg->rc_max_quantizer = 56; |
160 | | cfg->rc_undershoot_pct = 100; |
161 | | cfg->rc_overshoot_pct = 15; |
162 | | cfg->rc_buf_initial_sz = 500; |
163 | | cfg->rc_buf_optimal_sz = 600; |
164 | | cfg->rc_buf_sz = 1000; |
165 | | #endif /* 0 */ |
166 | | |
167 | 39 | return VPX_CODEC_OK; |
168 | 39 | } |
169 | | |
170 | | VCSession *vc_new(const Memory *mem, const Logger *log, const Mono_Time *mono_time, uint32_t friend_number, |
171 | | vc_video_receive_frame_cb *cb, void *user_data) |
172 | 23 | { |
173 | 23 | if (mono_time == nullptr) { |
174 | 0 | return nullptr; |
175 | 0 | } |
176 | | |
177 | 23 | VCSession *vc = (VCSession *)calloc(1, sizeof(VCSession)); |
178 | 23 | vpx_codec_err_t rc; |
179 | | |
180 | 23 | if (vc == nullptr) { |
181 | 0 | LOGGER_ERROR(log, "Allocation failed! Application might misbehave!"); |
182 | 0 | return nullptr; |
183 | 0 | } |
184 | | |
185 | 23 | vc->mem = mem; |
186 | | |
187 | 23 | vc->queue_mutex = (pthread_mutex_t *)mem_alloc(mem, sizeof(pthread_mutex_t)); |
188 | 23 | if (vc->queue_mutex == nullptr) { |
189 | 0 | LOGGER_ERROR(log, "Allocation failed! Application might misbehave!"); |
190 | 0 | free(vc); |
191 | 0 | return nullptr; |
192 | 0 | } |
193 | | |
194 | 23 | if (create_recursive_mutex(vc->queue_mutex) != 0) { |
195 | 0 | LOGGER_ERROR(log, "Failed to create recursive mutex!"); |
196 | 0 | mem_delete(mem, vc->queue_mutex); |
197 | 0 | free(vc); |
198 | 0 | return nullptr; |
199 | 0 | } |
200 | | |
201 | 23 | const int cpu_used_value = VP8E_SET_CPUUSED_VALUE; |
202 | | |
203 | 23 | vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE); |
204 | | |
205 | 23 | if (vc->vbuf_raw == nullptr) { |
206 | 0 | LOGGER_ERROR(log, "Failed to create ring buffer!"); |
207 | 0 | goto BASE_CLEANUP; |
208 | 0 | } |
209 | | |
210 | | /* |
211 | | * VPX_CODEC_USE_FRAME_THREADING |
212 | | * Enable frame-based multi-threading |
213 | | * |
214 | | * VPX_CODEC_USE_ERROR_CONCEALMENT |
215 | | * Conceal errors in decoded frames |
216 | | */ |
217 | 23 | vpx_codec_dec_cfg_t dec_cfg; |
218 | 23 | dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use |
219 | 23 | dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH; |
220 | 23 | dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT; |
221 | | |
222 | 23 | LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)"); |
223 | 23 | rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, |
224 | 23 | VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC); |
225 | | |
226 | 23 | if (rc == VPX_CODEC_INCAPABLE) { |
227 | 0 | LOGGER_WARNING(log, "Codec incapable of requested features, trying without postproc (0)"); |
228 | 0 | rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING); |
229 | 0 | } |
230 | | |
231 | 23 | if (rc == VPX_CODEC_INCAPABLE) { |
232 | 0 | LOGGER_WARNING(log, "Threading not supported by this decoder, trying without (0)"); |
233 | 0 | rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, 0); |
234 | 0 | } |
235 | | |
236 | 23 | if (rc != VPX_CODEC_OK) { |
237 | 0 | LOGGER_ERROR(log, "Init video_decoder failed (rc=%d): %s", (int)rc, vpx_codec_err_to_string(rc)); |
238 | 0 | goto BASE_CLEANUP; |
239 | 0 | } |
240 | | |
241 | 23 | if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) { |
242 | 0 | vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0}; |
243 | 0 | const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); |
244 | |
|
245 | 0 | if (cc_res != VPX_CODEC_OK) { |
246 | 0 | LOGGER_WARNING(log, "Failed to turn on postproc"); |
247 | 0 | } else { |
248 | 0 | LOGGER_DEBUG(log, "turn on postproc: OK"); |
249 | 0 | } |
250 | 23 | } else { |
251 | 23 | vp8_postproc_cfg_t pp = {0, 0, 0}; |
252 | 23 | const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp); |
253 | | |
254 | 23 | if (cc_res != VPX_CODEC_OK) { |
255 | 0 | LOGGER_WARNING(log, "Failed to turn OFF postproc"); |
256 | 23 | } else { |
257 | 23 | LOGGER_DEBUG(log, "Disable postproc: OK"); |
258 | 23 | } |
259 | 23 | } |
260 | | |
261 | | /* Set encoder to some initial values |
262 | | */ |
263 | 23 | vpx_codec_enc_cfg_t cfg; |
264 | 23 | rc = vc_init_encoder_cfg(log, &cfg, 1); |
265 | | |
266 | 23 | if (rc != VPX_CODEC_OK) { |
267 | 0 | LOGGER_ERROR(log, "Failed to initialize encoder config (rc=%d): %s", (int)rc, vpx_codec_err_to_string(rc)); |
268 | 0 | goto BASE_CLEANUP_1; |
269 | 0 | } |
270 | | |
271 | 23 | LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)"); |
272 | 23 | rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING); |
273 | | |
274 | 23 | if (rc == VPX_CODEC_INCAPABLE) { |
275 | 0 | LOGGER_WARNING(log, "Threading not supported by this encoder, trying without (0.1)"); |
276 | 0 | rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, 0); |
277 | 0 | } |
278 | | |
279 | 23 | if (rc != VPX_CODEC_OK) { |
280 | 0 | LOGGER_ERROR(log, "Failed to initialize encoder (rc=%d): %s", (int)rc, vpx_codec_err_to_string(rc)); |
281 | 0 | goto BASE_CLEANUP_1; |
282 | 0 | } |
283 | | |
284 | 23 | rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value); |
285 | | |
286 | 23 | if (rc != VPX_CODEC_OK) { |
287 | 0 | LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
288 | 0 | vpx_codec_destroy(vc->encoder); |
289 | 0 | goto BASE_CLEANUP_1; |
290 | 0 | } |
291 | | |
292 | | /* |
293 | | * VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY, unsigned int) |
294 | | * control function to set noise sensitivity |
295 | | * 0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive |
296 | | */ |
297 | | #if 0 |
298 | | rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2); |
299 | | |
300 | | if (rc != VPX_CODEC_OK) { |
301 | | LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
302 | | vpx_codec_destroy(vc->encoder); |
303 | | goto BASE_CLEANUP_1; |
304 | | } |
305 | | |
306 | | #endif /* 0 */ |
307 | | |
308 | 23 | vc->linfts = current_time_monotonic(mono_time); |
309 | 23 | vc->lcfd = 60; |
310 | 23 | vc->vcb = cb; |
311 | 23 | vc->user_data = user_data; |
312 | 23 | vc->friend_number = friend_number; |
313 | 23 | vc->log = log; |
314 | 23 | return vc; |
315 | | |
316 | 0 | BASE_CLEANUP_1: |
317 | 0 | vpx_codec_destroy(vc->decoder); |
318 | 0 | BASE_CLEANUP: |
319 | 0 | pthread_mutex_destroy(vc->queue_mutex); |
320 | 0 | mem_delete(vc->mem, vc->queue_mutex); |
321 | 0 | rb_kill(vc->vbuf_raw); |
322 | 0 | free(vc); |
323 | |
|
324 | 0 | return nullptr; |
325 | 0 | } |
326 | | |
327 | | void vc_kill(VCSession *vc) |
328 | 23 | { |
329 | 23 | if (vc == nullptr) { |
330 | 0 | return; |
331 | 0 | } |
332 | | |
333 | 23 | if (vc->raw_encoder_frame_allocated) { |
334 | 12 | vpx_img_free(&vc->raw_encoder_frame); |
335 | 12 | } |
336 | | |
337 | 23 | vpx_codec_destroy(vc->encoder); |
338 | 23 | vpx_codec_destroy(vc->decoder); |
339 | | |
340 | 23 | pthread_mutex_lock(vc->queue_mutex); |
341 | | |
342 | 23 | void *p; |
343 | 25 | while (rb_read(vc->vbuf_raw, &p)) { |
344 | 2 | free(p); |
345 | 2 | } |
346 | | |
347 | 23 | rb_kill(vc->vbuf_raw); |
348 | | |
349 | 23 | pthread_mutex_unlock(vc->queue_mutex); |
350 | 23 | pthread_mutex_destroy(vc->queue_mutex); |
351 | 23 | mem_delete(vc->mem, vc->queue_mutex); |
352 | 23 | LOGGER_DEBUG(vc->log, "Terminated video handler: %p", (void *)vc); |
353 | 23 | free(vc); |
354 | 23 | } |
355 | | |
356 | | void vc_iterate(VCSession *vc) |
357 | 196 | { |
358 | 196 | if (vc == nullptr) { |
359 | 0 | return; |
360 | 0 | } |
361 | | |
362 | 196 | pthread_mutex_lock(vc->queue_mutex); |
363 | | |
364 | 196 | struct RTPMessage *p; |
365 | | |
366 | 196 | if (!rb_read(vc->vbuf_raw, (void **)&p)) { |
367 | 82 | LOGGER_TRACE(vc->log, "no Video frame data available"); |
368 | 82 | pthread_mutex_unlock(vc->queue_mutex); |
369 | 82 | return; |
370 | 82 | } |
371 | | |
372 | 114 | const uint16_t log_rb_size = rb_size(vc->vbuf_raw); |
373 | 114 | pthread_mutex_unlock(vc->queue_mutex); |
374 | | |
375 | 114 | uint32_t full_data_len; |
376 | | |
377 | 114 | if ((rtp_message_flags(p) & RTP_LARGE_FRAME) != 0) { |
378 | 114 | full_data_len = rtp_message_data_length_full(p); |
379 | 114 | LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%u", full_data_len); |
380 | 114 | } else { |
381 | 0 | full_data_len = rtp_message_len(p); |
382 | 0 | LOGGER_DEBUG(vc->log, "vc_iterate:002"); |
383 | 0 | } |
384 | | |
385 | | /* Security check: Ensure the reported full data length does not exceed the actual buffer size. |
386 | | * rtp_message_len(p) returns the actual allocated payload size. |
387 | | */ |
388 | 114 | if (full_data_len > rtp_message_len(p)) { |
389 | 0 | LOGGER_ERROR(vc->log, "vc_iterate: Malicious packet detected! Lying length: %u actual: %u", |
390 | 0 | full_data_len, (uint32_t)rtp_message_len(p)); |
391 | 0 | free(p); |
392 | 0 | return; |
393 | 0 | } |
394 | | |
395 | 114 | LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%u", full_data_len); |
396 | 114 | LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)log_rb_size); |
397 | 114 | const vpx_codec_err_t rc = vpx_codec_decode(vc->decoder, rtp_message_data(p), full_data_len, nullptr, 0); |
398 | 114 | free(p); |
399 | | |
400 | 114 | if (rc != VPX_CODEC_OK) { |
401 | 0 | LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc)); |
402 | 0 | return; |
403 | 0 | } |
404 | | |
405 | | /* Play decoded images */ |
406 | 114 | vpx_codec_iter_t iter = nullptr; |
407 | | |
408 | 114 | for (vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter); |
409 | 228 | dest != nullptr; |
410 | 114 | dest = vpx_codec_get_frame(vc->decoder, &iter)) { |
411 | 114 | if (vc->vcb != nullptr) { |
412 | 114 | vc->vcb(vc->friend_number, dest->d_w, dest->d_h, |
413 | 114 | dest->planes[0], dest->planes[1], dest->planes[2], |
414 | 114 | dest->stride[0], dest->stride[1], dest->stride[2], vc->user_data); |
415 | 114 | } |
416 | 114 | } |
417 | 114 | } |
418 | | |
419 | | int vc_queue_message(const Mono_Time *mono_time, void *cs, struct RTPMessage *msg) |
420 | 117 | { |
421 | 117 | VCSession *vc = (VCSession *)cs; |
422 | | |
423 | | /* This function is called with complete messages |
424 | | * they have already been assembled. |
425 | | * this function gets called from handle_rtp_packet() |
426 | | */ |
427 | 117 | if (vc == nullptr || msg == nullptr) { |
428 | 0 | free(msg); |
429 | |
|
430 | 0 | return -1; |
431 | 0 | } |
432 | | |
433 | 117 | if (rtp_message_pt(msg) == (RTP_TYPE_VIDEO + 2) % 128) { |
434 | 1 | LOGGER_WARNING(vc->log, "Got dummy!"); |
435 | 1 | free(msg); |
436 | 1 | return 0; |
437 | 1 | } |
438 | | |
439 | 116 | if (rtp_message_pt(msg) != RTP_TYPE_VIDEO % 128) { |
440 | 0 | LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)rtp_message_pt(msg)); |
441 | 0 | free(msg); |
442 | 0 | return -1; |
443 | 0 | } |
444 | | |
445 | | /* Security check: Sanitize message size to prevent memory exhaustion */ |
446 | 116 | if (rtp_message_data_length_full(msg) > VIDEO_MAX_FRAME_SIZE) { |
447 | 0 | LOGGER_ERROR(vc->log, "Message too large! size=%u", (uint32_t)rtp_message_data_length_full(msg)); |
448 | 0 | free(msg); |
449 | 0 | return -1; |
450 | 0 | } |
451 | | |
452 | 116 | pthread_mutex_lock(vc->queue_mutex); |
453 | | |
454 | 116 | if ((rtp_message_flags(msg) & RTP_LARGE_FRAME) != 0 && rtp_message_pt(msg) == RTP_TYPE_VIDEO % 128) { |
455 | 116 | LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)rtp_message_len(msg), (int)rtp_message_data(msg)[0], (int)rtp_message_data(msg)[1]); |
456 | 116 | } |
457 | | |
458 | 116 | free(rb_write(vc->vbuf_raw, msg)); |
459 | | |
460 | | /* Calculate time it took for peer to send us this frame */ |
461 | 116 | const uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts; |
462 | 116 | vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd; |
463 | 116 | vc->linfts = current_time_monotonic(mono_time); |
464 | 116 | pthread_mutex_unlock(vc->queue_mutex); |
465 | 116 | return 0; |
466 | 116 | } |
467 | | |
468 | | int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist) |
469 | 67 | { |
470 | 67 | if (vc == nullptr) { |
471 | 0 | return -1; |
472 | 0 | } |
473 | | |
474 | | /* Security check: Sanitize resolution to prevent resource exhaustion */ |
475 | 67 | if (width == 0 || height == 0 || width > VIDEO_MAX_RESOLUTION_LIMIT || height > VIDEO_MAX_RESOLUTION_LIMIT) { |
476 | 1 | LOGGER_ERROR(vc->log, "Invalid resolution requested: %ux%u", (uint32_t)width, (uint32_t)height); |
477 | 1 | return -1; |
478 | 1 | } |
479 | | |
480 | 66 | vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc; |
481 | | |
482 | 66 | if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { |
483 | 46 | return 0; /* Nothing changed */ |
484 | 46 | } |
485 | | |
486 | 20 | if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) { |
487 | | /* Only bit rate changed */ |
488 | 4 | LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate); |
489 | 4 | cfg2.rc_target_bitrate = bit_rate; |
490 | 4 | const vpx_codec_err_t rc = vpx_codec_enc_config_set(vc->encoder, &cfg2); |
491 | | |
492 | 4 | if (rc != VPX_CODEC_OK) { |
493 | 0 | LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
494 | 0 | return -1; |
495 | 0 | } |
496 | 16 | } else { |
497 | | /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support |
498 | | * reconfiguring encoder to use resolutions greater than initially set. |
499 | | */ |
500 | 16 | LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", (void *)vc); |
501 | 16 | vpx_codec_enc_cfg_t cfg; |
502 | 16 | vpx_codec_err_t rc = vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist); |
503 | | |
504 | 16 | if (rc != VPX_CODEC_OK) { |
505 | 0 | LOGGER_ERROR(vc->log, "Failed to initialize encoder config: %s", vpx_codec_err_to_string(rc)); |
506 | 0 | return -1; |
507 | 0 | } |
508 | | |
509 | 16 | cfg.rc_target_bitrate = bit_rate; |
510 | 16 | cfg.g_w = width; |
511 | 16 | cfg.g_h = height; |
512 | | |
513 | | /* Atomic reconfiguration: Initialize new encoder first */ |
514 | 16 | vpx_codec_ctx_t new_encoder; |
515 | 16 | LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder"); |
516 | 16 | rc = vpx_codec_enc_init(&new_encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING); |
517 | | |
518 | 16 | if (rc == VPX_CODEC_INCAPABLE) { |
519 | 0 | LOGGER_WARNING(vc->log, "Threading not supported by this encoder, trying without"); |
520 | 0 | rc = vpx_codec_enc_init(&new_encoder, video_codec_encoder_interface(), &cfg, 0); |
521 | 0 | } |
522 | | |
523 | 16 | if (rc != VPX_CODEC_OK) { |
524 | 0 | LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc)); |
525 | 0 | return -1; |
526 | 0 | } |
527 | | |
528 | 16 | const int cpu_used_value = VP8E_SET_CPUUSED_VALUE; |
529 | | |
530 | 16 | rc = vpx_codec_control(&new_encoder, VP8E_SET_CPUUSED, cpu_used_value); |
531 | | |
532 | 16 | if (rc != VPX_CODEC_OK) { |
533 | 0 | LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc)); |
534 | 0 | vpx_codec_destroy(&new_encoder); |
535 | 0 | return -1; |
536 | 0 | } |
537 | | |
538 | | /* Swap only on success */ |
539 | 16 | vpx_codec_destroy(vc->encoder); |
540 | 16 | *vc->encoder = new_encoder; |
541 | 16 | return 0; |
542 | 16 | } |
543 | | |
544 | 4 | return 0; |
545 | 20 | } |
546 | | |
547 | | int vc_encode(VCSession *vc, uint16_t width, uint16_t height, const uint8_t *y, |
548 | | const uint8_t *u, const uint8_t *v, int encode_flags) |
549 | 121 | { |
550 | 121 | if (vc->raw_encoder_frame_allocated && (vc->raw_encoder_frame.d_w != width || vc->raw_encoder_frame.d_h != height)) { |
551 | 6 | vpx_img_free(&vc->raw_encoder_frame); |
552 | 6 | vc->raw_encoder_frame_allocated = false; |
553 | 6 | } |
554 | | |
555 | 121 | if (!vc->raw_encoder_frame_allocated) { |
556 | 18 | if (vpx_img_alloc(&vc->raw_encoder_frame, VPX_IMG_FMT_I420, width, height, 1) == nullptr) { |
557 | 0 | LOGGER_ERROR(vc->log, "Could not allocate image for frame"); |
558 | 0 | return -1; |
559 | 0 | } |
560 | | |
561 | 18 | vc->raw_encoder_frame_allocated = true; |
562 | 18 | } |
563 | | |
564 | 121 | vpx_image_t *img = &vc->raw_encoder_frame; |
565 | | |
566 | 121 | memcpy(img->planes[VPX_PLANE_Y], y, (size_t)width * height); |
567 | 121 | memcpy(img->planes[VPX_PLANE_U], u, ((size_t)width / 2) * (height / 2)); |
568 | 121 | memcpy(img->planes[VPX_PLANE_V], v, ((size_t)width / 2) * (height / 2)); |
569 | | |
570 | 121 | int vpx_flags = 0; |
571 | | |
572 | 121 | if ((encode_flags & VC_EFLAG_FORCE_KF) != 0) { |
573 | 29 | vpx_flags |= VPX_EFLAG_FORCE_KF; |
574 | 29 | } |
575 | | |
576 | 121 | const vpx_codec_err_t vrc = vpx_codec_encode(vc->encoder, img, |
577 | 121 | vc->frame_counter, 1, vpx_flags, VPX_DL_REALTIME); |
578 | | |
579 | 121 | if (vrc != VPX_CODEC_OK) { |
580 | 1 | LOGGER_ERROR(vc->log, "Could not encode video frame: %s", vpx_codec_err_to_string(vrc)); |
581 | 1 | return -1; |
582 | 1 | } |
583 | | |
584 | 120 | vc->iter = nullptr; |
585 | 120 | return 0; |
586 | 121 | } |
587 | | |
588 | | int vc_get_cx_data(VCSession *vc, uint8_t **data, uint32_t *size, bool *is_keyframe) |
589 | 228 | { |
590 | 228 | const vpx_codec_cx_pkt_t *pkt = vpx_codec_get_cx_data(vc->encoder, &vc->iter); |
591 | | |
592 | 228 | while (pkt != nullptr && pkt->kind != VPX_CODEC_CX_FRAME_PKT) { |
593 | 0 | pkt = vpx_codec_get_cx_data(vc->encoder, &vc->iter); |
594 | 0 | } |
595 | | |
596 | 228 | if (pkt == nullptr) { |
597 | 114 | return 0; |
598 | 114 | } |
599 | | |
600 | 114 | *data = (uint8_t *)pkt->data.frame.buf; |
601 | 114 | *size = (uint32_t)pkt->data.frame.sz; |
602 | 114 | *is_keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0; |
603 | | |
604 | 114 | return 1; |
605 | 228 | } |
606 | | |
607 | | uint32_t vc_get_lcfd(const VCSession *vc) |
608 | 131 | { |
609 | 131 | uint32_t lcfd; |
610 | 131 | pthread_mutex_lock(vc->queue_mutex); |
611 | 131 | lcfd = vc->lcfd; |
612 | 131 | pthread_mutex_unlock(vc->queue_mutex); |
613 | 131 | return lcfd; |
614 | 131 | } |
615 | | |
616 | | pthread_mutex_t *vc_get_queue_mutex(VCSession *vc) |
617 | 257 | { |
618 | 257 | return vc->queue_mutex; |
619 | 257 | } |
620 | | |
621 | | void vc_increment_frame_counter(VCSession *vc) |
622 | 114 | { |
623 | 114 | ++vc->frame_counter; |
624 | 114 | } |