Coverage Report

Created: 2024-01-26 01:52

/work/toxav/video.c
Line
Count
Source (jump to first uncovered line)
1
/* SPDX-License-Identifier: GPL-3.0-or-later
2
 * Copyright © 2016-2018 The TokTok team.
3
 * Copyright © 2013-2015 Tox project.
4
 */
5
#include "video.h"
6
7
#include <assert.h>
8
#include <stdlib.h>
9
#include <string.h>
10
11
#include "msi.h"
12
#include "ring_buffer.h"
13
#include "rtp.h"
14
15
#include "../toxcore/ccompat.h"
16
#include "../toxcore/logger.h"
17
#include "../toxcore/mono_time.h"
18
#include "../toxcore/network.h"
19
20
/**
21
 * Soft deadline the decoder should attempt to meet, in "us" (microseconds).
22
 * Set to zero for unlimited.
23
 *
24
 * By convention, the value 1 is used to mean "return as fast as possible."
25
 */
26
// TODO(zoff99): don't hardcode this, let the application choose it
27
126
#define WANTED_MAX_DECODER_FPS 40
28
29
/**
30
 * VPX_DL_REALTIME       (1)
31
 * deadline parameter analogous to VPx REALTIME mode.
32
 *
33
 * VPX_DL_GOOD_QUALITY   (1000000)
34
 * deadline parameter analogous to VPx GOOD QUALITY mode.
35
 *
36
 * VPX_DL_BEST_QUALITY   (0)
37
 * deadline parameter analogous to VPx BEST QUALITY mode.
38
 */
39
126
#define MAX_DECODE_TIME_US (1000000 / WANTED_MAX_DECODER_FPS) // to allow x fps
40
41
/**
42
 * Codec control function to set encoder internal speed settings. Changes in
43
 * this value influences, among others, the encoder's selection of motion
44
 * estimation methods. Values greater than 0 will increase encoder speed at the
45
 * expense of quality.
46
 *
47
 * Note Valid range for VP8: `-16..16`
48
 */
49
18
#define VP8E_SET_CPUUSED_VALUE 16
50
51
/**
52
 * Initialize encoder with this value.
53
 *
54
 * Target bandwidth to use for this stream, in kilobits per second.
55
 */
56
18
#define VIDEO_BITRATE_INITIAL_VALUE 5000
57
18
#define VIDEO_DECODE_BUFFER_SIZE 5 // this buffer has normally max. 1 entry
58
59
static vpx_codec_iface_t *video_codec_decoder_interface(void)
60
18
{
61
18
    return vpx_codec_vp8_dx();
62
18
}
63
static vpx_codec_iface_t *video_codec_encoder_interface(void)
64
36
{
65
36
    return vpx_codec_vp8_cx();
66
36
}
67
68
36
#define VIDEO_CODEC_DECODER_MAX_WIDTH  800 // its a dummy value, because the struct needs a value there
69
36
#define VIDEO_CODEC_DECODER_MAX_HEIGHT 600 // its a dummy value, because the struct needs a value there
70
71
18
#define VPX_MAX_DIST_START 40
72
73
18
#define VPX_MAX_ENCODER_THREADS 4
74
18
#define VPX_MAX_DECODER_THREADS 4
75
18
#define VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED 0
76
77
static void vc_init_encoder_cfg(const Logger *log, vpx_codec_enc_cfg_t *cfg, int16_t kf_max_dist)
78
18
{
79
18
    const vpx_codec_err_t rc = vpx_codec_enc_config_default(video_codec_encoder_interface(), cfg, 0);
80
81
18
    if (rc != VPX_CODEC_OK) {
82
0
        LOGGER_ERROR(log, "vc_init_encoder_cfg:Failed to get config: %s", vpx_codec_err_to_string(rc));
83
0
    }
84
85
    /* Target bandwidth to use for this stream, in kilobits per second */
86
18
    cfg->rc_target_bitrate = VIDEO_BITRATE_INITIAL_VALUE;
87
18
    cfg->g_w = VIDEO_CODEC_DECODER_MAX_WIDTH;
88
18
    cfg->g_h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
89
18
    cfg->g_pass = VPX_RC_ONE_PASS;
90
18
    cfg->g_error_resilient = VPX_ERROR_RESILIENT_DEFAULT | VPX_ERROR_RESILIENT_PARTITIONS;
91
18
    cfg->g_lag_in_frames = 0;
92
93
    /* Allow lagged encoding
94
     *
95
     * If set, this value allows the encoder to consume a number of input
96
     * frames before producing output frames. This allows the encoder to
97
     * base decisions for the current frame on future frames. This does
98
     * increase the latency of the encoding pipeline, so it is not appropriate
99
     * in all situations (ex: realtime encoding).
100
     *
101
     * Note that this is a maximum value -- the encoder may produce frames
102
     * sooner than the given limit. Set this value to 0 to disable this
103
     * feature.
104
     */
105
18
    cfg->kf_min_dist = 0;
106
18
    cfg->kf_mode = VPX_KF_AUTO; // Encoder determines optimal placement automatically
107
18
    cfg->rc_end_usage = VPX_VBR; // what quality mode?
108
109
    /*
110
     * VPX_VBR    Variable Bit Rate (VBR) mode
111
     * VPX_CBR    Constant Bit Rate (CBR) mode
112
     * VPX_CQ     Constrained Quality (CQ) mode -> give codec a hint that we may be on low bandwidth connection
113
     * VPX_Q    Constant Quality (Q) mode
114
     */
115
18
    if (kf_max_dist > 1) {
116
0
        cfg->kf_max_dist = kf_max_dist; // a full frame every x frames minimum (can be more often, codec decides automatically)
117
0
        LOGGER_DEBUG(log, "kf_max_dist=%d (1)", cfg->kf_max_dist);
118
18
    } else {
119
18
        cfg->kf_max_dist = VPX_MAX_DIST_START;
120
18
        LOGGER_DEBUG(log, "kf_max_dist=%d (2)", cfg->kf_max_dist);
121
18
    }
122
123
18
    cfg->g_threads = VPX_MAX_ENCODER_THREADS; // Maximum number of threads to use
124
    /* TODO: set these to something reasonable */
125
    // cfg->g_timebase.num = 1;
126
    // cfg->g_timebase.den = 60; // 60 fps
127
18
    cfg->rc_resize_allowed = 1; // allow encoder to resize to smaller resolution
128
18
    cfg->rc_resize_up_thresh = 40;
129
18
    cfg->rc_resize_down_thresh = 5;
130
131
    /* TODO: make quality setting an API call, but start with normal quality */
132
#if 0
133
    /* Highest-resolution encoder settings */
134
    cfg->rc_dropframe_thresh = 0;
135
    cfg->rc_resize_allowed = 0;
136
    cfg->rc_min_quantizer = 2;
137
    cfg->rc_max_quantizer = 56;
138
    cfg->rc_undershoot_pct = 100;
139
    cfg->rc_overshoot_pct = 15;
140
    cfg->rc_buf_initial_sz = 500;
141
    cfg->rc_buf_optimal_sz = 600;
142
    cfg->rc_buf_sz = 1000;
143
#endif /* 0 */
144
18
}
145
146
VCSession *vc_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
147
                  toxav_video_receive_frame_cb *cb, void *cb_data)
148
18
{
149
18
    VCSession *vc = (VCSession *)calloc(1, sizeof(VCSession));
150
18
    vpx_codec_err_t rc;
151
152
18
    if (vc == nullptr) {
153
0
        LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
154
0
        return nullptr;
155
0
    }
156
157
18
    if (create_recursive_mutex(vc->queue_mutex) != 0) {
158
0
        LOGGER_WARNING(log, "Failed to create recursive mutex!");
159
0
        free(vc);
160
0
        return nullptr;
161
0
    }
162
163
18
    const int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
164
165
18
    vc->vbuf_raw = rb_new(VIDEO_DECODE_BUFFER_SIZE);
166
167
18
    if (vc->vbuf_raw == nullptr) {
168
0
        goto BASE_CLEANUP;
169
0
    }
170
171
    /*
172
     * VPX_CODEC_USE_FRAME_THREADING
173
     *    Enable frame-based multi-threading
174
     *
175
     * VPX_CODEC_USE_ERROR_CONCEALMENT
176
     *    Conceal errors in decoded frames
177
     */
178
18
    vpx_codec_dec_cfg_t  dec_cfg;
179
18
    dec_cfg.threads = VPX_MAX_DECODER_THREADS; // Maximum number of threads to use
180
18
    dec_cfg.w = VIDEO_CODEC_DECODER_MAX_WIDTH;
181
18
    dec_cfg.h = VIDEO_CODEC_DECODER_MAX_HEIGHT;
182
183
18
    LOGGER_DEBUG(log, "Using VP8 codec for decoder (0)");
184
18
    rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg,
185
18
                            VPX_CODEC_USE_FRAME_THREADING | VPX_CODEC_USE_POSTPROC);
186
187
18
    if (rc == VPX_CODEC_INCAPABLE) {
188
0
        LOGGER_WARNING(log, "Postproc not supported by this decoder (0)");
189
0
        rc = vpx_codec_dec_init(vc->decoder, video_codec_decoder_interface(), &dec_cfg, VPX_CODEC_USE_FRAME_THREADING);
190
0
    }
191
192
18
    if (rc != VPX_CODEC_OK) {
193
0
        LOGGER_ERROR(log, "Init video_decoder failed: %s", vpx_codec_err_to_string(rc));
194
0
        goto BASE_CLEANUP;
195
0
    }
196
197
18
    if (VIDEO_VP8_DECODER_POST_PROCESSING_ENABLED == 1) {
198
0
        vp8_postproc_cfg_t pp = {VP8_DEBLOCK, 1, 0};
199
0
        const vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
200
201
0
        if (cc_res != VPX_CODEC_OK) {
202
0
            LOGGER_WARNING(log, "Failed to turn on postproc");
203
0
        } else {
204
0
            LOGGER_DEBUG(log, "turn on postproc: OK");
205
0
        }
206
18
    } else {
207
18
        vp8_postproc_cfg_t pp = {0, 0, 0};
208
18
        vpx_codec_err_t cc_res = vpx_codec_control(vc->decoder, VP8_SET_POSTPROC, &pp);
209
210
18
        if (cc_res != VPX_CODEC_OK) {
211
0
            LOGGER_WARNING(log, "Failed to turn OFF postproc");
212
18
        } else {
213
18
            LOGGER_DEBUG(log, "Disable postproc: OK");
214
18
        }
215
18
    }
216
217
    /* Set encoder to some initial values
218
     */
219
18
    vpx_codec_enc_cfg_t  cfg;
220
18
    vc_init_encoder_cfg(log, &cfg, 1);
221
222
18
    LOGGER_DEBUG(log, "Using VP8 codec for encoder (0.1)");
223
18
    rc = vpx_codec_enc_init(vc->encoder, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
224
225
18
    if (rc != VPX_CODEC_OK) {
226
0
        LOGGER_ERROR(log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
227
0
        goto BASE_CLEANUP_1;
228
0
    }
229
230
18
    rc = vpx_codec_control(vc->encoder, VP8E_SET_CPUUSED, cpu_used_value);
231
232
18
    if (rc != VPX_CODEC_OK) {
233
0
        LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
234
0
        vpx_codec_destroy(vc->encoder);
235
0
        goto BASE_CLEANUP_1;
236
0
    }
237
238
    /*
239
     * VPX_CTRL_USE_TYPE(VP8E_SET_NOISE_SENSITIVITY,  unsigned int)
240
     * control function to set noise sensitivity
241
     *   0: off, 1: OnYOnly, 2: OnYUV, 3: OnYUVAggressive, 4: Adaptive
242
     */
243
#if 0
244
    rc = vpx_codec_control(vc->encoder, VP8E_SET_NOISE_SENSITIVITY, 2);
245
246
    if (rc != VPX_CODEC_OK) {
247
        LOGGER_ERROR(log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
248
        vpx_codec_destroy(vc->encoder);
249
        goto BASE_CLEANUP_1;
250
    }
251
252
#endif /* 0 */
253
18
    vc->linfts = current_time_monotonic(mono_time);
254
18
    vc->lcfd = 60;
255
18
    vc->vcb = cb;
256
18
    vc->vcb_user_data = cb_data;
257
18
    vc->friend_number = friend_number;
258
18
    vc->av = av;
259
18
    vc->log = log;
260
18
    return vc;
261
0
BASE_CLEANUP_1:
262
0
    vpx_codec_destroy(vc->decoder);
263
0
BASE_CLEANUP:
264
0
    pthread_mutex_destroy(vc->queue_mutex);
265
0
    rb_kill(vc->vbuf_raw);
266
0
    free(vc);
267
0
    return nullptr;
268
0
}
269
270
void vc_kill(VCSession *vc)
271
18
{
272
18
    if (vc == nullptr) {
273
0
        return;
274
0
    }
275
276
18
    vpx_codec_destroy(vc->encoder);
277
18
    vpx_codec_destroy(vc->decoder);
278
18
    void *p;
279
280
25
    while (rb_read(vc->vbuf_raw, &p)) {
281
7
        free(p);
282
7
    }
283
284
18
    rb_kill(vc->vbuf_raw);
285
18
    pthread_mutex_destroy(vc->queue_mutex);
286
18
    LOGGER_DEBUG(vc->log, "Terminated video handler: %p", (void *)vc);
287
18
    free(vc);
288
18
}
289
290
void vc_iterate(VCSession *vc)
291
280
{
292
280
    if (vc == nullptr) {
293
0
        return;
294
0
    }
295
296
280
    pthread_mutex_lock(vc->queue_mutex);
297
298
280
    struct RTPMessage *p;
299
300
280
    if (!rb_read(vc->vbuf_raw, (void **)&p)) {
301
154
        LOGGER_TRACE(vc->log, "no Video frame data available");
302
154
        pthread_mutex_unlock(vc->queue_mutex);
303
154
        return;
304
154
    }
305
306
126
    const uint16_t log_rb_size = rb_size(vc->vbuf_raw);
307
126
    pthread_mutex_unlock(vc->queue_mutex);
308
126
    const struct RTPHeader *const header = &p->header;
309
310
126
    uint32_t full_data_len;
311
312
126
    if ((header->flags & RTP_LARGE_FRAME) != 0) {
313
126
        full_data_len = header->data_length_full;
314
126
        LOGGER_DEBUG(vc->log, "vc_iterate:001:full_data_len=%d", (int)full_data_len);
315
126
    } else {
316
0
        full_data_len = p->len;
317
0
        LOGGER_DEBUG(vc->log, "vc_iterate:002");
318
0
    }
319
320
126
    LOGGER_DEBUG(vc->log, "vc_iterate: rb_read p->len=%d p->header.xe=%d", (int)full_data_len, p->header.xe);
321
126
    LOGGER_DEBUG(vc->log, "vc_iterate: rb_read rb size=%d", (int)log_rb_size);
322
126
    const vpx_codec_err_t rc = vpx_codec_decode(vc->decoder, p->data, full_data_len, nullptr, MAX_DECODE_TIME_US);
323
126
    free(p);
324
325
126
    if (rc != VPX_CODEC_OK) {
326
0
        LOGGER_ERROR(vc->log, "Error decoding video: %d %s", (int)rc, vpx_codec_err_to_string(rc));
327
0
        return;
328
0
    }
329
330
    /* Play decoded images */
331
126
    vpx_codec_iter_t iter = nullptr;
332
333
126
    for (vpx_image_t *dest = vpx_codec_get_frame(vc->decoder, &iter);
334
252
            dest != nullptr;
335
126
            dest = vpx_codec_get_frame(vc->decoder, &iter)) {
336
126
        if (vc->vcb != nullptr) {
337
126
            vc->vcb(vc->av, vc->friend_number, dest->d_w, dest->d_h,
338
126
                    dest->planes[0], dest->planes[1], dest->planes[2],
339
126
                    dest->stride[0], dest->stride[1], dest->stride[2], vc->vcb_user_data);
340
126
        }
341
342
126
        vpx_img_free(dest); // is this needed? none of the VPx examples show that
343
126
    }
344
126
}
345
346
int vc_queue_message(Mono_Time *mono_time, void *cs, struct RTPMessage *msg)
347
137
{
348
137
    VCSession *vc = (VCSession *)cs;
349
350
    /* This function is called with complete messages
351
     * they have already been assembled.
352
     * this function gets called from handle_rtp_packet() and handle_rtp_packet_v3()
353
     */
354
137
    if (vc == nullptr || msg == nullptr) {
355
0
        free(msg);
356
357
0
        return -1;
358
0
    }
359
360
137
    const struct RTPHeader *const header = &msg->header;
361
362
137
    if (msg->header.pt == (RTP_TYPE_VIDEO + 2) % 128) {
363
0
        LOGGER_WARNING(vc->log, "Got dummy!");
364
0
        free(msg);
365
0
        return 0;
366
0
    }
367
368
137
    if (msg->header.pt != RTP_TYPE_VIDEO % 128) {
369
0
        LOGGER_WARNING(vc->log, "Invalid payload type! pt=%d", (int)msg->header.pt);
370
0
        free(msg);
371
0
        return -1;
372
0
    }
373
374
137
    pthread_mutex_lock(vc->queue_mutex);
375
376
137
    if ((header->flags & RTP_LARGE_FRAME) != 0 && header->pt == RTP_TYPE_VIDEO % 128) {
377
137
        LOGGER_DEBUG(vc->log, "rb_write msg->len=%d b0=%d b1=%d", (int)msg->len, (int)msg->data[0], (int)msg->data[1]);
378
137
    }
379
380
137
    free(rb_write(vc->vbuf_raw, msg));
381
382
    /* Calculate time it took for peer to send us this frame */
383
137
    const uint32_t t_lcfd = current_time_monotonic(mono_time) - vc->linfts;
384
137
    vc->lcfd = t_lcfd > 100 ? vc->lcfd : t_lcfd;
385
137
    vc->linfts = current_time_monotonic(mono_time);
386
137
    pthread_mutex_unlock(vc->queue_mutex);
387
137
    return 0;
388
137
}
389
390
int vc_reconfigure_encoder(VCSession *vc, uint32_t bit_rate, uint16_t width, uint16_t height, int16_t kf_max_dist)
391
142
{
392
142
    if (vc == nullptr) {
393
0
        return -1;
394
0
    }
395
396
142
    vpx_codec_enc_cfg_t cfg2 = *vc->encoder->config.enc;
397
398
142
    if (cfg2.rc_target_bitrate == bit_rate && cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
399
136
        return 0; /* Nothing changed */
400
136
    }
401
402
6
    if (cfg2.g_w == width && cfg2.g_h == height && kf_max_dist == -1) {
403
        /* Only bit rate changed */
404
6
        LOGGER_INFO(vc->log, "bitrate change from: %u to: %u", (uint32_t)cfg2.rc_target_bitrate, (uint32_t)bit_rate);
405
6
        cfg2.rc_target_bitrate = bit_rate;
406
6
        const vpx_codec_err_t rc = vpx_codec_enc_config_set(vc->encoder, &cfg2);
407
408
6
        if (rc != VPX_CODEC_OK) {
409
0
            LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
410
0
            return -1;
411
0
        }
412
6
    } else {
413
        /* Resolution is changed, must reinitialize encoder since libvpx v1.4 doesn't support
414
         * reconfiguring encoder to use resolutions greater than initially set.
415
         */
416
0
        LOGGER_DEBUG(vc->log, "Have to reinitialize vpx encoder on session %p", (void *)vc);
417
0
        vpx_codec_ctx_t new_c;
418
0
        vpx_codec_enc_cfg_t  cfg;
419
0
        vc_init_encoder_cfg(vc->log, &cfg, kf_max_dist);
420
0
        cfg.rc_target_bitrate = bit_rate;
421
0
        cfg.g_w = width;
422
0
        cfg.g_h = height;
423
424
0
        LOGGER_DEBUG(vc->log, "Using VP8 codec for encoder");
425
0
        vpx_codec_err_t rc = vpx_codec_enc_init(&new_c, video_codec_encoder_interface(), &cfg, VPX_CODEC_USE_FRAME_THREADING);
426
427
0
        if (rc != VPX_CODEC_OK) {
428
0
            LOGGER_ERROR(vc->log, "Failed to initialize encoder: %s", vpx_codec_err_to_string(rc));
429
0
            return -1;
430
0
        }
431
432
0
        const int cpu_used_value = VP8E_SET_CPUUSED_VALUE;
433
434
0
        rc = vpx_codec_control(&new_c, VP8E_SET_CPUUSED, cpu_used_value);
435
436
0
        if (rc != VPX_CODEC_OK) {
437
0
            LOGGER_ERROR(vc->log, "Failed to set encoder control setting: %s", vpx_codec_err_to_string(rc));
438
0
            vpx_codec_destroy(&new_c);
439
0
            return -1;
440
0
        }
441
442
0
        vpx_codec_destroy(vc->encoder);
443
0
        memcpy(vc->encoder, &new_c, sizeof(new_c));
444
0
    }
445
446
6
    return 0;
447
6
}