Coverage Report

Created: 2025-10-08 19:34

/work/toxav/toxav.c
Line
Count
Source (jump to first uncovered line)
1
/* SPDX-License-Identifier: GPL-3.0-or-later
2
 * Copyright © 2016-2025 The TokTok team.
3
 * Copyright © 2013-2015 Tox project.
4
 */
5
#include "toxav.h"
6
7
#include <assert.h>
8
#include <limits.h>
9
#include <stdlib.h>
10
#include <string.h>
11
12
#include "msi.h"
13
#include "rtp.h"
14
#include "toxav_hacks.h"
15
16
#include "../toxcore/Messenger.h"
17
#include "../toxcore/ccompat.h"
18
#include "../toxcore/logger.h"
19
#include "../toxcore/mono_time.h"
20
#include "../toxcore/net_crypto.h"
21
#include "../toxcore/network.h"
22
#include "../toxcore/tox.h"
23
#include "../toxcore/tox_private.h"
24
#include "../toxcore/tox_struct.h"  // IWYU pragma: keep
25
#include "../toxcore/util.h"
26
27
174
#define VIDEO_SEND_X_KEYFRAMES_FIRST 7 // force the first n frames to be keyframes!
28
29
// iteration interval that is used when no call is active
30
276
#define IDLE_ITERATION_INTERVAL_MS 1000
31
32
#ifndef TOXAV_CALL_DEFINED
33
#define TOXAV_CALL_DEFINED
34
typedef struct ToxAVCall ToxAVCall;
35
#endif /* TOXAV_CALL_DEFINED */
36
37
struct ToxAVCall {
38
    ToxAV *av;
39
40
    pthread_mutex_t mutex_audio[1];
41
    RTPSession *audio_rtp;
42
    ACSession *audio;
43
44
    pthread_mutex_t mutex_video[1];
45
    RTPSession *video_rtp;
46
    VCSession *video;
47
48
    BWController *bwc;
49
50
    bool active;
51
    MSICall *msi_call;
52
    uint32_t friend_number;
53
54
    uint32_t audio_bit_rate; /* Sending audio bit rate */
55
    uint32_t video_bit_rate; /* Sending video bit rate */
56
57
    /** Required for monitoring changes in states */
58
    uint8_t previous_self_capabilities;
59
60
    pthread_mutex_t toxav_call_mutex[1];
61
62
    struct ToxAVCall *prev;
63
    struct ToxAVCall *next;
64
};
65
66
/** Decode time statistics */
67
typedef struct DecodeTimeStats {
68
    /** Measure count */
69
    int32_t count;
70
    /** Last cycle total */
71
    int32_t total;
72
    /** Average decoding time in ms */
73
    int32_t average;
74
75
    /** Calculated iteration interval */
76
    uint32_t interval;
77
} DecodeTimeStats;
78
79
struct ToxAV {
80
    const struct Tox_Memory *mem;
81
    Logger *log;
82
    Tox *tox;
83
    MSISession *msi;
84
85
    /* Two-way storage: first is array of calls and second is list of calls with head and tail */
86
    ToxAVCall **calls;
87
    uint32_t calls_tail;
88
    uint32_t calls_head;
89
    pthread_mutex_t mutex[1];
90
91
    /* Call callback */
92
    toxav_call_cb *ccb;
93
    void *ccb_user_data;
94
    /* Call state callback */
95
    toxav_call_state_cb *scb;
96
    void *scb_user_data;
97
    /* Audio frame receive callback */
98
    toxav_audio_receive_frame_cb *acb;
99
    void *acb_user_data;
100
    /* Video frame receive callback */
101
    toxav_video_receive_frame_cb *vcb;
102
    void *vcb_user_data;
103
    /* Bit rate control callback */
104
    toxav_audio_bit_rate_cb *abcb;
105
    void *abcb_user_data;
106
    /* Bit rate control callback */
107
    toxav_video_bit_rate_cb *vbcb;
108
    void *vbcb_user_data;
109
110
    /* keep track of decode times for audio and video */
111
    DecodeTimeStats audio_stats;
112
    DecodeTimeStats video_stats;
113
114
    Mono_Time *toxav_mono_time; // ToxAV's own mono_time instance
115
};
116
117
static void callback_bwc(BWController *bwc, uint32_t friend_number, float loss, void *user_data);
118
119
static int callback_invite(void *object, MSICall *call);
120
static int callback_start(void *object, MSICall *call);
121
static int callback_end(void *object, MSICall *call);
122
static int callback_error(void *object, MSICall *call);
123
static int callback_capabilites(void *object, MSICall *call);
124
125
static bool audio_bit_rate_invalid(uint32_t bit_rate);
126
static bool video_bit_rate_invalid(uint32_t bit_rate);
127
static bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t state);
128
static ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, Toxav_Err_Call *error);
129
static ToxAVCall *call_remove(ToxAVCall *call);
130
static bool call_prepare_transmission(ToxAVCall *call);
131
static void call_kill_transmission(ToxAVCall *call);
132
133
MSISession *tox_av_msi_get(const ToxAV *av)
134
44
{
135
44
    if (av == nullptr) {
136
0
        return nullptr;
137
0
    }
138
139
44
    return av->msi;
140
44
}
141
142
ToxAVCall *call_get(ToxAV *av, uint32_t friend_number)
143
2.76k
{
144
2.76k
    if (av == nullptr) {
145
0
        return nullptr;
146
0
    }
147
148
    /* Assumes mutex locked */
149
2.76k
    if (av->calls == nullptr || av->calls_tail < friend_number) {
150
1.78k
        return nullptr;
151
1.78k
    }
152
153
984
    return av->calls[friend_number];
154
2.76k
}
155
156
RTPSession *rtp_session_get(ToxAVCall *call, int payload_type)
157
250
{
158
250
    if (call == nullptr) {
159
0
        return nullptr;
160
0
    }
161
162
250
    if (payload_type == RTP_TYPE_VIDEO) {
163
143
        return call->video_rtp;
164
143
    } else {
165
107
        return call->audio_rtp;
166
107
    }
167
250
}
168
169
BWController *bwc_controller_get(const ToxAVCall *call)
170
0
{
171
0
    if (call == nullptr) {
172
0
        return nullptr;
173
0
    }
174
175
0
    return call->bwc;
176
0
}
177
178
/**
179
 * @brief initialize d with default values
180
 * @param d struct to be initialized, must not be nullptr
181
 */
182
static void init_decode_time_stats(DecodeTimeStats *d)
183
12
{
184
12
    assert(d != nullptr);
185
12
    d->count = 0;
186
12
    d->total = 0;
187
12
    d->average = 0;
188
12
    d->interval = IDLE_ITERATION_INTERVAL_MS;
189
12
}
190
191
ToxAV *toxav_new(Tox *tox, Toxav_Err_New *error)
192
6
{
193
6
    Toxav_Err_New rc = TOXAV_ERR_NEW_OK;
194
6
    ToxAV *av = nullptr;
195
196
6
    if (tox == nullptr) {
197
0
        rc = TOXAV_ERR_NEW_NULL;
198
0
        goto RETURN;
199
0
    }
200
201
6
    av = (ToxAV *)calloc(1, sizeof(ToxAV));
202
203
6
    if (av == nullptr) {
204
0
        rc = TOXAV_ERR_NEW_MALLOC;
205
0
        goto RETURN;
206
0
    }
207
208
6
    if (create_recursive_mutex(av->mutex) != 0) {
209
0
        rc = TOXAV_ERR_NEW_MALLOC;
210
0
        goto RETURN;
211
0
    }
212
213
6
    av->mem = tox->sys.mem;
214
6
    av->log = tox->m->log;
215
6
    av->tox = tox;
216
6
    av->msi = msi_new(av->log, av->tox);
217
218
6
    rtp_allow_receiving(av->tox);
219
6
    bwc_allow_receiving(av->tox);
220
221
6
    av->toxav_mono_time = mono_time_new(tox->sys.mem, nullptr, nullptr);
222
223
6
    if (av->msi == nullptr) {
224
0
        pthread_mutex_destroy(av->mutex);
225
0
        rc = TOXAV_ERR_NEW_MALLOC;
226
0
        goto RETURN;
227
0
    }
228
229
6
    init_decode_time_stats(&av->audio_stats);
230
6
    init_decode_time_stats(&av->video_stats);
231
6
    av->msi->av = av;
232
233
    // save ToxAV object into toxcore
234
6
    tox_set_av_object(av->tox, av);
235
236
6
    msi_callback_invite(av->msi, callback_invite);
237
6
    msi_callback_start(av->msi, callback_start);
238
6
    msi_callback_end(av->msi, callback_end);
239
6
    msi_callback_error(av->msi, callback_error);
240
6
    msi_callback_peertimeout(av->msi, callback_error);
241
6
    msi_callback_capabilities(av->msi, callback_capabilites);
242
243
6
RETURN:
244
245
6
    if (error != nullptr) {
246
6
        *error = rc;
247
6
    }
248
249
6
    if (rc != TOXAV_ERR_NEW_OK) {
250
0
        if (av != nullptr) {
251
0
            free(av);
252
0
            av = nullptr;
253
0
        }
254
0
    }
255
256
6
    return av;
257
6
}
258
259
void toxav_kill(ToxAV *av)
260
6
{
261
6
    if (av == nullptr) {
262
0
        return;
263
0
    }
264
265
6
    pthread_mutex_lock(av->mutex);
266
267
    // unregister callbacks
268
54
    for (uint8_t i = PACKET_ID_RANGE_LOSSY_AV_START; i <= PACKET_ID_RANGE_LOSSY_AV_END; ++i) {
269
48
        tox_callback_friend_lossy_packet_per_pktid(av->tox, nullptr, i);
270
48
    }
271
272
6
    rtp_stop_receiving(av->tox);
273
6
    bwc_stop_receiving(av->tox);
274
275
    /* To avoid possible deadlocks */
276
6
    while (av->msi != nullptr && msi_kill(av->log, av->tox, av->msi) != 0) {
277
0
        pthread_mutex_unlock(av->mutex);
278
0
        pthread_mutex_lock(av->mutex);
279
0
    }
280
281
    /* Msi kill will hang up all calls so just clean these calls */
282
6
    if (av->calls != nullptr) {
283
0
        ToxAVCall *it = call_get(av, av->calls_head);
284
285
0
        while (it != nullptr) {
286
0
            call_kill_transmission(it);
287
0
            it->msi_call = nullptr; /* msi_kill() frees the call's msi_call handle; which causes #278 */
288
0
            it = call_remove(it); /* This will eventually free av->calls */
289
0
        }
290
0
    }
291
292
6
    mono_time_free(av->tox->sys.mem, av->toxav_mono_time);
293
294
6
    pthread_mutex_unlock(av->mutex);
295
6
    pthread_mutex_destroy(av->mutex);
296
297
    // set ToxAV object to NULL in toxcore, to signal ToxAV has been shutdown
298
6
    tox_set_av_object(av->tox, nullptr);
299
300
6
    free(av);
301
6
}
302
303
Tox *toxav_get_tox(const ToxAV *av)
304
0
{
305
0
    return av->tox;
306
0
}
307
308
const Logger *toxav_get_logger(const ToxAV *av)
309
304
{
310
304
    return av->log;
311
304
}
312
313
uint32_t toxav_audio_iteration_interval(const ToxAV *av)
314
0
{
315
0
    return av->calls != nullptr ? av->audio_stats.interval : IDLE_ITERATION_INTERVAL_MS;
316
0
}
317
318
uint32_t toxav_video_iteration_interval(const ToxAV *av)
319
0
{
320
0
    return av->calls != nullptr ? av->video_stats.interval : IDLE_ITERATION_INTERVAL_MS;
321
0
}
322
323
uint32_t toxav_iteration_interval(const ToxAV *av)
324
0
{
325
0
    return min_u32(toxav_audio_iteration_interval(av),
326
0
                   toxav_video_iteration_interval(av));
327
0
}
328
329
/**
330
 * @brief calc_interval Calculates the needed iteration interval based on previous decode times
331
 * @param av ToxAV struct to work on
332
 * @param stats Statistics to update
333
 * @param frame_time the duration of the current frame in ms
334
 * @param start_time the timestamp when decoding of this frame started
335
 */
336
static void calc_interval(const ToxAV *av, DecodeTimeStats *stats, int32_t frame_time, uint64_t start_time)
337
264
{
338
264
    stats->interval = frame_time < stats->average ? 0 : (frame_time - stats->average);
339
264
    stats->total += current_time_monotonic(av->toxav_mono_time) - start_time;
340
341
264
    if (++stats->count == 3) {
342
        /* NOTE: Magic Offset for precision */
343
86
        stats->average = stats->total / 3 + 5;
344
86
        stats->count = 0;
345
86
        stats->total = 0;
346
86
    }
347
264
}
348
349
/**
350
 * @brief common iterator function for audio and video calls
351
 * @param av pointer to ToxAV structure of current instance
352
 * @param audio if true, iterate audio, video else
353
 */
354
static void iterate_common(ToxAV *av, bool audio)
355
2.01k
{
356
2.01k
    pthread_mutex_lock(av->mutex);
357
358
2.01k
    if (av->calls == nullptr) {
359
1.75k
        pthread_mutex_unlock(av->mutex);
360
1.75k
        return;
361
1.75k
    }
362
363
264
    const uint64_t start = current_time_monotonic(av->toxav_mono_time);
364
264
    int32_t frame_time = IDLE_ITERATION_INTERVAL_MS;
365
366
784
    for (ToxAVCall *i = av->calls[av->calls_head]; i != nullptr; i = i->next) {
367
520
        if (!i->active) {
368
100
            continue;
369
100
        }
370
371
420
        pthread_mutex_lock(i->toxav_call_mutex);
372
420
        pthread_mutex_unlock(av->mutex);
373
374
420
        const uint32_t fid = i->friend_number;
375
420
        const bool is_offline = check_peer_offline_status(av->log, av->tox, i->msi_call->session, fid);
376
377
420
        if (is_offline) {
378
0
            pthread_mutex_unlock(i->toxav_call_mutex);
379
0
            pthread_mutex_lock(av->mutex);
380
0
            break;
381
0
        }
382
383
420
        if (audio) {
384
210
            ac_iterate(i->audio);
385
386
210
            if ((i->msi_call->self_capabilities & MSI_CAP_R_AUDIO) != 0 &&
387
210
                    (i->msi_call->peer_capabilities & MSI_CAP_S_AUDIO) != 0) {
388
210
                frame_time = min_s32(i->audio->lp_frame_duration, frame_time);
389
210
            }
390
210
        } else {
391
210
            vc_iterate(i->video);
392
393
210
            if ((i->msi_call->self_capabilities & MSI_CAP_R_VIDEO) != 0 &&
394
210
                    (i->msi_call->peer_capabilities & MSI_CAP_S_VIDEO) != 0) {
395
210
                pthread_mutex_lock(i->video->queue_mutex);
396
210
                frame_time = min_s32(i->video->lcfd, frame_time);
397
210
                pthread_mutex_unlock(i->video->queue_mutex);
398
210
            }
399
210
        }
400
401
420
        pthread_mutex_unlock(i->toxav_call_mutex);
402
420
        pthread_mutex_lock(av->mutex);
403
404
        /* In case this call is popped from container stop iteration */
405
420
        if (call_get(av, fid) != i) {
406
0
            break;
407
0
        }
408
420
    }
409
410
264
    DecodeTimeStats *stats = audio ? &av->audio_stats : &av->video_stats;
411
264
    calc_interval(av, stats, frame_time, start);
412
413
264
    pthread_mutex_unlock(av->mutex);
414
264
}
415
416
void toxav_audio_iterate(ToxAV *av)
417
1.00k
{
418
1.00k
    iterate_common(av, true);
419
1.00k
}
420
421
void toxav_video_iterate(ToxAV *av)
422
1.00k
{
423
1.00k
    iterate_common(av, false);
424
1.00k
}
425
426
void toxav_iterate(ToxAV *av)
427
1.00k
{
428
1.00k
    toxav_audio_iterate(av);
429
1.00k
    toxav_video_iterate(av);
430
1.00k
}
431
432
bool toxav_call(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
433
                Toxav_Err_Call *error)
434
11
{
435
11
    Toxav_Err_Call rc = TOXAV_ERR_CALL_OK;
436
11
    ToxAVCall *call;
437
438
11
    pthread_mutex_lock(av->mutex);
439
440
11
    if ((audio_bit_rate != 0 && audio_bit_rate_invalid(audio_bit_rate))
441
11
            || (video_bit_rate != 0 && video_bit_rate_invalid(video_bit_rate))) {
442
0
        rc = TOXAV_ERR_CALL_INVALID_BIT_RATE;
443
0
        goto RETURN;
444
0
    }
445
446
11
    call = call_new(av, friend_number, &rc);
447
448
11
    if (call == nullptr) {
449
0
        goto RETURN;
450
0
    }
451
452
11
    call->audio_bit_rate = audio_bit_rate;
453
11
    call->video_bit_rate = video_bit_rate;
454
455
11
    call->previous_self_capabilities = MSI_CAP_R_AUDIO | MSI_CAP_R_VIDEO;
456
457
11
    call->previous_self_capabilities |= audio_bit_rate > 0 ? MSI_CAP_S_AUDIO : 0;
458
11
    call->previous_self_capabilities |= video_bit_rate > 0 ? MSI_CAP_S_VIDEO : 0;
459
460
11
    if (msi_invite(av->log, av->msi, &call->msi_call, friend_number, call->previous_self_capabilities) != 0) {
461
0
        call_remove(call);
462
0
        rc = TOXAV_ERR_CALL_SYNC;
463
0
        goto RETURN;
464
0
    }
465
466
11
    call->msi_call->av_call = call;
467
468
11
RETURN:
469
11
    pthread_mutex_unlock(av->mutex);
470
471
11
    if (error != nullptr) {
472
11
        *error = rc;
473
11
    }
474
475
11
    return rc == TOXAV_ERR_CALL_OK;
476
11
}
477
478
void toxav_callback_call(ToxAV *av, toxav_call_cb *callback, void *user_data)
479
6
{
480
6
    pthread_mutex_lock(av->mutex);
481
6
    av->ccb = callback;
482
6
    av->ccb_user_data = user_data;
483
6
    pthread_mutex_unlock(av->mutex);
484
6
}
485
486
bool toxav_answer(ToxAV *av, uint32_t friend_number, uint32_t audio_bit_rate, uint32_t video_bit_rate,
487
                  Toxav_Err_Answer *error)
488
9
{
489
9
    pthread_mutex_lock(av->mutex);
490
491
9
    Toxav_Err_Answer rc = TOXAV_ERR_ANSWER_OK;
492
9
    ToxAVCall *call;
493
494
9
    if (!tox_friend_exists(av->tox, friend_number)) {
495
0
        rc = TOXAV_ERR_ANSWER_FRIEND_NOT_FOUND;
496
0
        goto RETURN;
497
0
    }
498
499
9
    if ((audio_bit_rate != 0 && audio_bit_rate_invalid(audio_bit_rate))
500
9
            || (video_bit_rate != 0 && video_bit_rate_invalid(video_bit_rate))
501
9
       ) {
502
0
        rc = TOXAV_ERR_ANSWER_INVALID_BIT_RATE;
503
0
        goto RETURN;
504
0
    }
505
506
9
    call = call_get(av, friend_number);
507
508
9
    if (call == nullptr) {
509
0
        rc = TOXAV_ERR_ANSWER_FRIEND_NOT_CALLING;
510
0
        goto RETURN;
511
0
    }
512
513
9
    if (!call_prepare_transmission(call)) {
514
0
        rc = TOXAV_ERR_ANSWER_CODEC_INITIALIZATION;
515
0
        goto RETURN;
516
0
    }
517
518
9
    call->audio_bit_rate = audio_bit_rate;
519
9
    call->video_bit_rate = video_bit_rate;
520
521
9
    call->previous_self_capabilities = MSI_CAP_R_AUDIO | MSI_CAP_R_VIDEO;
522
523
9
    call->previous_self_capabilities |= audio_bit_rate > 0 ? MSI_CAP_S_AUDIO : 0;
524
9
    call->previous_self_capabilities |= video_bit_rate > 0 ? MSI_CAP_S_VIDEO : 0;
525
526
9
    if (msi_answer(av->log, call->msi_call, call->previous_self_capabilities) != 0) {
527
0
        rc = TOXAV_ERR_ANSWER_SYNC;
528
0
    }
529
530
9
RETURN:
531
9
    pthread_mutex_unlock(av->mutex);
532
533
9
    if (error != nullptr) {
534
9
        *error = rc;
535
9
    }
536
537
9
    return rc == TOXAV_ERR_ANSWER_OK;
538
9
}
539
540
void toxav_callback_call_state(ToxAV *av, toxav_call_state_cb *callback, void *user_data)
541
6
{
542
6
    pthread_mutex_lock(av->mutex);
543
6
    av->scb = callback;
544
6
    av->scb_user_data = user_data;
545
6
    pthread_mutex_unlock(av->mutex);
546
6
}
547
548
static Toxav_Err_Call_Control call_control_handle_resume(const ToxAVCall *call)
549
2
{
550
    /* Only act if paused and had media transfer active before */
551
2
    if (call->msi_call->self_capabilities != 0 || call->previous_self_capabilities == 0) {
552
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
553
0
    }
554
555
2
    if (msi_change_capabilities(call->av->log, call->msi_call,
556
2
                                call->previous_self_capabilities) == -1) {
557
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
558
0
    }
559
560
2
    rtp_allow_receiving_mark(call->audio_rtp);
561
2
    rtp_allow_receiving_mark(call->video_rtp);
562
563
2
    return TOXAV_ERR_CALL_CONTROL_OK;
564
2
}
565
static Toxav_Err_Call_Control call_control_handle_pause(ToxAVCall *call)
566
2
{
567
    /* Only act if not already paused */
568
2
    if (call->msi_call->self_capabilities == 0) {
569
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
570
0
    }
571
572
2
    call->previous_self_capabilities = call->msi_call->self_capabilities;
573
574
2
    if (msi_change_capabilities(call->av->log, call->msi_call, 0) == -1) {
575
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
576
0
    }
577
578
2
    rtp_stop_receiving_mark(call->audio_rtp);
579
2
    rtp_stop_receiving_mark(call->video_rtp);
580
581
2
    return TOXAV_ERR_CALL_CONTROL_OK;
582
2
}
583
static Toxav_Err_Call_Control call_control_handle_cancel(ToxAVCall *call)
584
11
{
585
    /* Hang up */
586
11
    pthread_mutex_lock(call->toxav_call_mutex);
587
588
11
    if (msi_hangup(call->av->log, call->msi_call) != 0) {
589
0
        pthread_mutex_unlock(call->toxav_call_mutex);
590
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
591
0
    }
592
593
11
    call->msi_call = nullptr;
594
11
    pthread_mutex_unlock(call->toxav_call_mutex);
595
596
    /* No matter the case, terminate the call */
597
11
    call_kill_transmission(call);
598
11
    call_remove(call);
599
600
11
    return TOXAV_ERR_CALL_CONTROL_OK;
601
11
}
602
static Toxav_Err_Call_Control call_control_handle_mute_audio(const ToxAVCall *call)
603
2
{
604
2
    if ((call->msi_call->self_capabilities & MSI_CAP_R_AUDIO) == 0) {
605
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
606
0
    }
607
608
2
    if (msi_change_capabilities(call->av->log, call->msi_call, call->
609
2
                                msi_call->self_capabilities ^ MSI_CAP_R_AUDIO) == -1) {
610
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
611
612
0
    }
613
614
2
    rtp_stop_receiving_mark(call->audio_rtp);
615
2
    return TOXAV_ERR_CALL_CONTROL_OK;
616
2
}
617
static Toxav_Err_Call_Control call_control_handle_unmute_audio(const ToxAVCall *call)
618
2
{
619
2
    if ((call->msi_call->self_capabilities ^ MSI_CAP_R_AUDIO) == 0) {
620
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
621
0
    }
622
623
2
    if (msi_change_capabilities(call->av->log, call->msi_call, call->
624
2
                                msi_call->self_capabilities | MSI_CAP_R_AUDIO) == -1) {
625
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
626
0
    }
627
628
2
    rtp_allow_receiving_mark(call->audio_rtp);
629
2
    return TOXAV_ERR_CALL_CONTROL_OK;
630
2
}
631
static Toxav_Err_Call_Control call_control_handle_hide_video(const ToxAVCall *call)
632
1
{
633
1
    if ((call->msi_call->self_capabilities & MSI_CAP_R_VIDEO) == 0) {
634
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
635
0
    }
636
637
1
    if (msi_change_capabilities(call->av->log, call->msi_call, call->
638
1
                                msi_call->self_capabilities ^ MSI_CAP_R_VIDEO) == -1) {
639
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
640
0
    }
641
642
1
    rtp_stop_receiving_mark(call->video_rtp);
643
1
    return TOXAV_ERR_CALL_CONTROL_OK;
644
1
}
645
static Toxav_Err_Call_Control call_control_handle_show_video(const ToxAVCall *call)
646
1
{
647
1
    if ((call->msi_call->self_capabilities ^ MSI_CAP_R_VIDEO) == 0) {
648
0
        return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
649
0
    }
650
651
1
    if (msi_change_capabilities(call->av->log, call->msi_call, call->
652
1
                                msi_call->self_capabilities | MSI_CAP_R_VIDEO) == -1) {
653
0
        return TOXAV_ERR_CALL_CONTROL_SYNC;
654
0
    }
655
656
1
    rtp_allow_receiving_mark(call->video_rtp);
657
1
    return TOXAV_ERR_CALL_CONTROL_OK;
658
1
}
659
static Toxav_Err_Call_Control call_control_handle(ToxAVCall *call, Toxav_Call_Control control)
660
21
{
661
21
    switch (control) {
662
2
        case TOXAV_CALL_CONTROL_RESUME:
663
2
            return call_control_handle_resume(call);
664
665
2
        case TOXAV_CALL_CONTROL_PAUSE:
666
2
            return call_control_handle_pause(call);
667
668
11
        case TOXAV_CALL_CONTROL_CANCEL:
669
11
            return call_control_handle_cancel(call);
670
671
2
        case TOXAV_CALL_CONTROL_MUTE_AUDIO:
672
2
            return call_control_handle_mute_audio(call);
673
674
2
        case TOXAV_CALL_CONTROL_UNMUTE_AUDIO:
675
2
            return call_control_handle_unmute_audio(call);
676
677
1
        case TOXAV_CALL_CONTROL_HIDE_VIDEO:
678
1
            return call_control_handle_hide_video(call);
679
680
1
        case TOXAV_CALL_CONTROL_SHOW_VIDEO:
681
1
            return call_control_handle_show_video(call);
682
21
    }
683
684
0
    return TOXAV_ERR_CALL_CONTROL_INVALID_TRANSITION;
685
21
}
686
static Toxav_Err_Call_Control call_control(ToxAV *av, uint32_t friend_number, Toxav_Call_Control control)
687
27
{
688
27
    if (!tox_friend_exists(av->tox, friend_number)) {
689
0
        return TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_FOUND;
690
0
    }
691
692
27
    ToxAVCall *call = call_get(av, friend_number);
693
694
27
    if (call == nullptr || (!call->active && control != TOXAV_CALL_CONTROL_CANCEL)) {
695
6
        return TOXAV_ERR_CALL_CONTROL_FRIEND_NOT_IN_CALL;
696
6
    }
697
698
21
    return call_control_handle(call, control);
699
27
}
700
bool toxav_call_control(ToxAV *av, uint32_t friend_number, Toxav_Call_Control control, Toxav_Err_Call_Control *error)
701
27
{
702
27
    pthread_mutex_lock(av->mutex);
703
704
27
    const Toxav_Err_Call_Control rc = call_control(av, friend_number, control);
705
706
27
    pthread_mutex_unlock(av->mutex);
707
708
27
    if (error != nullptr) {
709
21
        *error = rc;
710
21
    }
711
712
27
    return rc == TOXAV_ERR_CALL_CONTROL_OK;
713
27
}
714
715
bool toxav_audio_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_rate,
716
                              Toxav_Err_Bit_Rate_Set *error)
717
1
{
718
1
    Toxav_Err_Bit_Rate_Set rc = TOXAV_ERR_BIT_RATE_SET_OK;
719
1
    ToxAVCall *call;
720
721
1
    if (!tox_friend_exists(av->tox, friend_number)) {
722
0
        rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_FOUND;
723
0
        goto RETURN;
724
0
    }
725
726
1
    if (bit_rate > 0 && audio_bit_rate_invalid(bit_rate)) {
727
0
        rc = TOXAV_ERR_BIT_RATE_SET_INVALID_BIT_RATE;
728
0
        goto RETURN;
729
0
    }
730
731
1
    pthread_mutex_lock(av->mutex);
732
1
    call = call_get(av, friend_number);
733
734
1
    if (call == nullptr || !call->active || call->msi_call->state != MSI_CALL_ACTIVE) {
735
0
        pthread_mutex_unlock(av->mutex);
736
0
        rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_IN_CALL;
737
0
        goto RETURN;
738
0
    }
739
740
1
    LOGGER_DEBUG(av->log, "Setting new audio bitrate to: %u", bit_rate);
741
742
1
    if (call->audio_bit_rate == bit_rate) {
743
0
        LOGGER_DEBUG(av->log, "Audio bitrate already set to: %u", bit_rate);
744
1
    } else if (bit_rate == 0) {
745
1
        LOGGER_DEBUG(av->log, "Turned off audio sending");
746
747
1
        if (msi_change_capabilities(av->log, call->msi_call, call->msi_call->
748
1
                                    self_capabilities ^ MSI_CAP_S_AUDIO) != 0) {
749
0
            pthread_mutex_unlock(av->mutex);
750
0
            rc = TOXAV_ERR_BIT_RATE_SET_SYNC;
751
0
            goto RETURN;
752
0
        }
753
754
        /* Audio sending is turned off; notify peer */
755
1
        call->audio_bit_rate = 0;
756
1
    } else {
757
0
        pthread_mutex_lock(call->toxav_call_mutex);
758
759
0
        if (call->audio_bit_rate == 0) {
760
0
            LOGGER_DEBUG(av->log, "Turned on audio sending");
761
762
            /* The audio has been turned off before this */
763
0
            if (msi_change_capabilities(av->log, call->msi_call, call->
764
0
                                        msi_call->self_capabilities | MSI_CAP_S_AUDIO) != 0) {
765
0
                pthread_mutex_unlock(call->toxav_call_mutex);
766
0
                pthread_mutex_unlock(av->mutex);
767
0
                rc = TOXAV_ERR_BIT_RATE_SET_SYNC;
768
0
                goto RETURN;
769
0
            }
770
0
        } else {
771
0
            LOGGER_DEBUG(av->log, "Set new audio bit rate %u", bit_rate);
772
0
        }
773
774
0
        call->audio_bit_rate = bit_rate;
775
0
        pthread_mutex_unlock(call->toxav_call_mutex);
776
0
    }
777
778
1
    pthread_mutex_unlock(av->mutex);
779
1
RETURN:
780
781
1
    if (error != nullptr) {
782
0
        *error = rc;
783
0
    }
784
785
1
    return rc == TOXAV_ERR_BIT_RATE_SET_OK;
786
1
}
787
788
bool toxav_video_set_bit_rate(ToxAV *av, uint32_t friend_number, uint32_t bit_rate,
789
                              Toxav_Err_Bit_Rate_Set *error)
790
2
{
791
2
    Toxav_Err_Bit_Rate_Set rc = TOXAV_ERR_BIT_RATE_SET_OK;
792
2
    ToxAVCall *call;
793
794
2
    if (!tox_friend_exists(av->tox, friend_number)) {
795
0
        rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_FOUND;
796
0
        goto RETURN;
797
0
    }
798
799
2
    if (bit_rate > 0 && video_bit_rate_invalid(bit_rate)) {
800
0
        rc = TOXAV_ERR_BIT_RATE_SET_INVALID_BIT_RATE;
801
0
        goto RETURN;
802
0
    }
803
804
2
    pthread_mutex_lock(av->mutex);
805
2
    call = call_get(av, friend_number);
806
807
2
    if (call == nullptr || !call->active || call->msi_call->state != MSI_CALL_ACTIVE) {
808
0
        pthread_mutex_unlock(av->mutex);
809
0
        rc = TOXAV_ERR_BIT_RATE_SET_FRIEND_NOT_IN_CALL;
810
0
        goto RETURN;
811
0
    }
812
813
2
    LOGGER_DEBUG(av->log, "Setting new video bitrate to: %u", bit_rate);
814
815
2
    if (call->video_bit_rate == bit_rate) {
816
0
        LOGGER_DEBUG(av->log, "Video bitrate already set to: %u", bit_rate);
817
2
    } else if (bit_rate == 0) {
818
1
        LOGGER_DEBUG(av->log, "Turned off video sending");
819
820
        /* Video sending is turned off; notify peer */
821
1
        if (msi_change_capabilities(av->log, call->msi_call, call->msi_call->
822
1
                                    self_capabilities ^ MSI_CAP_S_VIDEO) != 0) {
823
0
            pthread_mutex_unlock(av->mutex);
824
0
            rc = TOXAV_ERR_BIT_RATE_SET_SYNC;
825
0
            goto RETURN;
826
0
        }
827
828
1
        call->video_bit_rate = 0;
829
1
    } else {
830
1
        pthread_mutex_lock(call->toxav_call_mutex);
831
832
1
        if (call->video_bit_rate == 0) {
833
1
            LOGGER_DEBUG(av->log, "Turned on video sending");
834
835
            /* The video has been turned off before this */
836
1
            if (msi_change_capabilities(av->log, call->msi_call, call->
837
1
                                        msi_call->self_capabilities | MSI_CAP_S_VIDEO) != 0) {
838
0
                pthread_mutex_unlock(call->toxav_call_mutex);
839
0
                pthread_mutex_unlock(av->mutex);
840
0
                rc = TOXAV_ERR_BIT_RATE_SET_SYNC;
841
0
                goto RETURN;
842
0
            }
843
1
        } else {
844
0
            LOGGER_DEBUG(av->log, "Set new video bit rate %u", bit_rate);
845
0
        }
846
847
1
        call->video_bit_rate = bit_rate;
848
1
        pthread_mutex_unlock(call->toxav_call_mutex);
849
1
    }
850
851
2
    pthread_mutex_unlock(av->mutex);
852
2
RETURN:
853
854
2
    if (error != nullptr) {
855
0
        *error = rc;
856
0
    }
857
858
2
    return rc == TOXAV_ERR_BIT_RATE_SET_OK;
859
2
}
860
861
void toxav_callback_audio_bit_rate(ToxAV *av, toxav_audio_bit_rate_cb *callback, void *user_data)
862
0
{
863
0
    pthread_mutex_lock(av->mutex);
864
0
    av->abcb = callback;
865
0
    av->abcb_user_data = user_data;
866
0
    pthread_mutex_unlock(av->mutex);
867
0
}
868
869
void toxav_callback_video_bit_rate(ToxAV *av, toxav_video_bit_rate_cb *callback, void *user_data)
870
0
{
871
0
    pthread_mutex_lock(av->mutex);
872
0
    av->vbcb = callback;
873
0
    av->vbcb_user_data = user_data;
874
0
    pthread_mutex_unlock(av->mutex);
875
0
}
876
877
bool toxav_audio_send_frame(ToxAV *av, uint32_t friend_number, const int16_t *pcm, size_t sample_count,
878
                            uint8_t channels, uint32_t sampling_rate, Toxav_Err_Send_Frame *error)
879
1.01k
{
880
1.01k
    Toxav_Err_Send_Frame rc = TOXAV_ERR_SEND_FRAME_OK;
881
1.01k
    ToxAVCall *call;
882
883
1.01k
    if (!tox_friend_exists(av->tox, friend_number)) {
884
0
        rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND;
885
0
        goto RETURN;
886
0
    }
887
888
1.01k
    if (pthread_mutex_trylock(av->mutex) != 0) {
889
1
        rc = TOXAV_ERR_SEND_FRAME_SYNC;
890
1
        goto RETURN;
891
1
    }
892
893
1.01k
    call = call_get(av, friend_number);
894
895
1.01k
    if (call == nullptr || !call->active || call->msi_call->state != MSI_CALL_ACTIVE) {
896
899
        pthread_mutex_unlock(av->mutex);
897
899
        rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
898
899
        goto RETURN;
899
899
    }
900
901
112
    if (call->audio_bit_rate == 0 ||
902
112
            (call->msi_call->self_capabilities & MSI_CAP_S_AUDIO) == 0 ||
903
112
            (call->msi_call->peer_capabilities & MSI_CAP_R_AUDIO) == 0) {
904
2
        pthread_mutex_unlock(av->mutex);
905
2
        rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED;
906
2
        goto RETURN;
907
2
    }
908
909
110
    pthread_mutex_lock(call->mutex_audio);
910
110
    pthread_mutex_unlock(av->mutex);
911
912
110
    if (pcm == nullptr) {
913
0
        pthread_mutex_unlock(call->mutex_audio);
914
0
        rc = TOXAV_ERR_SEND_FRAME_NULL;
915
0
        goto RETURN;
916
0
    }
917
918
110
    if (channels > 2) {
919
0
        pthread_mutex_unlock(call->mutex_audio);
920
0
        rc = TOXAV_ERR_SEND_FRAME_INVALID;
921
0
        goto RETURN;
922
0
    }
923
924
110
    {   /* Encode and send */
925
110
        if (ac_reconfigure_encoder(call->audio, call->audio_bit_rate * 1000, sampling_rate, channels) != 0) {
926
0
            pthread_mutex_unlock(call->mutex_audio);
927
0
            rc = TOXAV_ERR_SEND_FRAME_INVALID;
928
0
            goto RETURN;
929
0
        }
930
931
        /* This is more than enough always */
932
110
        const uint16_t dest_size = sample_count + sizeof(sampling_rate);
933
110
        VLA(uint8_t, dest, dest_size);
934
935
110
        sampling_rate = net_htonl(sampling_rate);
936
110
        memcpy(dest, &sampling_rate, sizeof(sampling_rate));
937
110
        const int vrc = opus_encode(call->audio->encoder, pcm, sample_count,
938
110
                                    dest + sizeof(sampling_rate), dest_size - sizeof(sampling_rate));
939
940
110
        if (vrc < 0) {
941
0
            LOGGER_WARNING(av->log, "Failed to encode frame %s", opus_strerror(vrc));
942
0
            pthread_mutex_unlock(call->mutex_audio);
943
0
            rc = TOXAV_ERR_SEND_FRAME_INVALID;
944
0
            goto RETURN;
945
0
        }
946
947
110
        if (rtp_send_data(av->log, call->audio_rtp, dest, vrc + sizeof(sampling_rate), false) != 0) {
948
0
            LOGGER_WARNING(av->log, "Failed to send audio packet");
949
0
            rc = TOXAV_ERR_SEND_FRAME_RTP_FAILED;
950
0
        }
951
110
    }
952
953
0
    pthread_mutex_unlock(call->mutex_audio);
954
955
1.01k
RETURN:
956
957
1.01k
    if (error != nullptr) {
958
0
        *error = rc;
959
0
    }
960
961
1.01k
    return rc == TOXAV_ERR_SEND_FRAME_OK;
962
110
}
963
964
static Toxav_Err_Send_Frame send_frames(const ToxAV *av, ToxAVCall *call)
965
108
{
966
108
    vpx_codec_iter_t iter = nullptr;
967
968
108
    for (const vpx_codec_cx_pkt_t *pkt = vpx_codec_get_cx_data(call->video->encoder, &iter);
969
216
            pkt != nullptr;
970
108
            pkt = vpx_codec_get_cx_data(call->video->encoder, &iter)) {
971
108
        if (pkt->kind != VPX_CODEC_CX_FRAME_PKT) {
972
0
            continue;
973
0
        }
974
975
108
        const bool is_keyframe = (pkt->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
976
977
        // https://www.webmproject.org/docs/webm-sdk/structvpx__codec__cx__pkt.html
978
        // pkt->data.frame.sz -> size_t
979
108
        const uint32_t frame_length_in_bytes = pkt->data.frame.sz;
980
981
108
        const int res = rtp_send_data(
982
108
                            av->log,
983
108
                            call->video_rtp,
984
108
                            (const uint8_t *)pkt->data.frame.buf,
985
108
                            frame_length_in_bytes,
986
108
                            is_keyframe);
987
988
108
        if (res < 0) {
989
0
            Net_Strerror error_str;
990
0
            LOGGER_WARNING(av->log, "Could not send video frame: %s", net_strerror(net_error(), &error_str));
991
0
            return TOXAV_ERR_SEND_FRAME_RTP_FAILED;
992
0
        }
993
108
    }
994
995
108
    return TOXAV_ERR_SEND_FRAME_OK;
996
108
}
997
998
bool toxav_video_send_frame(ToxAV *av, uint32_t friend_number, uint16_t width, uint16_t height, const uint8_t *y,
999
                            const uint8_t *u, const uint8_t *v, Toxav_Err_Send_Frame *error)
1000
1.00k
{
1001
1.00k
    Toxav_Err_Send_Frame rc = TOXAV_ERR_SEND_FRAME_OK;
1002
1.00k
    ToxAVCall *call;
1003
1004
1.00k
    int vpx_encode_flags = 0;
1005
1006
1.00k
    if (!tox_friend_exists(av->tox, friend_number)) {
1007
0
        rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_FOUND;
1008
0
        goto RETURN;
1009
0
    }
1010
1011
1.00k
    if (pthread_mutex_trylock(av->mutex) != 0) {
1012
1
        rc = TOXAV_ERR_SEND_FRAME_SYNC;
1013
1
        goto RETURN;
1014
1
    }
1015
1016
1.00k
    call = call_get(av, friend_number);
1017
1018
1.00k
    if (call == nullptr || !call->active || call->msi_call->state != MSI_CALL_ACTIVE) {
1019
899
        pthread_mutex_unlock(av->mutex);
1020
899
        rc = TOXAV_ERR_SEND_FRAME_FRIEND_NOT_IN_CALL;
1021
899
        goto RETURN;
1022
899
    }
1023
1024
108
    if (call->video_bit_rate == 0 ||
1025
108
            (call->msi_call->self_capabilities & MSI_CAP_S_VIDEO) == 0 ||
1026
108
            (call->msi_call->peer_capabilities & MSI_CAP_R_VIDEO) == 0) {
1027
0
        pthread_mutex_unlock(av->mutex);
1028
0
        rc = TOXAV_ERR_SEND_FRAME_PAYLOAD_TYPE_DISABLED;
1029
0
        goto RETURN;
1030
0
    }
1031
1032
108
    pthread_mutex_lock(call->mutex_video);
1033
108
    pthread_mutex_unlock(av->mutex);
1034
1035
108
    if (y == nullptr || u == nullptr || v == nullptr) {
1036
0
        pthread_mutex_unlock(call->mutex_video);
1037
0
        rc = TOXAV_ERR_SEND_FRAME_NULL;
1038
0
        goto RETURN;
1039
0
    }
1040
1041
108
    if (vc_reconfigure_encoder(call->video, call->video_bit_rate, width, height, -1) != 0) {
1042
0
        pthread_mutex_unlock(call->mutex_video);
1043
0
        rc = TOXAV_ERR_SEND_FRAME_INVALID;
1044
0
        goto RETURN;
1045
0
    }
1046
1047
    // we start with I-frames (full frames) and then switch to normal mode later
1048
108
    if (call->video_rtp->ssrc < VIDEO_SEND_X_KEYFRAMES_FIRST) {
1049
        // Key frame flag for first frames
1050
42
        vpx_encode_flags = VPX_EFLAG_FORCE_KF;
1051
42
        LOGGER_DEBUG(av->log, "I_FRAME_FLAG:%u only-i-frame mode", call->video_rtp->ssrc);
1052
1053
42
        ++call->video_rtp->ssrc;
1054
66
    } else if (call->video_rtp->ssrc == VIDEO_SEND_X_KEYFRAMES_FIRST) {
1055
        // normal keyframe placement
1056
6
        vpx_encode_flags = 0;
1057
6
        LOGGER_DEBUG(av->log, "I_FRAME_FLAG:%u normal mode", call->video_rtp->ssrc);
1058
1059
6
        ++call->video_rtp->ssrc;
1060
6
    }
1061
1062
108
    {   /* Encode */
1063
108
        vpx_image_t img;
1064
108
        img.w = 0;
1065
108
        img.h = 0;
1066
108
        img.d_w = 0;
1067
108
        img.d_h = 0;
1068
108
        if (vpx_img_alloc(&img, VPX_IMG_FMT_I420, width, height, 0) == nullptr) {
1069
0
            pthread_mutex_unlock(call->mutex_video);
1070
0
            LOGGER_ERROR(av->log, "Could not allocate image for frame");
1071
0
            rc = TOXAV_ERR_SEND_FRAME_INVALID;
1072
0
            goto RETURN;
1073
0
        }
1074
1075
        /* I420 "It comprises an NxM Y plane followed by (N/2)x(M/2) V and U planes."
1076
         * http://fourcc.org/yuv.php#IYUV
1077
         */
1078
108
        memcpy(img.planes[VPX_PLANE_Y], y, width * height);
1079
108
        memcpy(img.planes[VPX_PLANE_U], u, (width / 2) * (height / 2));
1080
108
        memcpy(img.planes[VPX_PLANE_V], v, (width / 2) * (height / 2));
1081
1082
108
        const vpx_codec_err_t vrc = vpx_codec_encode(call->video->encoder, &img,
1083
108
                                    call->video->frame_counter, 1, vpx_encode_flags, VPX_DL_REALTIME);
1084
1085
108
        vpx_img_free(&img);
1086
1087
108
        if (vrc != VPX_CODEC_OK) {
1088
0
            pthread_mutex_unlock(call->mutex_video);
1089
0
            LOGGER_ERROR(av->log, "Could not encode video frame: %s", vpx_codec_err_to_string(vrc));
1090
0
            rc = TOXAV_ERR_SEND_FRAME_INVALID;
1091
0
            goto RETURN;
1092
0
        }
1093
108
    }
1094
1095
108
    ++call->video->frame_counter;
1096
1097
108
    rc = send_frames(av, call);
1098
1099
108
    pthread_mutex_unlock(call->mutex_video);
1100
1101
1.00k
RETURN:
1102
1103
1.00k
    if (error != nullptr) {
1104
0
        *error = rc;
1105
0
    }
1106
1107
1.00k
    return rc == TOXAV_ERR_SEND_FRAME_OK;
1108
108
}
1109
1110
void toxav_callback_audio_receive_frame(ToxAV *av, toxav_audio_receive_frame_cb *callback, void *user_data)
1111
6
{
1112
6
    pthread_mutex_lock(av->mutex);
1113
6
    av->acb = callback;
1114
6
    av->acb_user_data = user_data;
1115
6
    pthread_mutex_unlock(av->mutex);
1116
6
}
1117
1118
void toxav_callback_video_receive_frame(ToxAV *av, toxav_video_receive_frame_cb *callback, void *user_data)
1119
6
{
1120
6
    pthread_mutex_lock(av->mutex);
1121
6
    av->vcb = callback;
1122
6
    av->vcb_user_data = user_data;
1123
6
    pthread_mutex_unlock(av->mutex);
1124
6
}
1125
1126
/*******************************************************************************
1127
 *
1128
 * :: Internal
1129
 *
1130
 ******************************************************************************/
1131
static void callback_bwc(BWController *bwc, uint32_t friend_number, float loss, void *user_data)
1132
0
{
1133
    /* Callback which is called when the internal measure mechanism reported packet loss.
1134
     * We report suggested lowered bitrate to an app. If app is sending both audio and video,
1135
     * we will report lowered bitrate for video only because in that case video probably
1136
     * takes more than 90% bandwidth. Otherwise, we report lowered bitrate on audio.
1137
     * The application may choose to disable video totally if the stream is too bad.
1138
     */
1139
1140
0
    ToxAVCall *call = (ToxAVCall *)user_data;
1141
0
    assert(call != nullptr);
1142
1143
0
    LOGGER_DEBUG(call->av->log, "Reported loss of %f%%", (double)loss * 100);
1144
1145
    /* if less than 10% data loss we do nothing! */
1146
0
    if (loss < 0.1F) {
1147
0
        return;
1148
0
    }
1149
1150
0
    pthread_mutex_lock(call->av->mutex);
1151
1152
0
    if (call->video_bit_rate != 0) {
1153
0
        if (call->av->vbcb == nullptr) {
1154
0
            pthread_mutex_unlock(call->av->mutex);
1155
0
            LOGGER_WARNING(call->av->log, "No callback to report loss on");
1156
0
            return;
1157
0
        }
1158
1159
0
        call->av->vbcb(call->av, friend_number,
1160
0
                       call->video_bit_rate - (call->video_bit_rate * loss),
1161
0
                       call->av->vbcb_user_data);
1162
0
    } else if (call->audio_bit_rate != 0) {
1163
0
        if (call->av->abcb == nullptr) {
1164
0
            pthread_mutex_unlock(call->av->mutex);
1165
0
            LOGGER_WARNING(call->av->log, "No callback to report loss on");
1166
0
            return;
1167
0
        }
1168
1169
0
        call->av->abcb(call->av, friend_number,
1170
0
                       call->audio_bit_rate - (call->audio_bit_rate * loss),
1171
0
                       call->av->abcb_user_data);
1172
0
    }
1173
1174
0
    pthread_mutex_unlock(call->av->mutex);
1175
0
}
1176
1177
static int callback_invite(void *object, MSICall *call)
1178
11
{
1179
11
    ToxAV *toxav = (ToxAV *)object;
1180
11
    pthread_mutex_lock(toxav->mutex);
1181
1182
11
    ToxAVCall *av_call = call_new(toxav, call->friend_number, nullptr);
1183
1184
11
    if (av_call == nullptr) {
1185
0
        LOGGER_WARNING(toxav->log, "Failed to initialize call...");
1186
0
        pthread_mutex_unlock(toxav->mutex);
1187
0
        return -1;
1188
0
    }
1189
1190
11
    call->av_call = av_call;
1191
11
    av_call->msi_call = call;
1192
1193
11
    if (toxav->ccb != nullptr) {
1194
11
        toxav->ccb(toxav, call->friend_number, call->peer_capabilities & MSI_CAP_S_AUDIO,
1195
11
                   call->peer_capabilities & MSI_CAP_S_VIDEO, toxav->ccb_user_data);
1196
11
    } else {
1197
        /* No handler to capture the call request, send failure */
1198
0
        pthread_mutex_unlock(toxav->mutex);
1199
0
        return -1;
1200
0
    }
1201
1202
11
    pthread_mutex_unlock(toxav->mutex);
1203
11
    return 0;
1204
11
}
1205
1206
static int callback_start(void *object, MSICall *call)
1207
9
{
1208
9
    ToxAV *toxav = (ToxAV *)object;
1209
9
    pthread_mutex_lock(toxav->mutex);
1210
1211
9
    ToxAVCall *av_call = call_get(toxav, call->friend_number);
1212
1213
9
    if (av_call == nullptr) {
1214
        /* Should this ever happen? */
1215
0
        pthread_mutex_unlock(toxav->mutex);
1216
0
        return -1;
1217
0
    }
1218
1219
9
    if (!call_prepare_transmission(av_call)) {
1220
0
        callback_error(toxav, call);
1221
0
        pthread_mutex_unlock(toxav->mutex);
1222
0
        return -1;
1223
0
    }
1224
1225
9
    if (!invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities)) {
1226
0
        callback_error(toxav, call);
1227
0
        pthread_mutex_unlock(toxav->mutex);
1228
0
        return -1;
1229
0
    }
1230
1231
9
    pthread_mutex_unlock(toxav->mutex);
1232
9
    return 0;
1233
9
}
1234
1235
static int callback_end(void *object, MSICall *call)
1236
11
{
1237
11
    ToxAV *toxav = (ToxAV *)object;
1238
11
    pthread_mutex_lock(toxav->mutex);
1239
1240
11
    invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_FINISHED);
1241
1242
11
    if (call->av_call != nullptr) {
1243
11
        call_kill_transmission(call->av_call);
1244
11
        call_remove(call->av_call);
1245
11
    }
1246
1247
11
    pthread_mutex_unlock(toxav->mutex);
1248
11
    return 0;
1249
11
}
1250
1251
static int callback_error(void *object, MSICall *call)
1252
0
{
1253
0
    ToxAV *toxav = (ToxAV *)object;
1254
0
    pthread_mutex_lock(toxav->mutex);
1255
1256
0
    invoke_call_state_callback(toxav, call->friend_number, TOXAV_FRIEND_CALL_STATE_ERROR);
1257
1258
0
    if (call->av_call != nullptr) {
1259
0
        call_kill_transmission(call->av_call);
1260
0
        call_remove(call->av_call);
1261
0
    }
1262
1263
0
    pthread_mutex_unlock(toxav->mutex);
1264
0
    return 0;
1265
0
}
1266
1267
static int callback_capabilites(void *object, MSICall *call)
1268
13
{
1269
13
    ToxAV *toxav = (ToxAV *)object;
1270
13
    pthread_mutex_lock(toxav->mutex);
1271
1272
13
    if ((call->peer_capabilities & MSI_CAP_S_AUDIO) != 0) {
1273
10
        rtp_allow_receiving_mark(call->av_call->audio_rtp);
1274
10
    } else {
1275
3
        rtp_stop_receiving_mark(call->av_call->audio_rtp);
1276
3
    }
1277
1278
13
    if ((call->peer_capabilities & MSI_CAP_S_VIDEO) != 0) {
1279
8
        rtp_allow_receiving_mark(call->av_call->video_rtp);
1280
8
    } else {
1281
5
        rtp_stop_receiving_mark(call->av_call->video_rtp);
1282
5
    }
1283
1284
13
    invoke_call_state_callback(toxav, call->friend_number, call->peer_capabilities);
1285
1286
13
    pthread_mutex_unlock(toxav->mutex);
1287
13
    return 0;
1288
13
}
1289
1290
static bool audio_bit_rate_invalid(uint32_t bit_rate)
1291
18
{
1292
    /* Opus RFC 6716 section-2.1.1 dictates the following:
1293
     * Opus supports all bit rates from 6 kbit/s to 510 kbit/s.
1294
     */
1295
18
    return bit_rate < 6 || bit_rate > 510;
1296
18
}
1297
1298
static bool video_bit_rate_invalid(uint32_t bit_rate)
1299
13
{
1300
    /* Cap the target rate to 1000 Mbps to avoid some integer overflows in
1301
     * target bandwidth calculations.
1302
     * https://github.com/webmproject/libvpx/blob/027bbee30a0103b99d86327b48d29567fed11688/vp8/vp8_cx_iface.c#L350-L352
1303
     */
1304
13
    return bit_rate > 1000000;
1305
13
}
1306
1307
static bool invoke_call_state_callback(ToxAV *av, uint32_t friend_number, uint32_t state)
1308
33
{
1309
33
    if (av->scb != nullptr) {
1310
33
        av->scb(av, friend_number, state, av->scb_user_data);
1311
33
    } else {
1312
0
        return false;
1313
0
    }
1314
1315
33
    return true;
1316
33
}
1317
1318
static ToxAVCall *call_new(ToxAV *av, uint32_t friend_number, Toxav_Err_Call *error)
1319
22
{
1320
    /* Assumes mutex locked */
1321
22
    Toxav_Err_Call rc = TOXAV_ERR_CALL_OK;
1322
22
    ToxAVCall *call = nullptr;
1323
1324
22
    Tox_Err_Friend_Query f_con_query_error;
1325
22
    Tox_Connection f_con_status = TOX_CONNECTION_NONE;
1326
1327
22
    if (!tox_friend_exists(av->tox, friend_number)) {
1328
0
        rc = TOXAV_ERR_CALL_FRIEND_NOT_FOUND;
1329
0
        goto RETURN;
1330
0
    }
1331
1332
22
    f_con_status = tox_friend_get_connection_status(av->tox, friend_number, &f_con_query_error);
1333
1334
22
    if (f_con_status == TOX_CONNECTION_NONE) {
1335
0
        rc = TOXAV_ERR_CALL_FRIEND_NOT_CONNECTED;
1336
0
        goto RETURN;
1337
0
    }
1338
1339
22
    if (call_get(av, friend_number) != nullptr) {
1340
0
        rc = TOXAV_ERR_CALL_FRIEND_ALREADY_IN_CALL;
1341
0
        goto RETURN;
1342
0
    }
1343
1344
22
    call = (ToxAVCall *)calloc(1, sizeof(ToxAVCall));
1345
1346
22
    if (call == nullptr) {
1347
0
        rc = TOXAV_ERR_CALL_MALLOC;
1348
0
        goto RETURN;
1349
0
    }
1350
1351
22
    call->av = av;
1352
22
    call->friend_number = friend_number;
1353
1354
22
    if (create_recursive_mutex(call->toxav_call_mutex) != 0) {
1355
0
        free(call);
1356
0
        call = nullptr;
1357
0
        rc = TOXAV_ERR_CALL_MALLOC;
1358
0
        goto RETURN;
1359
0
    }
1360
1361
22
    if (av->calls == nullptr) { /* Creating */
1362
20
        av->calls = (ToxAVCall **)calloc(friend_number + 1, sizeof(ToxAVCall *));
1363
1364
20
        if (av->calls == nullptr) {
1365
0
            pthread_mutex_destroy(call->toxav_call_mutex);
1366
0
            free(call);
1367
0
            call = nullptr;
1368
0
            rc = TOXAV_ERR_CALL_MALLOC;
1369
0
            goto RETURN;
1370
0
        }
1371
1372
20
        av->calls_tail = friend_number;
1373
20
        av->calls_head = friend_number;
1374
20
    } else if (av->calls_tail < friend_number) { /* Appending */
1375
2
        ToxAVCall **tmp = (ToxAVCall **)realloc(av->calls, (friend_number + 1) * sizeof(ToxAVCall *));
1376
1377
2
        if (tmp == nullptr) {
1378
0
            pthread_mutex_destroy(call->toxav_call_mutex);
1379
0
            free(call);
1380
0
            call = nullptr;
1381
0
            rc = TOXAV_ERR_CALL_MALLOC;
1382
0
            goto RETURN;
1383
0
        }
1384
1385
2
        av->calls = tmp;
1386
1387
        /* Set fields in between to null */
1388
2
        for (uint32_t i = av->calls_tail + 1; i < friend_number; ++i) {
1389
0
            av->calls[i] = nullptr;
1390
0
        }
1391
1392
2
        call->prev = av->calls[av->calls_tail];
1393
2
        av->calls[av->calls_tail]->next = call;
1394
1395
2
        av->calls_tail = friend_number;
1396
2
    } else if (av->calls_head > friend_number) { /* Inserting at front */
1397
0
        call->next = av->calls[av->calls_head];
1398
0
        av->calls[av->calls_head]->prev = call;
1399
0
        av->calls_head = friend_number;
1400
0
    }
1401
1402
22
    av->calls[friend_number] = call;
1403
1404
22
RETURN:
1405
1406
22
    if (error != nullptr) {
1407
11
        *error = rc;
1408
11
    }
1409
1410
22
    return call;
1411
22
}
1412
1413
static ToxAVCall *call_remove(ToxAVCall *call)
1414
22
{
1415
22
    if (call == nullptr) {
1416
0
        return nullptr;
1417
0
    }
1418
1419
22
    const uint32_t friend_number = call->friend_number;
1420
22
    ToxAV *av = call->av;
1421
1422
22
    ToxAVCall *prev = call->prev;
1423
22
    ToxAVCall *next = call->next;
1424
1425
    /* Set av call in msi to NULL in order to know if call if ToxAVCall is
1426
     * removed from the msi call.
1427
     */
1428
22
    if (call->msi_call != nullptr) {
1429
11
        call->msi_call->av_call = nullptr;
1430
11
    }
1431
1432
22
    pthread_mutex_destroy(call->toxav_call_mutex);
1433
22
    free(call);
1434
1435
22
    if (prev != nullptr) {
1436
0
        prev->next = next;
1437
22
    } else if (next != nullptr) {
1438
2
        av->calls_head = next->friend_number;
1439
20
    } else {
1440
20
        goto CLEAR;
1441
20
    }
1442
1443
2
    if (next != nullptr) {
1444
2
        next->prev = prev;
1445
2
    } else if (prev != nullptr) {
1446
0
        av->calls_tail = prev->friend_number;
1447
0
    } else {
1448
0
        goto CLEAR;
1449
0
    }
1450
1451
2
    av->calls[friend_number] = nullptr;
1452
2
    return next;
1453
1454
20
CLEAR:
1455
20
    av->calls_head = 0;
1456
20
    av->calls_tail = 0;
1457
20
    free(av->calls);
1458
20
    av->calls = nullptr;
1459
1460
20
    return nullptr;
1461
2
}
1462
1463
static bool call_prepare_transmission(ToxAVCall *call)
1464
18
{
1465
    /* Assumes mutex locked */
1466
1467
18
    if (call == nullptr) {
1468
0
        return false;
1469
0
    }
1470
1471
18
    ToxAV *av = call->av;
1472
1473
18
    if (av->acb == nullptr && av->vcb == nullptr) {
1474
        /* It makes no sense to have CSession without callbacks */
1475
0
        return false;
1476
0
    }
1477
1478
18
    if (call->active) {
1479
0
        LOGGER_WARNING(av->log, "Call already active!");
1480
0
        return true;
1481
0
    }
1482
1483
18
    if (create_recursive_mutex(call->mutex_audio) != 0) {
1484
0
        return false;
1485
0
    }
1486
1487
18
    if (create_recursive_mutex(call->mutex_video) != 0) {
1488
0
        goto FAILURE_2;
1489
0
    }
1490
1491
    /* Prepare bwc */
1492
18
    call->bwc = bwc_new(av->log, av->tox, call->friend_number, callback_bwc, call, av->toxav_mono_time);
1493
1494
18
    { /* Prepare audio */
1495
18
        call->audio = ac_new(av->toxav_mono_time, av->log, av, call->friend_number, av->acb, av->acb_user_data);
1496
1497
18
        if (call->audio == nullptr) {
1498
0
            LOGGER_ERROR(av->log, "Failed to create audio codec session");
1499
0
            goto FAILURE;
1500
0
        }
1501
1502
18
        call->audio_rtp = rtp_new(av->log, av->mem, RTP_TYPE_AUDIO, av->tox, av, call->friend_number, call->bwc,
1503
18
                                  call->audio, ac_queue_message);
1504
1505
18
        if (call->audio_rtp == nullptr) {
1506
0
            LOGGER_ERROR(av->log, "Failed to create audio rtp session");
1507
0
            goto FAILURE;
1508
0
        }
1509
18
    }
1510
18
    { /* Prepare video */
1511
18
        call->video = vc_new(av->log, av->toxav_mono_time, av, call->friend_number, av->vcb, av->vcb_user_data);
1512
1513
18
        if (call->video == nullptr) {
1514
0
            LOGGER_ERROR(av->log, "Failed to create video codec session");
1515
0
            goto FAILURE;
1516
0
        }
1517
1518
18
        call->video_rtp = rtp_new(av->log, av->mem, RTP_TYPE_VIDEO, av->tox, av, call->friend_number, call->bwc,
1519
18
                                  call->video, vc_queue_message);
1520
1521
18
        if (call->video_rtp == nullptr) {
1522
0
            LOGGER_ERROR(av->log, "Failed to create video rtp session");
1523
0
            goto FAILURE;
1524
0
        }
1525
18
    }
1526
1527
18
    call->active = true;
1528
18
    return true;
1529
1530
0
FAILURE:
1531
0
    bwc_kill(call->bwc);
1532
0
    rtp_kill(av->log, call->audio_rtp);
1533
0
    ac_kill(call->audio);
1534
0
    call->audio_rtp = nullptr;
1535
0
    call->audio = nullptr;
1536
0
    rtp_kill(av->log, call->video_rtp);
1537
0
    vc_kill(call->video);
1538
0
    call->video_rtp = nullptr;
1539
0
    call->video = nullptr;
1540
0
    pthread_mutex_destroy(call->mutex_video);
1541
0
FAILURE_2:
1542
0
    pthread_mutex_destroy(call->mutex_audio);
1543
0
    return false;
1544
0
}
1545
1546
static void call_kill_transmission(ToxAVCall *call)
1547
22
{
1548
22
    if (call == nullptr || !call->active) {
1549
4
        return;
1550
4
    }
1551
1552
18
    call->active = false;
1553
1554
18
    pthread_mutex_lock(call->mutex_audio);
1555
18
    pthread_mutex_unlock(call->mutex_audio);
1556
18
    pthread_mutex_lock(call->mutex_video);
1557
18
    pthread_mutex_unlock(call->mutex_video);
1558
18
    pthread_mutex_lock(call->toxav_call_mutex);
1559
18
    pthread_mutex_unlock(call->toxav_call_mutex);
1560
1561
18
    bwc_kill(call->bwc);
1562
1563
18
    const ToxAV *av = call->av;
1564
1565
18
    rtp_kill(av->log, call->audio_rtp);
1566
18
    ac_kill(call->audio);
1567
18
    call->audio_rtp = nullptr;
1568
18
    call->audio = nullptr;
1569
1570
18
    rtp_kill(av->log, call->video_rtp);
1571
18
    vc_kill(call->video);
1572
18
    call->video_rtp = nullptr;
1573
18
    call->video = nullptr;
1574
1575
18
    pthread_mutex_destroy(call->mutex_audio);
1576
18
    pthread_mutex_destroy(call->mutex_video);
1577
18
}
1578
1579
Mono_Time *toxav_get_av_mono_time(const ToxAV *av)
1580
426
{
1581
426
    if (av == nullptr) {
1582
0
        return nullptr;
1583
0
    }
1584
1585
426
    return av->toxav_mono_time;
1586
426
}