Coverage Report

Created: 2025-10-08 19:34

/work/toxav/audio.c
Line
Count
Source (jump to first uncovered line)
1
/* SPDX-License-Identifier: GPL-3.0-or-later
2
 * Copyright © 2016-2025 The TokTok team.
3
 * Copyright © 2013-2015 Tox project.
4
 */
5
#include "audio.h"
6
7
#include <assert.h>
8
#include <stdlib.h>
9
#include <string.h>
10
11
#include "rtp.h"
12
13
#include "../toxcore/ccompat.h"
14
#include "../toxcore/logger.h"
15
#include "../toxcore/mono_time.h"
16
#include "../toxcore/network.h"
17
18
static struct JitterBuffer *jbuf_new(uint32_t capacity);
19
static void jbuf_clear(struct JitterBuffer *q);
20
static void jbuf_free(struct JitterBuffer *q);
21
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m);
22
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success);
23
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
24
        uint8_t channel_count);
25
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
26
                                      uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch);
27
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels);
28
29
30
31
ACSession *ac_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
32
                  toxav_audio_receive_frame_cb *cb, void *cb_data)
33
18
{
34
18
    ACSession *ac = (ACSession *)calloc(1, sizeof(ACSession));
35
36
18
    if (ac == nullptr) {
37
0
        LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
38
0
        return nullptr;
39
0
    }
40
41
18
    if (create_recursive_mutex(ac->queue_mutex) != 0) {
42
0
        LOGGER_WARNING(log, "Failed to create recursive mutex!");
43
0
        free(ac);
44
0
        return nullptr;
45
0
    }
46
47
18
    int status;
48
18
    ac->decoder = opus_decoder_create(AUDIO_DECODER_START_SAMPLE_RATE, AUDIO_DECODER_START_CHANNEL_COUNT, &status);
49
50
18
    if (status != OPUS_OK) {
51
0
        LOGGER_ERROR(log, "Error while starting audio decoder: %s", opus_strerror(status));
52
0
        goto BASE_CLEANUP;
53
0
    }
54
55
18
    ac->j_buf = jbuf_new(AUDIO_JITTERBUFFER_COUNT);
56
57
18
    if (ac->j_buf == nullptr) {
58
0
        LOGGER_WARNING(log, "Jitter buffer creaton failed!");
59
0
        opus_decoder_destroy(ac->decoder);
60
0
        goto BASE_CLEANUP;
61
0
    }
62
63
18
    ac->mono_time = mono_time;
64
18
    ac->log = log;
65
66
    /* Initialize encoders with default values */
67
18
    ac->encoder = create_audio_encoder(log, AUDIO_START_BITRATE, AUDIO_START_SAMPLE_RATE, AUDIO_START_CHANNEL_COUNT);
68
69
18
    if (ac->encoder == nullptr) {
70
0
        goto DECODER_CLEANUP;
71
0
    }
72
73
18
    ac->le_bit_rate = AUDIO_START_BITRATE;
74
18
    ac->le_sample_rate = AUDIO_START_SAMPLE_RATE;
75
18
    ac->le_channel_count = AUDIO_START_CHANNEL_COUNT;
76
77
18
    ac->ld_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
78
18
    ac->ld_sample_rate = AUDIO_DECODER_START_SAMPLE_RATE;
79
18
    ac->ldrts = 0; /* Make it possible to reconfigure straight away */
80
81
    /* These need to be set in order to properly
82
     * do error correction with opus */
83
18
    ac->lp_frame_duration = AUDIO_MAX_FRAME_DURATION_MS;
84
18
    ac->lp_sampling_rate = AUDIO_DECODER_START_SAMPLE_RATE;
85
18
    ac->lp_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
86
87
18
    ac->av = av;
88
18
    ac->friend_number = friend_number;
89
18
    ac->acb = cb;
90
18
    ac->acb_user_data = cb_data;
91
92
18
    return ac;
93
94
0
DECODER_CLEANUP:
95
0
    opus_decoder_destroy(ac->decoder);
96
0
    jbuf_free((struct JitterBuffer *)ac->j_buf);
97
0
BASE_CLEANUP:
98
0
    pthread_mutex_destroy(ac->queue_mutex);
99
0
    free(ac);
100
0
    return nullptr;
101
0
}
102
103
void ac_kill(ACSession *ac)
104
18
{
105
18
    if (ac == nullptr) {
106
0
        return;
107
0
    }
108
109
18
    opus_encoder_destroy(ac->encoder);
110
18
    opus_decoder_destroy(ac->decoder);
111
18
    jbuf_free((struct JitterBuffer *)ac->j_buf);
112
113
18
    pthread_mutex_destroy(ac->queue_mutex);
114
115
18
    LOGGER_DEBUG(ac->log, "Terminated audio handler: %p", (void *)ac);
116
18
    free(ac);
117
18
}
118
119
void ac_iterate(ACSession *ac)
120
210
{
121
210
    if (ac == nullptr) {
122
0
        return;
123
0
    }
124
125
    /* TODO: fix this and jitter buffering */
126
127
    /* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */
128
210
    int16_t *temp_audio_buffer = (int16_t *)malloc(AUDIO_MAX_BUFFER_SIZE_PCM16 * AUDIO_MAX_CHANNEL_COUNT * sizeof(int16_t));
129
130
210
    if (temp_audio_buffer == nullptr) {
131
0
        LOGGER_ERROR(ac->log, "Failed to allocate memory for audio buffer");
132
0
        return;
133
0
    }
134
135
210
    pthread_mutex_lock(ac->queue_mutex);
136
210
    struct JitterBuffer *const j_buf = (struct JitterBuffer *)ac->j_buf;
137
138
210
    int rc = 0;
139
140
210
    for (struct RTPMessage *msg = jbuf_read(j_buf, &rc); msg != nullptr || rc == 2; msg = jbuf_read(j_buf, &rc)) {
141
98
        pthread_mutex_unlock(ac->queue_mutex);
142
143
98
        if (rc == 2) {
144
0
            LOGGER_DEBUG(ac->log, "OPUS correction");
145
0
            const int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000;
146
0
            rc = opus_decode(ac->decoder, nullptr, 0, temp_audio_buffer, fs, 1);
147
98
        } else {
148
98
            assert(msg->len > 4);
149
150
            /* Pick up sampling rate from packet */
151
98
            memcpy(&ac->lp_sampling_rate, msg->data, 4);
152
98
            ac->lp_sampling_rate = net_ntohl(ac->lp_sampling_rate);
153
154
98
            ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4);
155
156
            /** NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
157
              * it didn't work quite well.
158
              */
159
98
            if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) {
160
0
                LOGGER_WARNING(ac->log, "Failed to reconfigure decoder!");
161
0
                free(msg);
162
0
                pthread_mutex_lock(ac->queue_mutex);
163
0
                continue;
164
0
            }
165
166
            /*
167
             * frame_size = opus_decode(dec, packet, len, decoded, max_size, 0);
168
             *   where
169
             * packet is the byte array containing the compressed data
170
             * len is the exact number of bytes contained in the packet
171
             * decoded is the decoded audio data in opus_int16 (or float for opus_decode_float())
172
             * max_size is the max duration of the frame in samples (per channel) that can fit
173
             * into the decoded_frame array
174
             */
175
98
            rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, temp_audio_buffer, 5760, 0);
176
98
            free(msg);
177
98
        }
178
179
98
        if (rc < 0) {
180
0
            LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc));
181
98
        } else if (ac->acb != nullptr) {
182
98
            ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate;
183
184
98
            ac->acb(ac->av, ac->friend_number, temp_audio_buffer, rc, ac->lp_channel_count,
185
98
                    ac->lp_sampling_rate, ac->acb_user_data);
186
98
        }
187
188
98
        free(temp_audio_buffer);
189
190
98
        return;
191
98
    }
192
193
112
    pthread_mutex_unlock(ac->queue_mutex);
194
195
112
    free(temp_audio_buffer);
196
112
}
197
198
int ac_queue_message(const Mono_Time *mono_time, void *cs, struct RTPMessage *msg)
199
107
{
200
107
    ACSession *ac = (ACSession *)cs;
201
202
107
    if (ac == nullptr || msg == nullptr) {
203
0
        free(msg);
204
0
        return -1;
205
0
    }
206
207
107
    if ((msg->header.pt & 0x7f) == (RTP_TYPE_AUDIO + 2) % 128) {
208
0
        LOGGER_WARNING(ac->log, "Got dummy!");
209
0
        free(msg);
210
0
        return 0;
211
0
    }
212
213
107
    if ((msg->header.pt & 0x7f) != RTP_TYPE_AUDIO % 128) {
214
0
        LOGGER_WARNING(ac->log, "Invalid payload type!");
215
0
        free(msg);
216
0
        return -1;
217
0
    }
218
219
107
    pthread_mutex_lock(ac->queue_mutex);
220
107
    const int rc = jbuf_write(ac->log, (struct JitterBuffer *)ac->j_buf, msg);
221
107
    pthread_mutex_unlock(ac->queue_mutex);
222
223
107
    if (rc == -1) {
224
0
        LOGGER_WARNING(ac->log, "Could not queue the message!");
225
0
        free(msg);
226
0
        return -1;
227
0
    }
228
229
107
    return 0;
230
107
}
231
232
int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels)
233
110
{
234
110
    if (ac == nullptr || !reconfigure_audio_encoder(
235
110
                ac->log, &ac->encoder, bit_rate,
236
110
                sampling_rate, channels,
237
110
                &ac->le_bit_rate,
238
110
                &ac->le_sample_rate,
239
110
                &ac->le_channel_count)) {
240
0
        return -1;
241
0
    }
242
243
110
    return 0;
244
110
}
245
246
struct JitterBuffer {
247
    struct RTPMessage **queue;
248
    uint32_t size;
249
    uint32_t capacity;
250
    uint16_t bottom;
251
    uint16_t top;
252
};
253
254
static struct JitterBuffer *jbuf_new(uint32_t capacity)
255
18
{
256
18
    unsigned int size = 1;
257
258
90
    while (size <= (capacity * 4)) {
259
72
        size *= 2;
260
72
    }
261
262
18
    struct JitterBuffer *q = (struct JitterBuffer *)calloc(1, sizeof(struct JitterBuffer));
263
264
18
    if (q == nullptr) {
265
0
        return nullptr;
266
0
    }
267
268
18
    q->queue = (struct RTPMessage **)calloc(size, sizeof(struct RTPMessage *));
269
270
18
    if (q->queue == nullptr) {
271
0
        free(q);
272
0
        return nullptr;
273
0
    }
274
275
18
    q->size = size;
276
18
    q->capacity = capacity;
277
18
    return q;
278
18
}
279
280
static void jbuf_clear(struct JitterBuffer *q)
281
18
{
282
27
    while (q->bottom != q->top) {
283
9
        free(q->queue[q->bottom % q->size]);
284
9
        q->queue[q->bottom % q->size] = nullptr;
285
9
        ++q->bottom;
286
9
    }
287
18
}
288
289
static void jbuf_free(struct JitterBuffer *q)
290
18
{
291
18
    if (q == nullptr) {
292
0
        return;
293
0
    }
294
295
18
    jbuf_clear(q);
296
18
    free(q->queue);
297
18
    free(q);
298
18
}
299
300
/*
301
 * if -1 is returned the RTPMessage m needs to be free'd by the caller
302
 * if  0 is returned the RTPMessage m is stored in the ringbuffer and must NOT be freed by the caller
303
 */
304
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m)
305
107
{
306
107
    const uint16_t sequnum = m->header.sequnum;
307
308
107
    const unsigned int num = sequnum % q->size;
309
310
107
    if ((uint32_t)(sequnum - q->bottom) > q->size) {
311
0
        LOGGER_DEBUG(log, "Clearing filled jitter buffer: %p", (void *)q);
312
313
0
        jbuf_clear(q);
314
0
        q->bottom = sequnum - q->capacity;
315
0
        q->queue[num] = m;
316
0
        q->top = sequnum + 1;
317
0
        return 0;
318
0
    }
319
320
107
    if (q->queue[num] != nullptr) {
321
0
        return -1;
322
0
    }
323
324
107
    q->queue[num] = m;
325
326
107
    if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
327
107
        q->top = sequnum + 1;
328
107
    }
329
330
107
    return 0;
331
107
}
332
333
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success)
334
210
{
335
210
    if (q->top == q->bottom) {
336
112
        *success = 0;
337
112
        return nullptr;
338
112
    }
339
340
98
    const unsigned int num = q->bottom % q->size;
341
342
98
    if (q->queue[num] != nullptr) {
343
98
        struct RTPMessage *ret = q->queue[num];
344
98
        q->queue[num] = nullptr;
345
98
        ++q->bottom;
346
98
        *success = 1;
347
98
        return ret;
348
98
    }
349
350
0
    if ((uint32_t)(q->top - q->bottom) > q->capacity) {
351
0
        ++q->bottom;
352
0
        *success = 2;
353
0
        return nullptr;
354
0
    }
355
356
0
    *success = 0;
357
0
    return nullptr;
358
0
}
359
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
360
        uint8_t channel_count)
361
26
{
362
26
    int status = OPUS_OK;
363
    /*
364
     * OPUS_APPLICATION_VOIP Process signal for improved speech intelligibility
365
     * OPUS_APPLICATION_AUDIO Favor faithfulness to the original input
366
     * OPUS_APPLICATION_RESTRICTED_LOWDELAY Configure the minimum possible coding delay
367
     */
368
26
    OpusEncoder *rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status);
369
370
26
    if (status != OPUS_OK) {
371
0
        LOGGER_ERROR(log, "Error while starting audio encoder: %s", opus_strerror(status));
372
0
        return nullptr;
373
0
    }
374
375
    /*
376
     * Rates from 500 to 512000 bits per second are meaningful as well as the special
377
     * values OPUS_BITRATE_AUTO and OPUS_BITRATE_MAX. The value OPUS_BITRATE_MAX can
378
     * be used to cause the codec to use as much rate as it can, which is useful for
379
     * controlling the rate by adjusting the output buffer size.
380
     *
381
     * Parameters:
382
     *   `[in]`    `x`   `opus_int32`: bitrate in bits per second.
383
     */
384
26
    status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate));
385
386
26
    if (status != OPUS_OK) {
387
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
388
0
        goto FAILURE;
389
0
    }
390
391
    /*
392
     * The libopus library defaults to VBR, which is unsafe in any VoIP environment
393
     * (see for example doi:10.1109/SP.2011.34). Switching to CBR very slightly
394
     * decreases audio quality at lower bitrates.
395
     *
396
     * Parameters:
397
     *  `[in]`    `x`   `opus_int32`: Whether to use VBR mode, 1 (VBR) is default
398
     */
399
26
    status = opus_encoder_ctl(rc, OPUS_SET_VBR(0));
400
401
26
    if (status != OPUS_OK) {
402
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
403
0
        goto FAILURE;
404
0
    }
405
406
    /*
407
     * Configures the encoder's use of inband forward error correction.
408
     * Note:
409
     *   This is only applicable to the LPC layer
410
     * Parameters:
411
     *   `[in]`    `x`   `int`: FEC flag, 0 (disabled) is default
412
     */
413
    /* Enable in-band forward error correction in codec */
414
26
    status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1));
415
416
26
    if (status != OPUS_OK) {
417
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
418
0
        goto FAILURE;
419
0
    }
420
421
    /*
422
     * Configures the encoder's expected packet loss percentage.
423
     * Higher values with trigger progressively more loss resistant behavior in
424
     * the encoder at the expense of quality at a given bitrate in the lossless case,
425
     * but greater quality under loss.
426
     * Parameters:
427
     *     `[in]`    `x`   `int`: Loss percentage in the range 0-100, inclusive.
428
     */
429
    /* Make codec resistant to up to 10% packet loss
430
     * NOTE This could also be adjusted on the fly, rather than hard-coded,
431
     *      with feedback from the receiving client.
432
     */
433
26
    status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(AUDIO_OPUS_PACKET_LOSS_PERC));
434
435
26
    if (status != OPUS_OK) {
436
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
437
0
        goto FAILURE;
438
0
    }
439
440
    /*
441
     * Configures the encoder's computational complexity.
442
     *
443
     * The supported range is 0-10 inclusive with 10 representing the highest complexity.
444
     * The default value is 10.
445
     *
446
     * Parameters:
447
     *   `[in]`    `x`   `int`: 0-10, inclusive
448
     */
449
    /* Set algorithm to the highest complexity, maximizing compression */
450
26
    status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(AUDIO_OPUS_COMPLEXITY));
451
452
26
    if (status != OPUS_OK) {
453
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
454
0
        goto FAILURE;
455
0
    }
456
457
26
    return rc;
458
459
0
FAILURE:
460
0
    opus_encoder_destroy(rc);
461
0
    return nullptr;
462
26
}
463
464
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
465
                                      uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch)
466
110
{
467
    /* Values are checked in toxav.c */
468
110
    if (*old_sr != new_sr || *old_ch != new_ch) {
469
8
        OpusEncoder *new_encoder = create_audio_encoder(log, new_br, new_sr, new_ch);
470
471
8
        if (new_encoder == nullptr) {
472
0
            return false;
473
0
        }
474
475
8
        opus_encoder_destroy(*e);
476
8
        *e = new_encoder;
477
102
    } else if (*old_br == new_br) {
478
102
        return true; /* Nothing changed */
479
102
    }
480
481
8
    const int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br));
482
483
8
    if (status != OPUS_OK) {
484
0
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
485
0
        return false;
486
0
    }
487
488
8
    *old_br = new_br;
489
8
    *old_sr = new_sr;
490
8
    *old_ch = new_ch;
491
492
8
    LOGGER_DEBUG(log, "Reconfigured audio encoder br: %u sr: %u cc:%d", new_br, new_sr, new_ch);
493
8
    return true;
494
8
}
495
496
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels)
497
98
{
498
98
    if (sampling_rate != ac->ld_sample_rate || channels != ac->ld_channel_count) {
499
0
        if (current_time_monotonic(ac->mono_time) - ac->ldrts < 500) {
500
0
            return false;
501
0
        }
502
503
0
        int status;
504
0
        OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status);
505
506
0
        if (status != OPUS_OK) {
507
0
            LOGGER_ERROR(ac->log, "Error while starting audio decoder(%u %u): %s", sampling_rate, channels, opus_strerror(status));
508
0
            return false;
509
0
        }
510
511
0
        ac->ld_sample_rate = sampling_rate;
512
0
        ac->ld_channel_count = channels;
513
0
        ac->ldrts = current_time_monotonic(ac->mono_time);
514
515
0
        opus_decoder_destroy(ac->decoder);
516
0
        ac->decoder = new_dec;
517
518
0
        LOGGER_DEBUG(ac->log, "Reconfigured audio decoder sr: %u cc: %u", sampling_rate, channels);
519
0
    }
520
521
98
    return true;
522
98
}