/work/toxav/audio.c

Source (jump to first uncovered line)
/* SPDX-License-Identifier: GPL-3.0-or-later
 * Copyright © 2016-2018 The TokTok team.
 * Copyright © 2013-2015 Tox project.
 */
#include "audio.h"

#include <assert.h>
#include <stdlib.h>
#include <string.h>

#include "rtp.h"

#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"

static struct JitterBuffer *jbuf_new(uint32_t capacity);
static void jbuf_clear(struct JitterBuffer *q);
static void jbuf_free(struct JitterBuffer *q);
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m);
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success);
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
        uint8_t channel_count);
static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
                                      uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch);
static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels);



ACSession *ac_new(Mono_Time *mono_time, const Logger *log, ToxAV *av, uint32_t friend_number,
                  toxav_audio_receive_frame_cb *cb, void *cb_data)
{
    ACSession *ac = (ACSession *)calloc(1, sizeof(ACSession));

    if (ac == nullptr) {
        LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
        return nullptr;
    }

    if (create_recursive_mutex(ac->queue_mutex) != 0) {
        LOGGER_WARNING(log, "Failed to create recursive mutex!");
        free(ac);
        return nullptr;
    }

    int status;
    ac->decoder = opus_decoder_create(AUDIO_DECODER_START_SAMPLE_RATE, AUDIO_DECODER_START_CHANNEL_COUNT, &status);

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while starting audio decoder: %s", opus_strerror(status));
        goto BASE_CLEANUP;
    }

    ac->j_buf = jbuf_new(AUDIO_JITTERBUFFER_COUNT);

    if (ac->j_buf == nullptr) {
        LOGGER_WARNING(log, "Jitter buffer creaton failed!");
        opus_decoder_destroy(ac->decoder);
        goto BASE_CLEANUP;
    }

    ac->mono_time = mono_time;
    ac->log = log;

    /* Initialize encoders with default values */
    ac->encoder = create_audio_encoder(log, AUDIO_START_BITRATE, AUDIO_START_SAMPLE_RATE, AUDIO_START_CHANNEL_COUNT);

    if (ac->encoder == nullptr) {
        goto DECODER_CLEANUP;
    }

    ac->le_bit_rate = AUDIO_START_BITRATE;
    ac->le_sample_rate = AUDIO_START_SAMPLE_RATE;
    ac->le_channel_count = AUDIO_START_CHANNEL_COUNT;

    ac->ld_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
    ac->ld_sample_rate = AUDIO_DECODER_START_SAMPLE_RATE;
    ac->ldrts = 0; /* Make it possible to reconfigure straight away */

    /* These need to be set in order to properly
     * do error correction with opus */
    ac->lp_frame_duration = AUDIO_MAX_FRAME_DURATION_MS;
    ac->lp_sampling_rate = AUDIO_DECODER_START_SAMPLE_RATE;
    ac->lp_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;

    ac->av = av;
    ac->friend_number = friend_number;
    ac->acb = cb;
    ac->acb_user_data = cb_data;

    return ac;

DECODER_CLEANUP:
    opus_decoder_destroy(ac->decoder);
    jbuf_free((struct JitterBuffer *)ac->j_buf);
BASE_CLEANUP:
    pthread_mutex_destroy(ac->queue_mutex);
    free(ac);
    return nullptr;
}

void ac_kill(ACSession *ac)
{
    if (ac == nullptr) {
        return;
    }

    opus_encoder_destroy(ac->encoder);
    opus_decoder_destroy(ac->decoder);
    jbuf_free((struct JitterBuffer *)ac->j_buf);

    pthread_mutex_destroy(ac->queue_mutex);

    LOGGER_DEBUG(ac->log, "Terminated audio handler: %p", (void *)ac);
    free(ac);
}

void ac_iterate(ACSession *ac)
{
    if (ac == nullptr) {
        return;
    }

    /* TODO: fix this and jitter buffering */

    /* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */
    int16_t *temp_audio_buffer = (int16_t *)malloc(AUDIO_MAX_BUFFER_SIZE_PCM16 * AUDIO_MAX_CHANNEL_COUNT * sizeof(int16_t));

    if (temp_audio_buffer == nullptr) {
        LOGGER_ERROR(ac->log, "Failed to allocate memory for audio buffer");
        return;
    }

    pthread_mutex_lock(ac->queue_mutex);
    struct JitterBuffer *const j_buf = (struct JitterBuffer *)ac->j_buf;

    int rc = 0;

    for (struct RTPMessage *msg = jbuf_read(j_buf, &rc); msg != nullptr || rc == 2; msg = jbuf_read(j_buf, &rc)) {
        pthread_mutex_unlock(ac->queue_mutex);

        if (rc == 2) {
            LOGGER_DEBUG(ac->log, "OPUS correction");
            const int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000;
            rc = opus_decode(ac->decoder, nullptr, 0, temp_audio_buffer, fs, 1);
        } else {
            assert(msg->len > 4);

            /* Pick up sampling rate from packet */
            memcpy(&ac->lp_sampling_rate, msg->data, 4);
            ac->lp_sampling_rate = net_ntohl(ac->lp_sampling_rate);

            ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4);

            /* NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
             * it didn't work quite well.
             */
            if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) {
                LOGGER_WARNING(ac->log, "Failed to reconfigure decoder!");
                free(msg);
                pthread_mutex_lock(ac->queue_mutex);
                continue;
            }

            /*
             * frame_size = opus_decode(dec, packet, len, decoded, max_size, 0);
             *   where
             * packet is the byte array containing the compressed data
             * len is the exact number of bytes contained in the packet
             * decoded is the decoded audio data in opus_int16 (or float for opus_decode_float())
             * max_size is the max duration of the frame in samples (per channel) that can fit
             * into the decoded_frame array
             */
            rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, temp_audio_buffer, 5760, 0);
            free(msg);
        }

        if (rc < 0) {
            LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc));
        } else if (ac->acb != nullptr) {
            ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate;

            ac->acb(ac->av, ac->friend_number, temp_audio_buffer, rc, ac->lp_channel_count,
                    ac->lp_sampling_rate, ac->acb_user_data);
        }

        free(temp_audio_buffer);

        return;
    }

    pthread_mutex_unlock(ac->queue_mutex);

    free(temp_audio_buffer);
}

int ac_queue_message(Mono_Time *mono_time, void *cs, struct RTPMessage *msg)
{
    ACSession *ac = (ACSession *)cs;

    if (ac == nullptr || msg == nullptr) {
        free(msg);
        return -1;
    }

    if ((msg->header.pt & 0x7f) == (RTP_TYPE_AUDIO + 2) % 128) {
        LOGGER_WARNING(ac->log, "Got dummy!");
        free(msg);
        return 0;
    }

    if ((msg->header.pt & 0x7f) != RTP_TYPE_AUDIO % 128) {
        LOGGER_WARNING(ac->log, "Invalid payload type!");
        free(msg);
        return -1;
    }

    pthread_mutex_lock(ac->queue_mutex);
    const int rc = jbuf_write(ac->log, (struct JitterBuffer *)ac->j_buf, msg);
    pthread_mutex_unlock(ac->queue_mutex);

    if (rc == -1) {
        LOGGER_WARNING(ac->log, "Could not queue the message!");
        free(msg);
        return -1;
    }

    return 0;
}

int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels)
{
    if (ac == nullptr || !reconfigure_audio_encoder(
                ac->log, &ac->encoder, bit_rate,
                sampling_rate, channels,
                &ac->le_bit_rate,
                &ac->le_sample_rate,
                &ac->le_channel_count)) {
        return -1;
    }

    return 0;
}



struct JitterBuffer {
    struct RTPMessage **queue;
    uint32_t size;
    uint32_t capacity;
    uint16_t bottom;
    uint16_t top;
};

static struct JitterBuffer *jbuf_new(uint32_t capacity)
{
    unsigned int size = 1;

    while (size <= (capacity * 4)) {
        size *= 2;
    }

    struct JitterBuffer *q = (struct JitterBuffer *)calloc(1, sizeof(struct JitterBuffer));

    if (q == nullptr) {
        return nullptr;
    }

    q->queue = (struct RTPMessage **)calloc(size, sizeof(struct RTPMessage *));

    if (q->queue == nullptr) {
        free(q);
        return nullptr;
    }

    q->size = size;
    q->capacity = capacity;
    return q;
}
static void jbuf_clear(struct JitterBuffer *q)
{
    while (q->bottom != q->top) {
        free(q->queue[q->bottom % q->size]);
        q->queue[q->bottom % q->size] = nullptr;
        ++q->bottom;
    }
}
static void jbuf_free(struct JitterBuffer *q)
{
    if (q == nullptr) {
        return;
    }

    jbuf_clear(q);
    free(q->queue);
    free(q);
}
static int jbuf_write(const Logger *log, struct JitterBuffer *q, struct RTPMessage *m)
{
    const uint16_t sequnum = m->header.sequnum;

    const unsigned int num = sequnum % q->size;

    if ((uint32_t)(sequnum - q->bottom) > q->size) {
        LOGGER_DEBUG(log, "Clearing filled jitter buffer: %p", (void *)q);

        jbuf_clear(q);
        q->bottom = sequnum - q->capacity;
        q->queue[num] = m;
        q->top = sequnum + 1;
        return 0;
    }

    if (q->queue[num] != nullptr) {
        return -1;
    }

    q->queue[num] = m;

    if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
        q->top = sequnum + 1;
    }

    return 0;
}
static struct RTPMessage *jbuf_read(struct JitterBuffer *q, int32_t *success)
{
    if (q->top == q->bottom) {
        *success = 0;
        return nullptr;
    }

    const unsigned int num = q->bottom % q->size;

    if (q->queue[num] != nullptr) {
        struct RTPMessage *ret = q->queue[num];
        q->queue[num] = nullptr;
        ++q->bottom;
        *success = 1;
        return ret;
    }

    if ((uint32_t)(q->top - q->bottom) > q->capacity) {
        ++q->bottom;
        *success = 2;
        return nullptr;
    }

    *success = 0;
    return nullptr;
}
static OpusEncoder *create_audio_encoder(const Logger *log, uint32_t bit_rate, uint32_t sampling_rate,
        uint8_t channel_count)
{
    int status = OPUS_OK;
    /*
     * OPUS_APPLICATION_VOIP Process signal for improved speech intelligibility
     * OPUS_APPLICATION_AUDIO Favor faithfulness to the original input
     * OPUS_APPLICATION_RESTRICTED_LOWDELAY Configure the minimum possible coding delay
     */
    OpusEncoder *rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status);

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while starting audio encoder: %s", opus_strerror(status));
        return nullptr;
    }


    /*
     * Rates from 500 to 512000 bits per second are meaningful as well as the special
     * values OPUS_BITRATE_AUTO and OPUS_BITRATE_MAX. The value OPUS_BITRATE_MAX can
     * be used to cause the codec to use as much rate as it can, which is useful for
     * controlling the rate by adjusting the output buffer size.
     *
     * Parameters:
     *   `[in]`    `x`   `opus_int32`: bitrate in bits per second.
     */
    status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate));

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
        goto FAILURE;
    }


    /*
     * Configures the encoder's use of inband forward error correction.
     * Note:
     *   This is only applicable to the LPC layer
     * Parameters:
     *   `[in]`    `x`   `int`: FEC flag, 0 (disabled) is default
     */
    /* Enable in-band forward error correction in codec */
    status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1));

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
        goto FAILURE;
    }


    /*
     * Configures the encoder's expected packet loss percentage.
     * Higher values with trigger progressively more loss resistant behavior in
     * the encoder at the expense of quality at a given bitrate in the lossless case,
     * but greater quality under loss.
     * Parameters:
     *     `[in]`    `x`   `int`: Loss percentage in the range 0-100, inclusive.
     */
    /* Make codec resistant to up to 10% packet loss
     * NOTE This could also be adjusted on the fly, rather than hard-coded,
     *      with feedback from the receiving client.
     */
    status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(AUDIO_OPUS_PACKET_LOSS_PERC));

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
        goto FAILURE;
    }


    /*
     * Configures the encoder's computational complexity.
     *
     * The supported range is 0-10 inclusive with 10 representing the highest complexity.
     * The default value is 10.
     *
     * Parameters:
     *   `[in]`    `x`   `int`: 0-10, inclusive
     */
    /* Set algorithm to the highest complexity, maximizing compression */
    status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(AUDIO_OPUS_COMPLEXITY));

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
        goto FAILURE;
    }

    return rc;

FAILURE:
    opus_encoder_destroy(rc);
    return nullptr;
}

static bool reconfigure_audio_encoder(const Logger *log, OpusEncoder **e, uint32_t new_br, uint32_t new_sr,
                                      uint8_t new_ch, uint32_t *old_br, uint32_t *old_sr, uint8_t *old_ch)
{
    /* Values are checked in toxav.c */
    if (*old_sr != new_sr || *old_ch != new_ch) {
        OpusEncoder *new_encoder = create_audio_encoder(log, new_br, new_sr, new_ch);

        if (new_encoder == nullptr) {
            return false;
        }

        opus_encoder_destroy(*e);
        *e = new_encoder;
    } else if (*old_br == new_br) {
        return true; /* Nothing changed */
    }

    const int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br));

    if (status != OPUS_OK) {
        LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
        return false;
    }

    *old_br = new_br;
    *old_sr = new_sr;
    *old_ch = new_ch;

    LOGGER_DEBUG(log, "Reconfigured audio encoder br: %d sr: %d cc:%d", new_br, new_sr, new_ch);
    return true;
}

static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels)
{
    if (sampling_rate != ac->ld_sample_rate || channels != ac->ld_channel_count) {
        if (current_time_monotonic(ac->mono_time) - ac->ldrts < 500) {
            return false;
        }

        int status;
        OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status);

        if (status != OPUS_OK) {
            LOGGER_ERROR(ac->log, "Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
            return false;
        }

        ac->ld_sample_rate = sampling_rate;
        ac->ld_channel_count = channels;
        ac->ldrts = current_time_monotonic(ac->mono_time);

        opus_decoder_destroy(ac->decoder);
        ac->decoder = new_dec;

        LOGGER_DEBUG(ac->log, "Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
    }

    return true;
}

Coverage Report

Created: 2024-01-26 01:52

Line	Count	Source (jump to first uncovered line)
1		/* SPDX-License-Identifier: GPL-3.0-or-later
2		* Copyright © 2016-2018 The TokTok team.
3		* Copyright © 2013-2015 Tox project.
4		*/
5		#include "audio.h"
6
7		#include <assert.h>
8		#include <stdlib.h>
9		#include <string.h>
10
11		#include "rtp.h"
12
13		#include "../toxcore/ccompat.h"
14		#include "../toxcore/logger.h"
15		#include "../toxcore/mono_time.h"
16
17		static struct JitterBuffer *jbuf_new(uint32_t capacity);
18		static void jbuf_clear(struct JitterBuffer *q);
19		static void jbuf_free(struct JitterBuffer *q);
20		static int jbuf_write(const Logger log, struct JitterBuffer q, struct RTPMessage *m);
21		static struct RTPMessage jbuf_read(struct JitterBuffer q, int32_t *success);
22		static OpusEncoder create_audio_encoder(const Logger log, uint32_t bit_rate, uint32_t sampling_rate,
23		uint8_t channel_count);
24		static bool reconfigure_audio_encoder(const Logger log, OpusEncoder *e, uint32_t new_br, uint32_t new_sr,
25		uint8_t new_ch, uint32_t old_br, uint32_t old_sr, uint8_t *old_ch);
26		static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels);
27
28
29
30		ACSession ac_new(Mono_Time mono_time, const Logger log, ToxAV av, uint32_t friend_number,
31		toxav_audio_receive_frame_cb cb, void cb_data)
32	18	{
33	18	ACSession ac = (ACSession )calloc(1, sizeof(ACSession));
34
35	18	if (ac == nullptr) {
36	0	LOGGER_WARNING(log, "Allocation failed! Application might misbehave!");
37	0	return nullptr;
38	0	}
39
40	18	if (create_recursive_mutex(ac->queue_mutex) != 0) {
41	0	LOGGER_WARNING(log, "Failed to create recursive mutex!");
42	0	free(ac);
43	0	return nullptr;
44	0	}
45
46	18	int status;
47	18	ac->decoder = opus_decoder_create(AUDIO_DECODER_START_SAMPLE_RATE, AUDIO_DECODER_START_CHANNEL_COUNT, &status);
48
49	18	if (status != OPUS_OK) {
50	0	LOGGER_ERROR(log, "Error while starting audio decoder: %s", opus_strerror(status));
51	0	goto BASE_CLEANUP;
52	0	}
53
54	18	ac->j_buf = jbuf_new(AUDIO_JITTERBUFFER_COUNT);
55
56	18	if (ac->j_buf == nullptr) {
57	0	LOGGER_WARNING(log, "Jitter buffer creaton failed!");
58	0	opus_decoder_destroy(ac->decoder);
59	0	goto BASE_CLEANUP;
60	0	}
61
62	18	ac->mono_time = mono_time;
63	18	ac->log = log;
64
65		/* Initialize encoders with default values */
66	18	ac->encoder = create_audio_encoder(log, AUDIO_START_BITRATE, AUDIO_START_SAMPLE_RATE, AUDIO_START_CHANNEL_COUNT);
67
68	18	if (ac->encoder == nullptr) {
69	0	goto DECODER_CLEANUP;
70	0	}
71
72	18	ac->le_bit_rate = AUDIO_START_BITRATE;
73	18	ac->le_sample_rate = AUDIO_START_SAMPLE_RATE;
74	18	ac->le_channel_count = AUDIO_START_CHANNEL_COUNT;
75
76	18	ac->ld_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
77	18	ac->ld_sample_rate = AUDIO_DECODER_START_SAMPLE_RATE;
78	18	ac->ldrts = 0; /* Make it possible to reconfigure straight away */
79
80		/* These need to be set in order to properly
81		* do error correction with opus */
82	18	ac->lp_frame_duration = AUDIO_MAX_FRAME_DURATION_MS;
83	18	ac->lp_sampling_rate = AUDIO_DECODER_START_SAMPLE_RATE;
84	18	ac->lp_channel_count = AUDIO_DECODER_START_CHANNEL_COUNT;
85
86	18	ac->av = av;
87	18	ac->friend_number = friend_number;
88	18	ac->acb = cb;
89	18	ac->acb_user_data = cb_data;
90
91	18	return ac;
92
93	0	DECODER_CLEANUP:
94	0	opus_decoder_destroy(ac->decoder);
95	0	jbuf_free((struct JitterBuffer *)ac->j_buf);
96	0	BASE_CLEANUP:
97	0	pthread_mutex_destroy(ac->queue_mutex);
98	0	free(ac);
99	0	return nullptr;
100	0	}
101
102		void ac_kill(ACSession *ac)
103	18	{
104	18	if (ac == nullptr) {
105	0	return;
106	0	}
107
108	18	opus_encoder_destroy(ac->encoder);
109	18	opus_decoder_destroy(ac->decoder);
110	18	jbuf_free((struct JitterBuffer *)ac->j_buf);
111
112	18	pthread_mutex_destroy(ac->queue_mutex);
113
114	18	LOGGER_DEBUG(ac->log, "Terminated audio handler: %p", (void *)ac);
115	18	free(ac);
116	18	}
117
118		void ac_iterate(ACSession *ac)
119	280	{
120	280	if (ac == nullptr) {
121	0	return;
122	0	}
123
124		/* TODO: fix this and jitter buffering */
125
126		/* Enough space for the maximum frame size (120 ms 48 KHz stereo audio) */
127	280	int16_t temp_audio_buffer = (int16_t )malloc(AUDIO_MAX_BUFFER_SIZE_PCM16 * AUDIO_MAX_CHANNEL_COUNT * sizeof(int16_t));
128
129	280	if (temp_audio_buffer == nullptr) {
130	0	LOGGER_ERROR(ac->log, "Failed to allocate memory for audio buffer");
131	0	return;
132	0	}
133
134	280	pthread_mutex_lock(ac->queue_mutex);
135	280	struct JitterBuffer const j_buf = (struct JitterBuffer )ac->j_buf;
136
137	280	int rc = 0;
138
139	280	for (struct RTPMessage *msg = jbuf_read(j_buf, &rc); msg != nullptr \|\| rc == 2; msg = jbuf_read(j_buf, &rc)) {
140	131	pthread_mutex_unlock(ac->queue_mutex);
141
142	131	if (rc == 2) {
143	0	LOGGER_DEBUG(ac->log, "OPUS correction");
144	0	const int fs = (ac->lp_sampling_rate * ac->lp_frame_duration) / 1000;
145	0	rc = opus_decode(ac->decoder, nullptr, 0, temp_audio_buffer, fs, 1);
146	131	} else {
147	131	assert(msg->len > 4);
148
149		/* Pick up sampling rate from packet */
150	131	memcpy(&ac->lp_sampling_rate, msg->data, 4);
151	131	ac->lp_sampling_rate = net_ntohl(ac->lp_sampling_rate);
152
153	131	ac->lp_channel_count = opus_packet_get_nb_channels(msg->data + 4);
154
155		/* NOTE: even though OPUS supports decoding mono frames with stereo decoder and vice versa,
156		* it didn't work quite well.
157		*/
158	131	if (!reconfigure_audio_decoder(ac, ac->lp_sampling_rate, ac->lp_channel_count)) {
159	0	LOGGER_WARNING(ac->log, "Failed to reconfigure decoder!");
160	0	free(msg);
161	0	pthread_mutex_lock(ac->queue_mutex);
162	0	continue;
163	0	}
164
165		/*
166		* frame_size = opus_decode(dec, packet, len, decoded, max_size, 0);
167		* where
168		* packet is the byte array containing the compressed data
169		* len is the exact number of bytes contained in the packet
170		* decoded is the decoded audio data in opus_int16 (or float for opus_decode_float())
171		* max_size is the max duration of the frame in samples (per channel) that can fit
172		* into the decoded_frame array
173		*/
174	131	rc = opus_decode(ac->decoder, msg->data + 4, msg->len - 4, temp_audio_buffer, 5760, 0);
175	131	free(msg);
176	131	}
177
178	131	if (rc < 0) {
179	0	LOGGER_WARNING(ac->log, "Decoding error: %s", opus_strerror(rc));
180	131	} else if (ac->acb != nullptr) {
181	131	ac->lp_frame_duration = (rc * 1000) / ac->lp_sampling_rate;
182
183	131	ac->acb(ac->av, ac->friend_number, temp_audio_buffer, rc, ac->lp_channel_count,
184	131	ac->lp_sampling_rate, ac->acb_user_data);
185	131	}
186
187	131	free(temp_audio_buffer);
188
189	131	return;
190	131	}
191
192	149	pthread_mutex_unlock(ac->queue_mutex);
193
194	149	free(temp_audio_buffer);
195	149	}
196
197		int ac_queue_message(Mono_Time mono_time, void cs, struct RTPMessage *msg)
198	142	{
199	142	ACSession ac = (ACSession )cs;
200
201	142	if (ac == nullptr \|\| msg == nullptr) {
202	0	free(msg);
203	0	return -1;
204	0	}
205
206	142	if ((msg->header.pt & 0x7f) == (RTP_TYPE_AUDIO + 2) % 128) {
207	0	LOGGER_WARNING(ac->log, "Got dummy!");
208	0	free(msg);
209	0	return 0;
210	0	}
211
212	142	if ((msg->header.pt & 0x7f) != RTP_TYPE_AUDIO % 128) {
213	0	LOGGER_WARNING(ac->log, "Invalid payload type!");
214	0	free(msg);
215	0	return -1;
216	0	}
217
218	142	pthread_mutex_lock(ac->queue_mutex);
219	142	const int rc = jbuf_write(ac->log, (struct JitterBuffer *)ac->j_buf, msg);
220	142	pthread_mutex_unlock(ac->queue_mutex);
221
222	142	if (rc == -1) {
223	0	LOGGER_WARNING(ac->log, "Could not queue the message!");
224	0	free(msg);
225	0	return -1;
226	0	}
227
228	142	return 0;
229	142	}
230
231		int ac_reconfigure_encoder(ACSession *ac, uint32_t bit_rate, uint32_t sampling_rate, uint8_t channels)
232	145	{
233	145	if (ac == nullptr \|\| !reconfigure_audio_encoder(
234	145	ac->log, &ac->encoder, bit_rate,
235	145	sampling_rate, channels,
236	145	&ac->le_bit_rate,
237	145	&ac->le_sample_rate,
238	145	&ac->le_channel_count)) {
239	0	return -1;
240	0	}
241
242	145	return 0;
243	145	}
244
245
246
247		struct JitterBuffer {
248		struct RTPMessage **queue;
249		uint32_t size;
250		uint32_t capacity;
251		uint16_t bottom;
252		uint16_t top;
253		};
254
255		static struct JitterBuffer *jbuf_new(uint32_t capacity)
256	18	{
257	18	unsigned int size = 1;
258
259	90	while (size <= (capacity * 4)) {
260	72	size *= 2;
261	72	}
262
263	18	struct JitterBuffer q = (struct JitterBuffer )calloc(1, sizeof(struct JitterBuffer));
264
265	18	if (q == nullptr) {
266	0	return nullptr;
267	0	}
268
269	18	q->queue = (struct RTPMessage *)calloc(size, sizeof(struct RTPMessage ));
270
271	18	if (q->queue == nullptr) {
272	0	free(q);
273	0	return nullptr;
274	0	}
275
276	18	q->size = size;
277	18	q->capacity = capacity;
278	18	return q;
279	18	}
280		static void jbuf_clear(struct JitterBuffer *q)
281	18	{
282	29	while (q->bottom != q->top) {
283	11	free(q->queue[q->bottom % q->size]);
284	11	q->queue[q->bottom % q->size] = nullptr;
285	11	++q->bottom;
286	11	}
287	18	}
288		static void jbuf_free(struct JitterBuffer *q)
289	18	{
290	18	if (q == nullptr) {
291	0	return;
292	0	}
293
294	18	jbuf_clear(q);
295	18	free(q->queue);
296	18	free(q);
297	18	}
298		static int jbuf_write(const Logger log, struct JitterBuffer q, struct RTPMessage *m)
299	142	{
300	142	const uint16_t sequnum = m->header.sequnum;
301
302	142	const unsigned int num = sequnum % q->size;
303
304	142	if ((uint32_t)(sequnum - q->bottom) > q->size) {
305	0	LOGGER_DEBUG(log, "Clearing filled jitter buffer: %p", (void *)q);
306
307	0	jbuf_clear(q);
308	0	q->bottom = sequnum - q->capacity;
309	0	q->queue[num] = m;
310	0	q->top = sequnum + 1;
311	0	return 0;
312	0	}
313
314	142	if (q->queue[num] != nullptr) {
315	0	return -1;
316	0	}
317
318	142	q->queue[num] = m;
319
320	142	if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
321	142	q->top = sequnum + 1;
322	142	}
323
324	142	return 0;
325	142	}
326		static struct RTPMessage jbuf_read(struct JitterBuffer q, int32_t *success)
327	280	{
328	280	if (q->top == q->bottom) {
329	149	*success = 0;
330	149	return nullptr;
331	149	}
332
333	131	const unsigned int num = q->bottom % q->size;
334
335	131	if (q->queue[num] != nullptr) {
336	131	struct RTPMessage *ret = q->queue[num];
337	131	q->queue[num] = nullptr;
338	131	++q->bottom;
339	131	*success = 1;
340	131	return ret;
341	131	}
342
343	0	if ((uint32_t)(q->top - q->bottom) > q->capacity) {
344	0	++q->bottom;
345	0	*success = 2;
346	0	return nullptr;
347	0	}
348
349	0	*success = 0;
350	0	return nullptr;
351	0	}
352		static OpusEncoder create_audio_encoder(const Logger log, uint32_t bit_rate, uint32_t sampling_rate,
353		uint8_t channel_count)
354	26	{
355	26	int status = OPUS_OK;
356		/*
357		* OPUS_APPLICATION_VOIP Process signal for improved speech intelligibility
358		* OPUS_APPLICATION_AUDIO Favor faithfulness to the original input
359		* OPUS_APPLICATION_RESTRICTED_LOWDELAY Configure the minimum possible coding delay
360		*/
361	26	OpusEncoder *rc = opus_encoder_create(sampling_rate, channel_count, OPUS_APPLICATION_VOIP, &status);
362
363	26	if (status != OPUS_OK) {
364	0	LOGGER_ERROR(log, "Error while starting audio encoder: %s", opus_strerror(status));
365	0	return nullptr;
366	0	}
367
368
369		/*
370		* Rates from 500 to 512000 bits per second are meaningful as well as the special
371		* values OPUS_BITRATE_AUTO and OPUS_BITRATE_MAX. The value OPUS_BITRATE_MAX can
372		* be used to cause the codec to use as much rate as it can, which is useful for
373		* controlling the rate by adjusting the output buffer size.
374		*
375		* Parameters:
376		* `[in]` `x` `opus_int32`: bitrate in bits per second.
377		*/
378	26	status = opus_encoder_ctl(rc, OPUS_SET_BITRATE(bit_rate));
379
380	26	if (status != OPUS_OK) {
381	0	LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
382	0	goto FAILURE;
383	0	}
384
385
386		/*
387		* Configures the encoder's use of inband forward error correction.
388		* Note:
389		* This is only applicable to the LPC layer
390		* Parameters:
391		* `[in]` `x` `int`: FEC flag, 0 (disabled) is default
392		*/
393		/* Enable in-band forward error correction in codec */
394	26	status = opus_encoder_ctl(rc, OPUS_SET_INBAND_FEC(1));
395
396	26	if (status != OPUS_OK) {
397	0	LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
398	0	goto FAILURE;
399	0	}
400
401
402		/*
403		* Configures the encoder's expected packet loss percentage.
404		* Higher values with trigger progressively more loss resistant behavior in
405		* the encoder at the expense of quality at a given bitrate in the lossless case,
406		* but greater quality under loss.
407		* Parameters:
408		* `[in]` `x` `int`: Loss percentage in the range 0-100, inclusive.
409		*/
410		/* Make codec resistant to up to 10% packet loss
411		* NOTE This could also be adjusted on the fly, rather than hard-coded,
412		* with feedback from the receiving client.
413		*/
414	26	status = opus_encoder_ctl(rc, OPUS_SET_PACKET_LOSS_PERC(AUDIO_OPUS_PACKET_LOSS_PERC));
415
416	26	if (status != OPUS_OK) {
417	0	LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
418	0	goto FAILURE;
419	0	}
420
421
422		/*
423		* Configures the encoder's computational complexity.
424		*
425		* The supported range is 0-10 inclusive with 10 representing the highest complexity.
426		* The default value is 10.
427		*
428		* Parameters:
429		* `[in]` `x` `int`: 0-10, inclusive
430		*/
431		/* Set algorithm to the highest complexity, maximizing compression */
432	26	status = opus_encoder_ctl(rc, OPUS_SET_COMPLEXITY(AUDIO_OPUS_COMPLEXITY));
433
434	26	if (status != OPUS_OK) {
435	0	LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
436	0	goto FAILURE;
437	0	}
438
439	26	return rc;
440
441	0	FAILURE:
442	0	opus_encoder_destroy(rc);
443	0	return nullptr;
444	26	}
445
446		static bool reconfigure_audio_encoder(const Logger log, OpusEncoder *e, uint32_t new_br, uint32_t new_sr,
447		uint8_t new_ch, uint32_t old_br, uint32_t old_sr, uint8_t *old_ch)
448	145	{
449		/* Values are checked in toxav.c */
450	145	if (old_sr != new_sr \|\| old_ch != new_ch) {
451	8	OpusEncoder *new_encoder = create_audio_encoder(log, new_br, new_sr, new_ch);
452
453	8	if (new_encoder == nullptr) {
454	0	return false;
455	0	}
456
457	8	opus_encoder_destroy(*e);
458	8	*e = new_encoder;
459	137	} else if (*old_br == new_br) {
460	137	return true; /* Nothing changed */
461	137	}
462
463	8	const int status = opus_encoder_ctl(*e, OPUS_SET_BITRATE(new_br));
464
465	8	if (status != OPUS_OK) {
466	0	LOGGER_ERROR(log, "Error while setting encoder ctl: %s", opus_strerror(status));
467	0	return false;
468	0	}
469
470	8	*old_br = new_br;
471	8	*old_sr = new_sr;
472	8	*old_ch = new_ch;
473
474	8	LOGGER_DEBUG(log, "Reconfigured audio encoder br: %d sr: %d cc:%d", new_br, new_sr, new_ch);
475	8	return true;
476	8	}
477
478		static bool reconfigure_audio_decoder(ACSession *ac, uint32_t sampling_rate, uint8_t channels)
479	131	{
480	131	if (sampling_rate != ac->ld_sample_rate \|\| channels != ac->ld_channel_count) {
481	0	if (current_time_monotonic(ac->mono_time) - ac->ldrts < 500) {
482	0	return false;
483	0	}
484
485	0	int status;
486	0	OpusDecoder *new_dec = opus_decoder_create(sampling_rate, channels, &status);
487
488	0	if (status != OPUS_OK) {
489	0	LOGGER_ERROR(ac->log, "Error while starting audio decoder(%d %d): %s", sampling_rate, channels, opus_strerror(status));
490	0	return false;
491	0	}
492
493	0	ac->ld_sample_rate = sampling_rate;
494	0	ac->ld_channel_count = channels;
495	0	ac->ldrts = current_time_monotonic(ac->mono_time);
496
497	0	opus_decoder_destroy(ac->decoder);
498	0	ac->decoder = new_dec;
499
500	0	LOGGER_DEBUG(ac->log, "Reconfigured audio decoder sr: %d cc: %d", sampling_rate, channels);
501	0	}
502
503	131	return true;
504	131	}