/work/toxav/groupav.c

Source (jump to first uncovered line)
/* SPDX-License-Identifier: GPL-3.0-or-later
 * Copyright © 2016-2018 The TokTok team.
 * Copyright © 2014 Tox project.
 */
#include "groupav.h"

#include <stdlib.h>
#include <string.h>

#include "../toxcore/ccompat.h"
#include "../toxcore/logger.h"
#include "../toxcore/mono_time.h"
#include "../toxcore/tox_struct.h"
#include "../toxcore/util.h"

#define GROUP_JBUF_SIZE 6
#define GROUP_JBUF_DEAD_SECONDS 4

typedef struct Group_Audio_Packet {
    uint16_t sequnum;
    uint16_t length;
    uint8_t *data;
} Group_Audio_Packet;

typedef struct Group_JitterBuffer {
    Group_Audio_Packet **queue;
    uint32_t size;
    uint32_t capacity;
    uint16_t bottom;
    uint16_t top;
    uint64_t last_queued_time;
} Group_JitterBuffer;

static void free_audio_packet(Group_Audio_Packet *pk)
{
    if (pk == nullptr) {
        return;
    }

    free(pk->data);
    free(pk);
}

static Group_JitterBuffer *create_queue(unsigned int capacity)
{
    unsigned int size = 1;

    while (size <= capacity) {
        size *= 2;
    }

    Group_JitterBuffer *q = (Group_JitterBuffer *)calloc(1, sizeof(Group_JitterBuffer));

    if (q == nullptr) {
        return nullptr;
    }

    q->queue = (Group_Audio_Packet **)calloc(size, sizeof(Group_Audio_Packet *));

    if (q->queue == nullptr) {
        free(q);
        return nullptr;
    }

    q->size = size;
    q->capacity = capacity;
    return q;
}

static void clear_queue(Group_JitterBuffer *q)
{
    while (q->bottom != q->top) {
        const size_t idx = q->bottom % q->size;
        free_audio_packet(q->queue[idx]);
        q->queue[idx] = nullptr;
        ++q->bottom;
    }
}

static void terminate_queue(Group_JitterBuffer *q)
{
    if (q == nullptr) {
        return;
    }

    clear_queue(q);
    free(q->queue);
    free(q);
}

/** @retval 0 if packet was queued
 * @retval -1 if it wasn't.
 */
static int queue(Group_JitterBuffer *q, const Mono_Time *mono_time, Group_Audio_Packet *pk)
{
    const uint16_t sequnum = pk->sequnum;

    const unsigned int num = sequnum % q->size;

    if (!mono_time_is_timeout(mono_time, q->last_queued_time, GROUP_JBUF_DEAD_SECONDS)) {
        if ((uint32_t)(sequnum - q->bottom) > (1 << 15)) {
            /* Drop old packet. */
            return -1;
        }
    }

    if ((uint32_t)(sequnum - q->bottom) > q->size) {
        clear_queue(q);
        q->bottom = sequnum - q->capacity;
        q->queue[num] = pk;
        q->top = sequnum + 1;
        q->last_queued_time = mono_time_get(mono_time);
        return 0;
    }

    if (q->queue[num] != nullptr) {
        return -1;
    }

    q->queue[num] = pk;

    if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
        q->top = sequnum + 1;
    }

    q->last_queued_time = mono_time_get(mono_time);
    return 0;
}

/**
 * success is:
 * - 0 when there is nothing to dequeue
 * - 1 when there's a good packet
 * - 2 when there's a lost packet
 */
static Group_Audio_Packet *dequeue(Group_JitterBuffer *q, int *success)
{
    if (q->top == q->bottom) {
        *success = 0;
        return nullptr;
    }

    const unsigned int num = q->bottom % q->size;

    if (q->queue[num] != nullptr) {
        Group_Audio_Packet *ret = q->queue[num];
        q->queue[num] = nullptr;
        ++q->bottom;
        *success = 1;
        return ret;
    }

    if ((uint32_t)(q->top - q->bottom) > q->capacity) {
        ++q->bottom;
        *success = 2;
        return nullptr;
    }

    *success = 0;
    return nullptr;
}

typedef struct Group_AV {
    const Logger *log;
    Tox *tox;
    Group_Chats *g_c;
    OpusEncoder *audio_encoder;

    unsigned int audio_channels;
    unsigned int audio_sample_rate;
    unsigned int audio_bitrate;

    uint16_t audio_sequnum;

    audio_data_cb *audio_data;
    void *userdata;
} Group_AV;

typedef struct Group_Peer_AV {
    const Mono_Time *mono_time;
    Group_JitterBuffer *buffer;

    OpusDecoder *audio_decoder;
    int decoder_channels;
    unsigned int last_packet_samples;
} Group_Peer_AV;

static void kill_group_av(Group_AV *group_av)
{
    if (group_av->audio_encoder != nullptr) {
        opus_encoder_destroy(group_av->audio_encoder);
    }

    free(group_av);
}

static int recreate_encoder(Group_AV *group_av)
{
    if (group_av->audio_encoder != nullptr) {
        opus_encoder_destroy(group_av->audio_encoder);
        group_av->audio_encoder = nullptr;
    }

    int rc = OPUS_OK;
    group_av->audio_encoder = opus_encoder_create(group_av->audio_sample_rate, group_av->audio_channels,
                              OPUS_APPLICATION_AUDIO, &rc);

    if (rc != OPUS_OK) {
        LOGGER_ERROR(group_av->log, "Error while starting audio encoder: %s", opus_strerror(rc));
        group_av->audio_encoder = nullptr;
        return -1;
    }

    rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_BITRATE(group_av->audio_bitrate));

    if (rc != OPUS_OK) {
        LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
        opus_encoder_destroy(group_av->audio_encoder);
        group_av->audio_encoder = nullptr;
        return -1;
    }

    rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_COMPLEXITY(10));

    if (rc != OPUS_OK) {
        LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
        opus_encoder_destroy(group_av->audio_encoder);
        group_av->audio_encoder = nullptr;
        return -1;
    }

    return 0;
}

static Group_AV *new_group_av(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback,
                              void *userdata)
{
    if (g_c == nullptr) {
        return nullptr;
    }

    Group_AV *group_av = (Group_AV *)calloc(1, sizeof(Group_AV));

    if (group_av == nullptr) {
        return nullptr;
    }

    group_av->log = log;
    group_av->tox = tox;
    group_av->g_c = g_c;

    group_av->audio_data = audio_callback;
    group_av->userdata = userdata;

    return group_av;
}

static void group_av_peer_new(void *object, uint32_t conference_number, uint32_t peer_number)
{
    const Group_AV *group_av = (const Group_AV *)object;
    Group_Peer_AV *peer_av = (Group_Peer_AV *)calloc(1, sizeof(Group_Peer_AV));

    if (peer_av == nullptr) {
        return;
    }

    peer_av->mono_time = g_mono_time(group_av->g_c);
    peer_av->buffer = create_queue(GROUP_JBUF_SIZE);

    if (group_peer_set_object(group_av->g_c, conference_number, peer_number, peer_av) == -1) {
        free(peer_av);
    }
}

static void group_av_peer_delete(void *object, uint32_t conference_number, void *peer_object)
{
    Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;

    if (peer_av == nullptr) {
        return;
    }

    if (peer_av->audio_decoder != nullptr) {
        opus_decoder_destroy(peer_av->audio_decoder);
    }

    terminate_queue(peer_av->buffer);
    free(peer_object);
}

static void group_av_groupchat_delete(void *object, uint32_t conference_number)
{
    Group_AV *group_av = (Group_AV *)object;
    if (group_av != nullptr) {
        kill_group_av(group_av);
    }
}

static int decode_audio_packet(Group_AV *group_av, Group_Peer_AV *peer_av, uint32_t conference_number,
                               uint32_t peer_number)
{
    if (group_av == nullptr || peer_av == nullptr) {
        return -1;
    }

    int success;
    Group_Audio_Packet *pk = dequeue(peer_av->buffer, &success);

    if (success == 0) {
        return -1;
    }

    int16_t *out_audio = nullptr;
    int out_audio_samples = 0;

    const unsigned int sample_rate = 48000;

    if (success == 1) {
        const int channels = opus_packet_get_nb_channels(pk->data);

        if (channels == OPUS_INVALID_PACKET) {
            free_audio_packet(pk);
            return -1;
        }

        if (channels != 1 && channels != 2) {
            free_audio_packet(pk);
            return -1;
        }

        if (channels != peer_av->decoder_channels) {
            if (peer_av->audio_decoder != nullptr) {
                opus_decoder_destroy(peer_av->audio_decoder);
                peer_av->audio_decoder = nullptr;
            }

            int rc;
            peer_av->audio_decoder = opus_decoder_create(sample_rate, channels, &rc);

            if (rc != OPUS_OK) {
                LOGGER_ERROR(group_av->log, "Error while starting audio decoder: %s", opus_strerror(rc));
                free_audio_packet(pk);
                return -1;
            }

            peer_av->decoder_channels = channels;
        }

        const int num_samples = opus_decoder_get_nb_samples(peer_av->audio_decoder, pk->data, pk->length);

        out_audio = (int16_t *)malloc(num_samples * peer_av->decoder_channels * sizeof(int16_t));

        if (out_audio == nullptr) {
            free_audio_packet(pk);
            return -1;
        }

        out_audio_samples = opus_decode(peer_av->audio_decoder, pk->data, pk->length, out_audio, num_samples, 0);
        free_audio_packet(pk);

        if (out_audio_samples <= 0) {
            free(out_audio);
            return -1;
        }

        peer_av->last_packet_samples = out_audio_samples;
    } else {
        if (peer_av->audio_decoder == nullptr) {
            return -1;
        }

        if (peer_av->last_packet_samples == 0) {
            return -1;
        }

        out_audio = (int16_t *)malloc(peer_av->last_packet_samples * peer_av->decoder_channels * sizeof(int16_t));

        if (out_audio == nullptr) {
            free_audio_packet(pk);
            return -1;
        }

        out_audio_samples = opus_decode(peer_av->audio_decoder, nullptr, 0, out_audio, peer_av->last_packet_samples, 1);

        if (out_audio_samples <= 0) {
            free(out_audio);
            return -1;
        }
    }

    if (out_audio != nullptr) {

        if (group_av->audio_data != nullptr) {
            group_av->audio_data(group_av->tox, conference_number, peer_number, out_audio, out_audio_samples,
                                 peer_av->decoder_channels, sample_rate, group_av->userdata);
        }

        free(out_audio);
        return 0;
    }

    return -1;
}

static int handle_group_audio_packet(void *object, uint32_t conference_number, uint32_t peer_number, void *peer_object,
                                     const uint8_t *packet, uint16_t length)
{
    Group_AV *group_av = (Group_AV *)object;
    Group_Peer_AV *peer_av = (Group_Peer_AV *)peer_object;

    if (group_av == nullptr || peer_av == nullptr || length <= sizeof(uint16_t)) {
        return -1;
    }

    Group_Audio_Packet *pk = (Group_Audio_Packet *)calloc(1, sizeof(Group_Audio_Packet));

    if (pk == nullptr) {
        return -1;
    }

    net_unpack_u16(packet, &pk->sequnum);
    pk->length = length - sizeof(uint16_t);

    pk->data = (uint8_t *)malloc(pk->length);

    if (pk->data == nullptr) {
        free_audio_packet(pk);
        return -1;
    }

    memcpy(pk->data, packet + sizeof(uint16_t), pk->length);

    if (queue(peer_av->buffer, peer_av->mono_time, pk) == -1) {
        free_audio_packet(pk);
        return -1;
    }

    while (decode_audio_packet(group_av, peer_av, conference_number, peer_number) == 0) {
        /* Continue. */
    }

    return 0;
}

/** @brief Enable A/V in a conference.
 *
 * @retval 0 on success.
 * @retval -1 on failure.
 */
int groupchat_enable_av(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t conference_number,
                        audio_data_cb *audio_callback, void *userdata)
{
    if (group_get_type(g_c, conference_number) != GROUPCHAT_TYPE_AV
            || group_get_object(g_c, conference_number) != nullptr) {
        return -1;
    }

    Group_AV *group_av = new_group_av(log, tox, g_c, audio_callback, userdata);

    if (group_av == nullptr) {
        return -1;
    }

    if (group_set_object(g_c, conference_number, group_av) == -1
            || callback_groupchat_peer_new(g_c, conference_number, group_av_peer_new) == -1
            || callback_groupchat_peer_delete(g_c, conference_number, group_av_peer_delete) == -1
            || callback_groupchat_delete(g_c, conference_number, group_av_groupchat_delete) == -1) {
        kill_group_av(group_av);
        return -1;
    }

    const int numpeers = group_number_peers(g_c, conference_number, false);

    if (numpeers < 0) {
        kill_group_av(group_av);
        return -1;
    }

    for (uint32_t i = 0; i < numpeers; ++i) {
        group_av_peer_new(group_av, conference_number, i);
    }

    group_lossy_packet_registerhandler(g_c, GROUP_AUDIO_PACKET_ID, &handle_group_audio_packet);
    return 0;
}

/** @brief Disable A/V in a conference.
 *
 * @retval 0 on success.
 * @retval -1 on failure.
 */
int groupchat_disable_av(const Group_Chats *g_c, uint32_t conference_number)
{
    if (group_get_type(g_c, conference_number) != GROUPCHAT_TYPE_AV) {
        return -1;
    }

    Group_AV *group_av = (Group_AV *)group_get_object(g_c, conference_number);

    if (group_av == nullptr) {
        return -1;
    }

    const int numpeers = group_number_peers(g_c, conference_number, false);

    if (numpeers < 0) {
        kill_group_av(group_av);
        return -1;
    }

    for (uint32_t i = 0; i < numpeers; ++i) {
        group_av_peer_delete(group_av, conference_number, group_peer_get_object(g_c, conference_number, i));
        group_peer_set_object(g_c, conference_number, i, nullptr);
    }

    kill_group_av(group_av);

    if (group_set_object(g_c, conference_number, nullptr) == -1
            || callback_groupchat_peer_new(g_c, conference_number, nullptr) == -1
            || callback_groupchat_peer_delete(g_c, conference_number, nullptr) == -1
            || callback_groupchat_delete(g_c, conference_number, nullptr) == -1) {
        return -1;
    }

    return 0;
}

/** Return whether A/V is enabled in the conference. */
bool groupchat_av_enabled(const Group_Chats *g_c, uint32_t conference_number)
{
    return group_get_object(g_c, conference_number) != nullptr;
}

/** @brief Create and connect to a new toxav group.
 *
 * @return conference number on success.
 * @retval -1 on failure.
 */
int add_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, audio_data_cb *audio_callback, void *userdata)
{
    const int conference_number = add_groupchat(g_c, tox->sys.rng, GROUPCHAT_TYPE_AV);

    if (conference_number == -1) {
        return -1;
    }

    if (groupchat_enable_av(log, tox, g_c, conference_number, audio_callback, userdata) == -1) {
        del_groupchat(g_c, conference_number, true);
        return -1;
    }

    return conference_number;
}

/** @brief Join a AV group (you need to have been invited first).
 *
 * @return conference number on success
 * @retval -1 on failure.
 */
int join_av_groupchat(const Logger *log, Tox *tox, Group_Chats *g_c, uint32_t peer_number, const uint8_t *data,
                      uint16_t length, audio_data_cb *audio_callback, void *userdata)
{
    const int conference_number = join_groupchat(g_c, peer_number, GROUPCHAT_TYPE_AV, data, length);

    if (conference_number == -1) {
        return -1;
    }

    if (groupchat_enable_av(log, tox, g_c, conference_number, audio_callback, userdata) == -1) {
        del_groupchat(g_c, conference_number, true);
        return -1;
    }

    return conference_number;
}

/** @brief Send an encoded audio packet to the conference.
 *
 * @retval 0 on success.
 * @retval -1 on failure.
 */
static int send_audio_packet(const Group_Chats *g_c, uint32_t conference_number, const uint8_t *packet, uint16_t length)
{
    if (length == 0 || length > MAX_CRYPTO_DATA_SIZE - 1 - sizeof(uint16_t)) {
        return -1;
    }

    const uint16_t plen = 1 + sizeof(uint16_t) + length;

    Group_AV *const group_av = (Group_AV *)group_get_object(g_c, conference_number);

    if (group_av == nullptr) {
        return -1;
    }

    uint8_t data[MAX_CRYPTO_DATA_SIZE];
    uint8_t *ptr = data;
    *ptr = GROUP_AUDIO_PACKET_ID;
    ++ptr;

    ptr += net_pack_u16(ptr, group_av->audio_sequnum);
    memcpy(ptr, packet, length);

    if (send_group_lossy_packet(g_c, conference_number, data, plen) == -1) {
        return -1;
    }

    ++group_av->audio_sequnum;
    return 0;
}

/** @brief Send audio to the conference.
 *
 * @retval 0 on success.
 * @retval -1 on failure.
 */
int group_send_audio(const Group_Chats *g_c, uint32_t conference_number, const int16_t *pcm, unsigned int samples, uint8_t channels,
                     uint32_t sample_rate)
{
    Group_AV *group_av = (Group_AV *)group_get_object(g_c, conference_number);

    if (group_av == nullptr) {
        return -1;
    }

    if (channels != 1 && channels != 2) {
        return -1;
    }

    if (sample_rate != 8000 && sample_rate != 12000 && sample_rate != 16000 && sample_rate != 24000
            && sample_rate != 48000) {
        return -1;
    }

    if (group_av->audio_encoder == nullptr || group_av->audio_channels != channels
            || group_av->audio_sample_rate != sample_rate) {
        group_av->audio_channels = channels;
        group_av->audio_sample_rate = sample_rate;

        if (channels == 1) {
            group_av->audio_bitrate = 32000; // TODO(mannol): add way of adjusting bitrate
        } else {
            group_av->audio_bitrate = 64000; // TODO(mannol): add way of adjusting bitrate
        }

        if (recreate_encoder(group_av) == -1) {
            return -1;
        }
    }

    uint8_t encoded[1024];
    const int32_t size = opus_encode(group_av->audio_encoder, pcm, samples, encoded, sizeof(encoded));

    if (size <= 0) {
        return -1;
    }

    return send_audio_packet(g_c, conference_number, encoded, size);
}

Coverage Report

Created: 2024-01-26 01:52

Line	Count	Source (jump to first uncovered line)
1		/* SPDX-License-Identifier: GPL-3.0-or-later
2		* Copyright © 2016-2018 The TokTok team.
3		* Copyright © 2014 Tox project.
4		*/
5		#include "groupav.h"
6
7		#include <stdlib.h>
8		#include <string.h>
9
10		#include "../toxcore/ccompat.h"
11		#include "../toxcore/logger.h"
12		#include "../toxcore/mono_time.h"
13		#include "../toxcore/tox_struct.h"
14		#include "../toxcore/util.h"
15
16	488	#define GROUP_JBUF_SIZE 6
17	10.4k	#define GROUP_JBUF_DEAD_SECONDS 4
18
19		typedef struct Group_Audio_Packet {
20		uint16_t sequnum;
21		uint16_t length;
22		uint8_t *data;
23		} Group_Audio_Packet;
24
25		typedef struct Group_JitterBuffer {
26		Group_Audio_Packet **queue;
27		uint32_t size;
28		uint32_t capacity;
29		uint16_t bottom;
30		uint16_t top;
31		uint64_t last_queued_time;
32		} Group_JitterBuffer;
33
34		static void free_audio_packet(Group_Audio_Packet *pk)
35	10.4k	{
36	10.4k	if (pk == nullptr) {
37	0	return;
38	0	}
39
40	10.4k	free(pk->data);
41	10.4k	free(pk);
42	10.4k	}
43
44		static Group_JitterBuffer *create_queue(unsigned int capacity)
45	488	{
46	488	unsigned int size = 1;
47
48	1.95k	while (size <= capacity) {
49	1.46k	size *= 2;
50	1.46k	}
51
52	488	Group_JitterBuffer q = (Group_JitterBuffer )calloc(1, sizeof(Group_JitterBuffer));
53
54	488	if (q == nullptr) {
55	0	return nullptr;
56	0	}
57
58	488	q->queue = (Group_Audio_Packet *)calloc(size, sizeof(Group_Audio_Packet ));
59
60	488	if (q->queue == nullptr) {
61	0	free(q);
62	0	return nullptr;
63	0	}
64
65	488	q->size = size;
66	488	q->capacity = capacity;
67	488	return q;
68	488	}
69
70		static void clear_queue(Group_JitterBuffer *q)
71	656	{
72	656	while (q->bottom != q->top) {
73	0	const size_t idx = q->bottom % q->size;
74	0	free_audio_packet(q->queue[idx]);
75	0	q->queue[idx] = nullptr;
76	0	++q->bottom;
77	0	}
78	656	}
79
80		static void terminate_queue(Group_JitterBuffer *q)
81	488	{
82	488	if (q == nullptr) {
83	0	return;
84	0	}
85
86	488	clear_queue(q);
87	488	free(q->queue);
88	488	free(q);
89	488	}
90
91		/** @retval 0 if packet was queued
92		* @retval -1 if it wasn't.
93		*/
94		static int queue(Group_JitterBuffer q, const Mono_Time mono_time, Group_Audio_Packet *pk)
95	10.4k	{
96	10.4k	const uint16_t sequnum = pk->sequnum;
97
98	10.4k	const unsigned int num = sequnum % q->size;
99
100	10.4k	if (!mono_time_is_timeout(mono_time, q->last_queued_time, GROUP_JBUF_DEAD_SECONDS)) {
101	8.95k	if ((uint32_t)(sequnum - q->bottom) > (1 << 15)) {
102		/* Drop old packet. */
103	0	return -1;
104	0	}
105	8.95k	}
106
107	10.4k	if ((uint32_t)(sequnum - q->bottom) > q->size) {
108	168	clear_queue(q);
109	168	q->bottom = sequnum - q->capacity;
110	168	q->queue[num] = pk;
111	168	q->top = sequnum + 1;
112	168	q->last_queued_time = mono_time_get(mono_time);
113	168	return 0;
114	168	}
115
116	10.2k	if (q->queue[num] != nullptr) {
117	0	return -1;
118	0	}
119
120	10.2k	q->queue[num] = pk;
121
122	10.2k	if ((sequnum - q->bottom) >= (q->top - q->bottom)) {
123	10.2k	q->top = sequnum + 1;
124	10.2k	}
125
126	10.2k	q->last_queued_time = mono_time_get(mono_time);
127	10.2k	return 0;
128	10.2k	}
129
130		/**
131		* success is:
132		* - 0 when there is nothing to dequeue
133		* - 1 when there's a good packet
134		* - 2 when there's a lost packet
135		*/
136		static Group_Audio_Packet dequeue(Group_JitterBuffer q, int *success)
137	21.0k	{
138	21.0k	if (q->top == q->bottom) {
139	9.56k	*success = 0;
140	9.56k	return nullptr;
141	9.56k	}
142
143	11.4k	const unsigned int num = q->bottom % q->size;
144
145	11.4k	if (q->queue[num] != nullptr) {
146	10.4k	Group_Audio_Packet *ret = q->queue[num];
147	10.4k	q->queue[num] = nullptr;
148	10.4k	++q->bottom;
149	10.4k	*success = 1;
150	10.4k	return ret;
151	10.4k	}
152
153	1.02k	if ((uint32_t)(q->top - q->bottom) > q->capacity) {
154	1.00k	++q->bottom;
155	1.00k	*success = 2;
156	1.00k	return nullptr;
157	1.00k	}
158
159	22	*success = 0;
160	22	return nullptr;
161	1.02k	}
162
163		typedef struct Group_AV {
164		const Logger *log;
165		Tox *tox;
166		Group_Chats *g_c;
167		OpusEncoder *audio_encoder;
168
169		unsigned int audio_channels;
170		unsigned int audio_sample_rate;
171		unsigned int audio_bitrate;
172
173		uint16_t audio_sequnum;
174
175		audio_data_cb *audio_data;
176		void *userdata;
177		} Group_AV;
178
179		typedef struct Group_Peer_AV {
180		const Mono_Time *mono_time;
181		Group_JitterBuffer *buffer;
182
183		OpusDecoder *audio_decoder;
184		int decoder_channels;
185		unsigned int last_packet_samples;
186		} Group_Peer_AV;
187
188		static void kill_group_av(Group_AV *group_av)
189	23	{
190	23	if (group_av->audio_encoder != nullptr) {
191	23	opus_encoder_destroy(group_av->audio_encoder);
192	23	}
193
194	23	free(group_av);
195	23	}
196
197		static int recreate_encoder(Group_AV *group_av)
198	23	{
199	23	if (group_av->audio_encoder != nullptr) {
200	0	opus_encoder_destroy(group_av->audio_encoder);
201	0	group_av->audio_encoder = nullptr;
202	0	}
203
204	23	int rc = OPUS_OK;
205	23	group_av->audio_encoder = opus_encoder_create(group_av->audio_sample_rate, group_av->audio_channels,
206	23	OPUS_APPLICATION_AUDIO, &rc);
207
208	23	if (rc != OPUS_OK) {
209	0	LOGGER_ERROR(group_av->log, "Error while starting audio encoder: %s", opus_strerror(rc));
210	0	group_av->audio_encoder = nullptr;
211	0	return -1;
212	0	}
213
214	23	rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_BITRATE(group_av->audio_bitrate));
215
216	23	if (rc != OPUS_OK) {
217	0	LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
218	0	opus_encoder_destroy(group_av->audio_encoder);
219	0	group_av->audio_encoder = nullptr;
220	0	return -1;
221	0	}
222
223	23	rc = opus_encoder_ctl(group_av->audio_encoder, OPUS_SET_COMPLEXITY(10));
224
225	23	if (rc != OPUS_OK) {
226	0	LOGGER_ERROR(group_av->log, "Error while setting encoder ctl: %s", opus_strerror(rc));
227	0	opus_encoder_destroy(group_av->audio_encoder);
228	0	group_av->audio_encoder = nullptr;
229	0	return -1;
230	0	}
231
232	23	return 0;
233	23	}
234
235		static Group_AV new_group_av(const Logger log, Tox tox, Group_Chats g_c, audio_data_cb *audio_callback,
236		void *userdata)
237	23	{
238	23	if (g_c == nullptr) {
239	0	return nullptr;
240	0	}
241
242	23	Group_AV group_av = (Group_AV )calloc(1, sizeof(Group_AV));
243
244	23	if (group_av == nullptr) {
245	0	return nullptr;
246	0	}
247
248	23	group_av->log = log;
249	23	group_av->tox = tox;
250	23	group_av->g_c = g_c;
251
252	23	group_av->audio_data = audio_callback;
253	23	group_av->userdata = userdata;
254
255	23	return group_av;
256	23	}
257
258		static void group_av_peer_new(void *object, uint32_t conference_number, uint32_t peer_number)
259	488	{
260	488	const Group_AV group_av = (const Group_AV )object;
261	488	Group_Peer_AV peer_av = (Group_Peer_AV )calloc(1, sizeof(Group_Peer_AV));
262
263	488	if (peer_av == nullptr) {
264	0	return;
265	0	}
266
267	488	peer_av->mono_time = g_mono_time(group_av->g_c);
268	488	peer_av->buffer = create_queue(GROUP_JBUF_SIZE);
269
270	488	if (group_peer_set_object(group_av->g_c, conference_number, peer_number, peer_av) == -1) {
271	0	free(peer_av);
272	0	}
273	488	}
274
275		static void group_av_peer_delete(void object, uint32_t conference_number, void peer_object)
276	488	{
277	488	Group_Peer_AV peer_av = (Group_Peer_AV )peer_object;
278
279	488	if (peer_av == nullptr) {
280	0	return;
281	0	}
282
283	488	if (peer_av->audio_decoder != nullptr) {
284	450	opus_decoder_destroy(peer_av->audio_decoder);
285	450	}
286
287	488	terminate_queue(peer_av->buffer);
288	488	free(peer_object);
289	488	}
290
291		static void group_av_groupchat_delete(void *object, uint32_t conference_number)
292	15	{
293	15	Group_AV group_av = (Group_AV )object;
294	15	if (group_av != nullptr) {
295	15	kill_group_av(group_av);
296	15	}
297	15	}
298
299		static int decode_audio_packet(Group_AV group_av, Group_Peer_AV peer_av, uint32_t conference_number,
300		uint32_t peer_number)
301	21.0k	{
302	21.0k	if (group_av == nullptr \|\| peer_av == nullptr) {
303	0	return -1;
304	0	}
305
306	21.0k	int success;
307	21.0k	Group_Audio_Packet *pk = dequeue(peer_av->buffer, &success);
308
309	21.0k	if (success == 0) {
310	9.58k	return -1;
311	9.58k	}
312
313	11.4k	int16_t *out_audio = nullptr;
314	11.4k	int out_audio_samples = 0;
315
316	11.4k	const unsigned int sample_rate = 48000;
317
318	11.4k	if (success == 1) {
319	10.4k	const int channels = opus_packet_get_nb_channels(pk->data);
320
321	10.4k	if (channels == OPUS_INVALID_PACKET) {
322	0	free_audio_packet(pk);
323	0	return -1;
324	0	}
325
326	10.4k	if (channels != 1 && channels != 2) {
327	0	free_audio_packet(pk);
328	0	return -1;
329	0	}
330
331	10.4k	if (channels != peer_av->decoder_channels) {
332	450	if (peer_av->audio_decoder != nullptr) {
333	0	opus_decoder_destroy(peer_av->audio_decoder);
334	0	peer_av->audio_decoder = nullptr;
335	0	}
336
337	450	int rc;
338	450	peer_av->audio_decoder = opus_decoder_create(sample_rate, channels, &rc);
339
340	450	if (rc != OPUS_OK) {
341	0	LOGGER_ERROR(group_av->log, "Error while starting audio decoder: %s", opus_strerror(rc));
342	0	free_audio_packet(pk);
343	0	return -1;
344	0	}
345
346	450	peer_av->decoder_channels = channels;
347	450	}
348
349	10.4k	const int num_samples = opus_decoder_get_nb_samples(peer_av->audio_decoder, pk->data, pk->length);
350
351	10.4k	out_audio = (int16_t )malloc(num_samples peer_av->decoder_channels * sizeof(int16_t));
352
353	10.4k	if (out_audio == nullptr) {
354	0	free_audio_packet(pk);
355	0	return -1;
356	0	}
357
358	10.4k	out_audio_samples = opus_decode(peer_av->audio_decoder, pk->data, pk->length, out_audio, num_samples, 0);
359	10.4k	free_audio_packet(pk);
360
361	10.4k	if (out_audio_samples <= 0) {
362	0	free(out_audio);
363	0	return -1;
364	0	}
365
366	10.4k	peer_av->last_packet_samples = out_audio_samples;
367	10.4k	} else {
368	1.00k	if (peer_av->audio_decoder == nullptr) {
369	862	return -1;
370	862	}
371
372	144	if (peer_av->last_packet_samples == 0) {
373	0	return -1;
374	0	}
375
376	144	out_audio = (int16_t )malloc(peer_av->last_packet_samples peer_av->decoder_channels * sizeof(int16_t));
377
378	144	if (out_audio == nullptr) {
379	0	free_audio_packet(pk);
380	0	return -1;
381	0	}
382
383	144	out_audio_samples = opus_decode(peer_av->audio_decoder, nullptr, 0, out_audio, peer_av->last_packet_samples, 1);
384
385	144	if (out_audio_samples <= 0) {
386	0	free(out_audio);
387	0	return -1;
388	0	}
389	144	}
390
391	10.5k	if (out_audio != nullptr) {
392
393	10.5k	if (group_av->audio_data != nullptr) {
394	10.5k	group_av->audio_data(group_av->tox, conference_number, peer_number, out_audio, out_audio_samples,
395	10.5k	peer_av->decoder_channels, sample_rate, group_av->userdata);
396	10.5k	}
397
398	10.5k	free(out_audio);
399	10.5k	return 0;
400	10.5k	}
401
402	0	return -1;
403	10.5k	}
404
405		static int handle_group_audio_packet(void object, uint32_t conference_number, uint32_t peer_number, void peer_object,
406		const uint8_t *packet, uint16_t length)
407	12.5k	{
408	12.5k	Group_AV group_av = (Group_AV )object;
409	12.5k	Group_Peer_AV peer_av = (Group_Peer_AV )peer_object;
410
411	12.5k	if (group_av == nullptr \|\| peer_av == nullptr \|\| length <= sizeof(uint16_t)) {
412	2.07k	return -1;
413	2.07k	}
414
415	10.4k	Group_Audio_Packet pk = (Group_Audio_Packet )calloc(1, sizeof(Group_Audio_Packet));
416
417	10.4k	if (pk == nullptr) {
418	0	return -1;
419	0	}
420
421	10.4k	net_unpack_u16(packet, &pk->sequnum);
422	10.4k	pk->length = length - sizeof(uint16_t);
423
424	10.4k	pk->data = (uint8_t *)malloc(pk->length);
425
426	10.4k	if (pk->data == nullptr) {
427	0	free_audio_packet(pk);
428	0	return -1;
429	0	}
430
431	10.4k	memcpy(pk->data, packet + sizeof(uint16_t), pk->length);
432
433	10.4k	if (queue(peer_av->buffer, peer_av->mono_time, pk) == -1) {
434	0	free_audio_packet(pk);
435	0	return -1;
436	0	}
437
438	21.0k	while (decode_audio_packet(group_av, peer_av, conference_number, peer_number) == 0) {
439		/* Continue. */
440	10.5k	}
441
442	10.4k	return 0;
443	10.4k	}
444
445		/** @brief Enable A/V in a conference.
446		*
447		* @retval 0 on success.
448		* @retval -1 on failure.
449		*/
450		int groupchat_enable_av(const Logger log, Tox tox, Group_Chats *g_c, uint32_t conference_number,
451		audio_data_cb audio_callback, void userdata)
452	31	{
453	31	if (group_get_type(g_c, conference_number) != GROUPCHAT_TYPE_AV
454	31	\|\| group_get_object(g_c, conference_number) != nullptr) {
455	8	return -1;
456	8	}
457
458	23	Group_AV *group_av = new_group_av(log, tox, g_c, audio_callback, userdata);
459
460	23	if (group_av == nullptr) {
461	0	return -1;
462	0	}
463
464	23	if (group_set_object(g_c, conference_number, group_av) == -1
465	23	\|\| callback_groupchat_peer_new(g_c, conference_number, group_av_peer_new) == -1
466	23	\|\| callback_groupchat_peer_delete(g_c, conference_number, group_av_peer_delete) == -1
467	23	\|\| callback_groupchat_delete(g_c, conference_number, group_av_groupchat_delete) == -1) {
468	0	kill_group_av(group_av);
469	0	return -1;
470	0	}
471
472	23	const int numpeers = group_number_peers(g_c, conference_number, false);
473
474	23	if (numpeers < 0) {
475	0	kill_group_av(group_av);
476	0	return -1;
477	0	}
478
479	136	for (uint32_t i = 0; i < numpeers; ++i) {
480	113	group_av_peer_new(group_av, conference_number, i);
481	113	}
482
483	23	group_lossy_packet_registerhandler(g_c, GROUP_AUDIO_PACKET_ID, &handle_group_audio_packet);
484	23	return 0;
485	23	}
486
487		/** @brief Disable A/V in a conference.
488		*
489		* @retval 0 on success.
490		* @retval -1 on failure.
491		*/
492		int groupchat_disable_av(const Group_Chats *g_c, uint32_t conference_number)
493	11	{
494	11	if (group_get_type(g_c, conference_number) != GROUPCHAT_TYPE_AV) {
495	0	return -1;
496	0	}
497
498	11	Group_AV group_av = (Group_AV )group_get_object(g_c, conference_number);
499
500	11	if (group_av == nullptr) {
501	3	return -1;
502	3	}
503
504	8	const int numpeers = group_number_peers(g_c, conference_number, false);
505
506	8	if (numpeers < 0) {
507	0	kill_group_av(group_av);
508	0	return -1;
509	0	}
510
511	136	for (uint32_t i = 0; i < numpeers; ++i) {
512	128	group_av_peer_delete(group_av, conference_number, group_peer_get_object(g_c, conference_number, i));
513	128	group_peer_set_object(g_c, conference_number, i, nullptr);
514	128	}
515
516	8	kill_group_av(group_av);
517
518	8	if (group_set_object(g_c, conference_number, nullptr) == -1
519	8	\|\| callback_groupchat_peer_new(g_c, conference_number, nullptr) == -1
520	8	\|\| callback_groupchat_peer_delete(g_c, conference_number, nullptr) == -1
521	8	\|\| callback_groupchat_delete(g_c, conference_number, nullptr) == -1) {
522	0	return -1;
523	0	}
524
525	8	return 0;
526	8	}
527
528		/** Return whether A/V is enabled in the conference. */
529		bool groupchat_av_enabled(const Group_Chats *g_c, uint32_t conference_number)
530	11	{
531	11	return group_get_object(g_c, conference_number) != nullptr;
532	11	}
533
534		/** @brief Create and connect to a new toxav group.
535		*
536		* @return conference number on success.
537		* @retval -1 on failure.
538		*/
539		int add_av_groupchat(const Logger log, Tox tox, Group_Chats g_c, audio_data_cb audio_callback, void *userdata)
540	1	{
541	1	const int conference_number = add_groupchat(g_c, tox->sys.rng, GROUPCHAT_TYPE_AV);
542
543	1	if (conference_number == -1) {
544	0	return -1;
545	0	}
546
547	1	if (groupchat_enable_av(log, tox, g_c, conference_number, audio_callback, userdata) == -1) {
548	0	del_groupchat(g_c, conference_number, true);
549	0	return -1;
550	0	}
551
552	1	return conference_number;
553	1	}
554
555		/** @brief Join a AV group (you need to have been invited first).
556		*
557		* @return conference number on success
558		* @retval -1 on failure.
559		*/
560		int join_av_groupchat(const Logger log, Tox tox, Group_Chats g_c, uint32_t peer_number, const uint8_t data,
561		uint16_t length, audio_data_cb audio_callback, void userdata)
562	15	{
563	15	const int conference_number = join_groupchat(g_c, peer_number, GROUPCHAT_TYPE_AV, data, length);
564
565	15	if (conference_number == -1) {
566	0	return -1;
567	0	}
568
569	15	if (groupchat_enable_av(log, tox, g_c, conference_number, audio_callback, userdata) == -1) {
570	0	del_groupchat(g_c, conference_number, true);
571	0	return -1;
572	0	}
573
574	15	return conference_number;
575	15	}
576
577		/** @brief Send an encoded audio packet to the conference.
578		*
579		* @retval 0 on success.
580		* @retval -1 on failure.
581		*/
582		static int send_audio_packet(const Group_Chats g_c, uint32_t conference_number, const uint8_t packet, uint16_t length)
583	848	{
584	848	if (length == 0 \|\| length > MAX_CRYPTO_DATA_SIZE - 1 - sizeof(uint16_t)) {
585	0	return -1;
586	0	}
587
588	848	const uint16_t plen = 1 + sizeof(uint16_t) + length;
589
590	848	Group_AV const group_av = (Group_AV )group_get_object(g_c, conference_number);
591
592	848	if (group_av == nullptr) {
593	0	return -1;
594	0	}
595
596	848	uint8_t data[MAX_CRYPTO_DATA_SIZE];
597	848	uint8_t *ptr = data;
598	848	*ptr = GROUP_AUDIO_PACKET_ID;
599	848	++ptr;
600
601	848	ptr += net_pack_u16(ptr, group_av->audio_sequnum);
602	848	memcpy(ptr, packet, length);
603
604	848	if (send_group_lossy_packet(g_c, conference_number, data, plen) == -1) {
605	0	return -1;
606	0	}
607
608	848	++group_av->audio_sequnum;
609	848	return 0;
610	848	}
611
612		/** @brief Send audio to the conference.
613		*
614		* @retval 0 on success.
615		* @retval -1 on failure.
616		*/
617		int group_send_audio(const Group_Chats g_c, uint32_t conference_number, const int16_t pcm, unsigned int samples, uint8_t channels,
618		uint32_t sample_rate)
619	848	{
620	848	Group_AV group_av = (Group_AV )group_get_object(g_c, conference_number);
621
622	848	if (group_av == nullptr) {
623	0	return -1;
624	0	}
625
626	848	if (channels != 1 && channels != 2) {
627	0	return -1;
628	0	}
629
630	848	if (sample_rate != 8000 && sample_rate != 12000 && sample_rate != 16000 && sample_rate != 24000
631	848	&& sample_rate != 48000) {
632	0	return -1;
633	0	}
634
635	848	if (group_av->audio_encoder == nullptr \|\| group_av->audio_channels != channels
636	848	\|\| group_av->audio_sample_rate != sample_rate) {
637	23	group_av->audio_channels = channels;
638	23	group_av->audio_sample_rate = sample_rate;
639
640	23	if (channels == 1) {
641	23	group_av->audio_bitrate = 32000; // TODO(mannol): add way of adjusting bitrate
642	23	} else {
643	0	group_av->audio_bitrate = 64000; // TODO(mannol): add way of adjusting bitrate
644	0	}
645
646	23	if (recreate_encoder(group_av) == -1) {
647	0	return -1;
648	0	}
649	23	}
650
651	848	uint8_t encoded[1024];
652	848	const int32_t size = opus_encode(group_av->audio_encoder, pcm, samples, encoded, sizeof(encoded));
653
654	848	if (size <= 0) {
655	0	return -1;
656	0	}
657
658	848	return send_audio_packet(g_c, conference_number, encoded, size);
659	848	}