Asterisk - The Open Source Telephony Project  18.5.0
bridge_softmix.c
Go to the documentation of this file.
1 /*
2  * Asterisk -- An open source telephony toolkit.
3  *
4  * Copyright (C) 2011, Digium, Inc.
5  *
6  * Joshua Colp <[email protected]>
7  * David Vossel <[email protected]>
8  *
9  * See http://www.asterisk.org for more information about
10  * the Asterisk project. Please do not directly contact
11  * any of the maintainers of this project for assistance;
12  * the project provides a web site, mailing lists and IRC
13  * channels for your use.
14  *
15  * This program is free software, distributed under the terms of
16  * the GNU General Public License Version 2. See the LICENSE file
17  * at the top of the source tree.
18  */
19 
20 /*! \file
21  *
22  * \brief Multi-party software based channel mixing
23  *
24  * \author Joshua Colp <[email protected]>
25  * \author David Vossel <[email protected]>
26  *
27  * \ingroup bridges
28  */
29 
30 /*** MODULEINFO
31  <support_level>core</support_level>
32  ***/
33 
34 #include "asterisk.h"
35 
36 #include <math.h>
37 
38 #include "asterisk/stream.h"
39 #include "asterisk/test.h"
40 #include "asterisk/vector.h"
41 #include "asterisk/message.h"
43 
44 /*! The minimum sample rate of the bridge. */
45 #define SOFTMIX_MIN_SAMPLE_RATE 8000 /* 8 kHz sample rate */
46 
47 /*! \brief Interval at which mixing will take place. Valid options are 10, 20, and 40. */
48 #define DEFAULT_SOFTMIX_INTERVAL 20
49 
50 /*! \brief Size of the buffer used for sample manipulation */
51 #define SOFTMIX_DATALEN(rate, interval) ((rate/50) * (interval / 10))
52 
53 /*! \brief Number of samples we are dealing with */
54 #define SOFTMIX_SAMPLES(rate, interval) (SOFTMIX_DATALEN(rate, interval) / 2)
55 
56 /*! \brief Number of mixing iterations to perform between gathering statistics. */
57 #define SOFTMIX_STAT_INTERVAL 100
58 
59 /*!
60  * \brief Default time in ms of silence necessary to declare talking stopped by the bridge.
61  *
62  * \details
63  * This is the time at which a channel's own audio will stop getting
64  * mixed out of its own write audio stream because it is no longer talking.
65  */
66 #define DEFAULT_SOFTMIX_SILENCE_THRESHOLD 2500
67 
68 /*! Default minimum average magnitude threshold to determine talking by the DSP. */
69 #define DEFAULT_SOFTMIX_TALKING_THRESHOLD 160
70 
71 #define SOFTBRIDGE_VIDEO_DEST_PREFIX "softbridge_dest"
72 #define SOFTBRIDGE_VIDEO_DEST_LEN strlen(SOFTBRIDGE_VIDEO_DEST_PREFIX)
73 #define SOFTBRIDGE_VIDEO_DEST_SEPARATOR '_'
74 
76  /*! The frame which will be given to each source stream */
77  struct ast_frame frame;
78  /*! The REMB to send to the source which is collecting REMB reports */
80  /*! The maximum bitrate (A single precision floating point is big enough) */
81  float bitrate;
82 };
83 
84 struct softmix_stats {
85  /*! Each index represents a sample rate used above the internal rate. */
86  unsigned int sample_rates[16];
87  /*! Each index represents the number of channels using the same index in the sample_rates array. */
88  unsigned int num_channels[16];
89  /*! The number of channels above the internal sample rate */
91  /*! The number of channels above the maximum sample rate */
92  unsigned int num_above_maximum_rate;
93  /*! The number of channels at the internal sample rate */
94  unsigned int num_at_internal_rate;
95  /*! The absolute highest sample rate preferred by any channel in the bridge */
96  unsigned int highest_supported_rate;
97  /*! Is the sample rate locked by the bridge, if so what is that rate.*/
98  unsigned int locked_rate;
99  /*! The maximum sample rate the bridge may use */
100  unsigned int maximum_rate;
101 };
102 
104  int num_times_requested; /*!< Once this entry is no longer requested, free the trans_pvt
105  and re-init if it was usable. */
106  struct ast_format *dst_format; /*!< The destination format for this helper */
107  struct ast_trans_pvt *trans_pvt; /*!< the translator for this slot. */
108  struct ast_frame *out_frame; /*!< The output frame from the last translation */
110 };
111 
113  struct ast_format *slin_src; /*!< the source format expected for all the translators */
115 };
116 
118 {
120  if (!(entry = ast_calloc(1, sizeof(*entry)))) {
121  return NULL;
122  }
123  entry->dst_format = ao2_bump(dst);
124  /* initialize this to one so that the first time through the cleanup code after
125  allocation it won't be removed from the entry list */
126  entry->num_times_requested = 1;
127  return entry;
128 }
129 
131 {
132  ao2_cleanup(entry->dst_format);
133 
134  if (entry->trans_pvt) {
136  }
137  if (entry->out_frame) {
138  ast_frfree(entry->out_frame);
139  }
140  ast_free(entry);
141  return NULL;
142 }
143 
144 static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
145 {
146  memset(trans_helper, 0, sizeof(*trans_helper));
147  trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
148 }
149 
151 {
153 
154  while ((entry = AST_LIST_REMOVE_HEAD(&trans_helper->entries, entry))) {
156  }
157 }
158 
159 static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
160 {
162 
163  trans_helper->slin_src = ast_format_cache_get_slin_by_rate(sample_rate);
164  AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
165  if (entry->trans_pvt) {
167  if (!(entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src))) {
170  }
171  }
172  }
174 }
175 
176 /*!
177  * \internal
178  * \brief Get the next available audio on the softmix channel's read stream
179  * and determine if it should be mixed out or not on the write stream.
180  *
181  * \retval pointer to buffer containing the exact number of samples requested on success.
182  * \retval NULL if no samples are present
183  */
184 static int16_t *softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
185 {
186  if ((ast_slinfactory_available(&sc->factory) >= num_samples) &&
187  ast_slinfactory_read(&sc->factory, sc->our_buf, num_samples)) {
188  sc->have_audio = 1;
189  return sc->our_buf;
190  }
191  sc->have_audio = 0;
192  return NULL;
193 }
194 
195 /*!
196  * \internal
197  * \brief Process a softmix channel's write audio
198  *
199  * \details This function will remove the channel's talking from its own audio if present and
200  * possibly even do the channel's write translation for it depending on how many other
201  * channels use the same write format.
202  */
203 static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper,
204  struct ast_format *raw_write_fmt,
205  struct softmix_channel *sc, unsigned int default_sample_size)
206 {
208  int i;
209 
210  /* If we provided any audio then take it out while in slinear format. */
211  if (sc->have_audio && !sc->binaural) {
212  for (i = 0; i < sc->write_frame.samples; i++) {
214  }
215  /* check to see if any entries exist for the format. if not we'll want
216  to remove it during cleanup */
217  AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
218  if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
219  ++entry->num_times_requested;
220  break;
221  }
222  }
223  /* do not do any special write translate optimization if we had to make
224  * a special mix for them to remove their own audio. */
225  return;
226  } else if (sc->have_audio && sc->binaural > 0) {
227  /*
228  * Binaural audio requires special saturated substract since we have two
229  * audio signals per channel now.
230  */
231  softmix_process_write_binaural_audio(sc, default_sample_size);
232  return;
233  }
234 
235  /* Attempt to optimize channels using the same translation path/codec. Build a list of entries
236  of translation paths and track the number of references for each type. Each one of the same
237  type should be able to use the same out_frame. Since the optimization is only necessary for
238  multiple channels (>=2) using the same codec make sure resources are allocated only when
239  needed and released when not (see also softmix_translate_helper_cleanup */
240  AST_LIST_TRAVERSE(&trans_helper->entries, entry, entry) {
241  if (sc->binaural != 0) {
242  continue;
243  }
244  if (ast_format_cmp(entry->dst_format, raw_write_fmt) == AST_FORMAT_CMP_EQUAL) {
245  entry->num_times_requested++;
246  } else {
247  continue;
248  }
249  if (!entry->trans_pvt && (entry->num_times_requested > 1)) {
250  entry->trans_pvt = ast_translator_build_path(entry->dst_format, trans_helper->slin_src);
251  }
252  if (entry->trans_pvt && !entry->out_frame) {
253  entry->out_frame = ast_translate(entry->trans_pvt, &sc->write_frame, 0);
254  }
255  if (entry->out_frame && entry->out_frame->frametype == AST_FRAME_VOICE
256  && entry->out_frame->datalen < MAX_DATALEN) {
258  memcpy(sc->final_buf, entry->out_frame->data.ptr, entry->out_frame->datalen);
259  sc->write_frame.datalen = entry->out_frame->datalen;
260  sc->write_frame.samples = entry->out_frame->samples;
261  }
262  break;
263  }
264 
265  /* add new entry into list if this format destination was not matched. */
266  if (!entry && (entry = softmix_translate_helper_entry_alloc(raw_write_fmt))) {
267  AST_LIST_INSERT_HEAD(&trans_helper->entries, entry, entry);
268  }
269 }
270 
272 {
274 
275  AST_LIST_TRAVERSE_SAFE_BEGIN(&trans_helper->entries, entry, entry) {
276  /* if it hasn't been requested then remove it */
277  if (!entry->num_times_requested) {
280  continue;
281  }
282 
283  if (entry->out_frame) {
284  ast_frfree(entry->out_frame);
285  entry->out_frame = NULL;
286  }
287 
288  /* nothing is optimized for a single path reference, so there is
289  no reason to continue to hold onto the codec */
290  if (entry->num_times_requested == 1 && entry->trans_pvt) {
292  entry->trans_pvt = NULL;
293  }
294 
295  /* for each iteration (a mixing run) in the bridge softmix thread the number
296  of references to a given entry is recalculated, so reset the number of
297  times requested */
298  entry->num_times_requested = 0;
299  }
301 }
302 
303 static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
304 {
305  struct softmix_channel *sc = bridge_channel->tech_pvt;
306  struct ast_format *slin_format;
307  int setup_fail;
308 
309 #ifdef BINAURAL_RENDERING
310  if (interval != BINAURAL_MIXING_INTERVAL) {
311  interval = BINAURAL_MIXING_INTERVAL;
312  }
313 #endif
314 
315  /* The callers have already ensured that sc is never NULL. */
316  ast_assert(sc != NULL);
317 
318  slin_format = ast_format_cache_get_slin_by_rate(rate);
319 
320  ast_mutex_lock(&sc->lock);
321  if (reset) {
323  ast_dsp_free(sc->dsp);
324  }
325 
326  /* Setup write frame parameters */
328  /*
329  * NOTE: The write_frame format holds a reference because translation
330  * could be needed and the format changed to the translated format
331  * for the channel. The translated format may not be a
332  * static cached format.
333  */
334  ao2_replace(sc->write_frame.subclass.format, slin_format);
335  sc->write_frame.data.ptr = sc->final_buf;
336  sc->write_frame.datalen = SOFTMIX_DATALEN(rate, interval);
337  sc->write_frame.samples = SOFTMIX_SAMPLES(rate, interval);
338 
339  /* We will store the rate here cause we need to set the data again when a channel is unsuspended */
340  sc->rate = rate;
341 
342  /* If the channel will contain binaural data we will set a identifier in the channel
343  * if set_binaural == -1 this is just a sample rate update, will ignore it. */
344  if (set_binaural == 1) {
345  sc->binaural = 1;
346  } else if (set_binaural == 0) {
347  sc->binaural = 0;
348  }
349 
350  /* Setting the binaural position. This doesn't require a change of the overlaying channel infos
351  * and doesn't have to be done if we just updating sample rates. */
352  if (binaural_pos_id != -1) {
353  sc->binaural_pos = binaural_pos_id;
354  }
355  if (is_announcement != -1) {
356  sc->is_announcement = is_announcement;
357  }
358 
359  /*
360  * NOTE: The read_slin_format does not hold a reference because it
361  * will always be a signed linear format.
362  */
363  sc->read_slin_format = slin_format;
364 
365  /* Setup smoother */
366  setup_fail = ast_slinfactory_init_with_format(&sc->factory, slin_format);
367 
368  /* set new read and write formats on channel. */
369  ast_channel_lock(bridge_channel->chan);
370  setup_fail |= ast_set_read_format_path(bridge_channel->chan,
371  ast_channel_rawreadformat(bridge_channel->chan), slin_format);
372  ast_channel_unlock(bridge_channel->chan);
373 
374  /* If channel contains binaural data we will set it here for the trans_pvt. */
375  if (set_binaural == 1 || (set_binaural == -1 && sc->binaural == 1)) {
376  setup_fail |= ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
377  } else if (set_binaural == 0) {
378  setup_fail |= ast_set_write_format(bridge_channel->chan, slin_format);
379  }
380 
381  /* set up new DSP. This is on the read side only right before the read frame enters the smoother. */
382  sc->dsp = ast_dsp_new_with_rate(rate);
383  if (setup_fail || !sc->dsp) {
384  /* Bad news. Could not setup the channel for softmix. */
385  ast_mutex_unlock(&sc->lock);
387  return;
388  }
389 
390  /* we want to aggressively detect silence to avoid feedback */
391  if (bridge_channel->tech_args.talking_threshold) {
393  } else {
395  }
396 
397  ast_mutex_unlock(&sc->lock);
398 }
399 
400 /*!
401  * \internal
402  * \brief Poke the mixing thread in case it is waiting for an active channel.
403  * \since 12.0.0
404  *
405  * \param softmix_data Bridge mixing data.
406  *
407  * \return Nothing
408  */
409 static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)
410 {
411  ast_mutex_lock(&softmix_data->lock);
412  ast_cond_signal(&softmix_data->cond);
413  ast_mutex_unlock(&softmix_data->lock);
414 }
415 
416 /*! \brief Function called when a channel is unsuspended from the bridge */
417 static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
418 {
419 #ifdef BINAURAL_RENDERING
420  struct softmix_channel *sc = bridge_channel->tech_pvt;
421  if (sc->binaural) {
422  /* Restore some usefull data if it was a binaural channel */
423  struct ast_format *slin_format;
424 
425  slin_format = ast_format_cache_get_slin_by_rate(sc->rate);
426  ast_set_write_format_interleaved_stereo(bridge_channel->chan, slin_format);
427  }
428 #endif
429  if (bridge->tech_pvt) {
430  softmix_poke_thread(bridge->tech_pvt);
431  }
432 }
433 
434 /*!
435  * \brief Determine if a stream is a video source stream.
436  *
437  * \param stream The stream to test
438  * \retval 1 The stream is a video source
439  * \retval 0 The stream is not a video source
440  */
441 static int is_video_source(const struct ast_stream *stream)
442 {
447  return 1;
448  }
449 
450  return 0;
451 }
452 
453 /*!
454  * \brief Determine if a stream is a video destination stream.
455  *
456  * A source channel name can be provided to narrow this to a destination stream
457  * for a particular source channel. Further, a source stream name can be provided
458  * to narrow this to a particular source stream's destination. However, empty strings
459  * can be provided to match any destination video stream, regardless of source channel
460  * or source stream.
461  *
462  * \param stream The stream to test
463  * \param source_channel_name The name of a source video channel to match
464  * \param source_channel_stream_position The position of the video on the source channel
465  * \retval 1 The stream is a video destination stream
466  * \retval 0 The stream is not a video destination stream
467  */
468 static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name,
469  int source_channel_stream_position)
470 {
471  char *dest_video_name;
472  size_t dest_video_name_len;
473 
476  return 0;
477  }
478 
479  dest_video_name_len = SOFTBRIDGE_VIDEO_DEST_LEN + 1;
480  if (!ast_strlen_zero(source_channel_name)) {
481  dest_video_name_len += strlen(source_channel_name) + 1;
482  if (source_channel_stream_position != -1) {
483  dest_video_name_len += 11;
484  }
485 
486  dest_video_name = ast_alloca(dest_video_name_len);
487  if (source_channel_stream_position != -1) {
488  /* We are looking for an exact stream position */
489  snprintf(dest_video_name, dest_video_name_len, "%s%c%s%c%d",
491  source_channel_name, SOFTBRIDGE_VIDEO_DEST_SEPARATOR,
492  source_channel_stream_position);
493  return !strcmp(ast_stream_get_name(stream), dest_video_name);
494  }
495  snprintf(dest_video_name, dest_video_name_len, "%s%c%s",
497  source_channel_name);
498  } else {
499  dest_video_name = SOFTBRIDGE_VIDEO_DEST_PREFIX;
500  }
501 
502  return !strncmp(ast_stream_get_name(stream), dest_video_name, dest_video_name_len - 1);
503 }
504 
506  const char *channel_name, const char *sdp_label,
507  struct ast_stream *stream, int index)
508 {
509  char *stream_clone_name = NULL;
510  struct ast_stream *stream_clone;
511 
512  /* We use the stream topology index for the stream to uniquely identify and recognize it.
513  * This is guaranteed to remain the same across renegotiation of the source channel and
514  * ensures that the stream name is unique.
515  */
516  if (ast_asprintf(&stream_clone_name, "%s%c%s%c%d", SOFTBRIDGE_VIDEO_DEST_PREFIX,
518  index) < 0) {
519  return -1;
520  }
521 
522  stream_clone = ast_stream_clone(stream, stream_clone_name);
523  ast_free(stream_clone_name);
524  if (!stream_clone) {
525  return -1;
526  }
527 
528  /* Sends an "a:label" attribute in the SDP for participant event correlation */
529  if (!ast_strlen_zero(sdp_label)) {
530  ast_stream_set_metadata(stream_clone, "SDP:LABEL", sdp_label);
531  }
532 
533  /* We will be sending them a stream and not expecting anything in return */
535 
536  if (ast_stream_topology_append_stream(dest, stream_clone) < 0) {
537  ast_stream_free(stream_clone);
538  return -1;
539  }
540 
541  return 0;
542 }
543 
544 
546  const char *channel_name, const char *sdp_label,
547  const struct ast_stream_topology *source)
548 {
549  int i;
550 
551  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
552  struct ast_stream *stream;
553 
554  stream = ast_stream_topology_get_stream(source, i);
555 
556  if (!is_video_source(stream)) {
557  continue;
558  }
559 
560  if (append_source_stream(dest, channel_name, sdp_label, stream, i)) {
561  return -1;
562  }
563  }
564 
565  return 0;
566 }
567 
568 static int append_all_streams(struct ast_stream_topology *dest,
569  const struct ast_stream_topology *source)
570 {
571  int i;
572  int dest_index = 0;
573 
574  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
575  struct ast_stream *clone;
576  int added = 0;
577 
579  if (!clone) {
580  return -1;
581  }
582 
583  /* If we can reuse an existing removed stream then do so */
584  while (dest_index < ast_stream_topology_get_count(dest)) {
585  struct ast_stream *stream = ast_stream_topology_get_stream(dest, dest_index);
586 
587  dest_index++;
588 
590  /* This cannot fail because dest_index - 1 is less than the
591  * current count in dest. */
592  ast_stream_topology_set_stream(dest, dest_index - 1, clone);
593  added = 1;
594  break;
595  }
596  }
597 
598  /* If no removed stream exists that we took the place of append the stream */
599  if (!added && ast_stream_topology_append_stream(dest, clone) < 0) {
600  ast_stream_free(clone);
601  return -1;
602  }
603  }
604 
605  return 0;
606 }
607 
608 /*!
609  * \brief Issue channel stream topology change requests.
610  *
611  * When in SFU mode, each participant needs to be able to
612  * send video directly to other participants in the bridge.
613  * This means that all participants need to have their topologies
614  * updated. The joiner needs to have destination streams for
615  * all current participants, and the current participants need
616  * to have destinations streams added for the joiner's sources.
617  *
618  * \param joiner The channel that is joining the softmix bridge
619  * \param participants The current participants in the softmix bridge
620  */
621 static void sfu_topologies_on_join(struct ast_bridge *bridge,
622  struct ast_bridge_channel *joiner)
623 {
624  RAII_VAR(struct ast_stream_topology *, joiner_video, NULL, ast_stream_topology_free);
625  struct ast_bridge_channels_list *participants = &bridge->channels;
626  struct ast_bridge_channel *participant;
627  int res;
628  struct softmix_channel *sc;
629  SCOPE_ENTER(3, "%s: \n", ast_channel_name(joiner->chan));
630 
631  joiner_video = ast_stream_topology_alloc();
632  if (!joiner_video) {
633  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't alloc topology\n", ast_channel_name(joiner->chan));
634  }
635 
636  sc = joiner->tech_pvt;
637 
638  ast_channel_lock(joiner->chan);
639  res = append_source_streams(joiner_video, ast_channel_name(joiner->chan),
640  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(joiner->chan) : NULL,
643  ast_channel_unlock(joiner->chan);
644 
645  if (res || !sc->topology) {
646  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't append source streams\n", ast_channel_name(joiner->chan));
647  }
648 
649  AST_LIST_TRAVERSE(participants, participant, entry) {
650  if (participant == joiner) {
651  continue;
652  }
653  ast_trace(-1, "%s: Appending existing participant %s\n", ast_channel_name(joiner->chan),
654  ast_channel_name(participant->chan));
655  ast_channel_lock(participant->chan);
656  res = append_source_streams(sc->topology, ast_channel_name(participant->chan),
657  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(participant->chan) : NULL,
658  ast_channel_get_stream_topology(participant->chan));
659  ast_channel_unlock(participant->chan);
660  if (res) {
661  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append source streams\n",
662  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
663  }
664  }
665 
666  ast_trace(-1, "%s: Requesting topology change.\n", ast_channel_name(joiner->chan));
668  if (res) {
669  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s: Couldn't request topology change\n", ast_channel_name(joiner->chan));
670  }
671 
672  AST_LIST_TRAVERSE(participants, participant, entry) {
673  if (participant == joiner) {
674  continue;
675  }
676 
677  sc = participant->tech_pvt;
678  ast_trace(-1, "%s: Appending joiner %s\n", ast_channel_name(participant->chan),
679  ast_channel_name(joiner->chan));
680 
681  if (append_all_streams(sc->topology, joiner_video)) {
682  SCOPE_EXIT_LOG_RTN(LOG_ERROR, "%s/%s: Couldn't append streams\n",
683  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
684  }
685  ast_trace(-1, "%s: Requesting topology change\n", ast_channel_name(participant->chan));
687  if (res) {
688  ast_trace(-1, "%s/%s: Couldn't request topology change\n",
689  ast_channel_name(participant->chan), ast_channel_name(joiner->chan));
690  }
691  }
692 
693  SCOPE_EXIT();
694 }
695 
696 /*! \brief Function called when a channel is joined into the bridge */
697 static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
698 {
699  struct softmix_channel *sc;
700  struct softmix_bridge_data *softmix_data;
701  int set_binaural = 0;
702  /*
703  * If false, the channel will be convolved, but since it is a non stereo channel, output
704  * will be mono.
705  */
706  int skip_binaural_output = 1;
707  int pos_id;
708  int is_announcement = 0;
709  int samplerate_change;
710  SCOPE_ENTER(3, "%s:\n", ast_channel_name(bridge_channel->chan));
711 
712  softmix_data = bridge->tech_pvt;
713  if (!softmix_data) {
714  SCOPE_EXIT_RTN_VALUE(-1, "No tech_pvt\n");
715  }
716 
717  /* Create a new softmix_channel structure and allocate various things on it */
718  if (!(sc = ast_calloc(1, sizeof(*sc)))) {
719  SCOPE_EXIT_RTN_VALUE(-1, "Couldn't alloc tech_pvt\n");
720  }
721 
722  samplerate_change = softmix_data->internal_rate;
723  pos_id = -1;
724  if (bridge->softmix.binaural_active) {
725  if (strncmp(ast_channel_name(bridge_channel->chan), "CBAnn", 5) != 0) {
726  set_binaural = ast_format_get_channel_count(bridge_channel->write_format) > 1 ? 1 : 0;
727  if (set_binaural) {
728  softmix_data->internal_rate = samplerate_change;
729  }
730  skip_binaural_output = 0;
731  } else {
732  is_announcement = 1;
733  }
734  if (set_binaural) {
735  softmix_data->convolve.binaural_active = 1;
736  }
737  if (!skip_binaural_output) {
738  pos_id = set_binaural_data_join(&softmix_data->convolve, softmix_data->default_sample_size);
739  if (pos_id == -1) {
740  ast_log(LOG_ERROR, "Bridge %s: Failed to join channel %s. "
741  "Could not allocate enough memory.\n", bridge->uniqueid,
742  ast_channel_name(bridge_channel->chan));
743  ast_free(sc);
744  SCOPE_EXIT_RTN_VALUE(-1, "Couldn't do binaural join\n");
745  }
746  }
747  }
748 
749  /* Can't forget the lock */
750  ast_mutex_init(&sc->lock);
751 
752  /* Can't forget to record our pvt structure within the bridged channel structure */
753  bridge_channel->tech_pvt = sc;
754 
756  softmix_data->internal_mixing_interval
757  ? softmix_data->internal_mixing_interval
759  bridge_channel, 0, set_binaural, pos_id, is_announcement);
760 
762  sfu_topologies_on_join(bridge, bridge_channel);
763  }
764 
765  /* Complete any active hold before entering, or transitioning to softmix. */
766  if (ast_channel_hold_state(bridge_channel->chan) == AST_CONTROL_HOLD) {
767  ast_debug(1, "Channel %s simulating UNHOLD for bridge softmix join.\n",
768  ast_channel_name(bridge_channel->chan));
769  ast_indicate(bridge_channel->chan, AST_CONTROL_UNHOLD);
770  }
771 
772  softmix_poke_thread(softmix_data);
774 }
775 
777  const char *channel_name)
778 {
779  int i;
780  int stream_removed = 0;
781 
782  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
783  struct ast_stream *stream;
784 
785  stream = ast_stream_topology_get_stream(topology, i);
786 
787  if (is_video_dest(stream, channel_name, -1)) {
789  stream_removed = 1;
790  }
791  }
792  return stream_removed;
793 }
794 
795 static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
796 {
797  struct ast_bridge_channel *participant;
798  struct softmix_channel *sc;
799 
800  AST_LIST_TRAVERSE(participants, participant, entry) {
801  sc = participant->tech_pvt;
803  continue;
804  }
806  }
807 
808  sc = leaver->tech_pvt;
809  if (remove_destination_streams(sc->topology, "")) {
811  }
812 
813  return 0;
814 }
815 
816 /*! \brief Function called when a channel leaves the bridge */
817 static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
818 {
819  struct softmix_channel *sc;
820  struct softmix_bridge_data *softmix_data;
821 
822  softmix_data = bridge->tech_pvt;
823  sc = bridge_channel->tech_pvt;
824  if (!sc) {
825  return;
826  }
827 
829  sfu_topologies_on_leave(bridge_channel, &bridge->channels);
830  }
831 
832  if (bridge->softmix.binaural_active) {
833  if (sc->binaural) {
834  set_binaural_data_leave(&softmix_data->convolve, sc->binaural_pos,
835  softmix_data->default_sample_size);
836  }
837  }
838 
839  bridge_channel->tech_pvt = NULL;
840 
842 
844 
845  AST_VECTOR_FREE(&sc->video_sources);
846 
847  /* Drop mutex lock */
848  ast_mutex_destroy(&sc->lock);
849 
850  /* Drop the factory */
852 
853  /* Drop any formats on the frames */
855 
856  /* Drop the DSP */
857  ast_dsp_free(sc->dsp);
858 
859  /* Eep! drop ourselves */
860  ast_free(sc);
861 }
862 
864 {
865  struct ast_bridge_channel *cur;
866 
867  AST_LIST_TRAVERSE(&bridge->channels, cur, entry) {
868  if (cur->suspended) {
869  continue;
870  }
871  if (ast_bridge_is_video_src(bridge, cur->chan) == 1) {
872  ast_bridge_channel_queue_frame(cur, frame);
873  break;
874  }
875  }
876 }
877 
878 /*!
879  * \internal
880  * \brief Determine what to do with a video frame.
881  * \since 12.0.0
882  *
883  * \param bridge Which bridge is getting the frame
884  * \param bridge_channel Which channel is writing the frame.
885  * \param frame What is being written.
886  *
887  * \return Nothing
888  */
889 static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
890 {
891  struct softmix_channel *sc;
892  int video_src_priority;
893 
894  /* Determine if the video frame should be distributed or not */
895  switch (bridge->softmix.video_mode.mode) {
897  break;
899  video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
900  if (video_src_priority == 1) {
901  /* Pass to me and everyone else. */
902  ast_bridge_queue_everyone_else(bridge, NULL, frame);
903  }
904  break;
906  sc = bridge_channel->tech_pvt;
907  ast_mutex_lock(&sc->lock);
908  ast_bridge_update_talker_src_video_mode(bridge, bridge_channel->chan,
910  frame->subclass.frame_ending);
911  ast_mutex_unlock(&sc->lock);
912  video_src_priority = ast_bridge_is_video_src(bridge, bridge_channel->chan);
913  if (video_src_priority == 1) {
914  int num_src = ast_bridge_number_video_src(bridge);
915  int echo = num_src > 1 ? 0 : 1;
916 
917  ast_bridge_queue_everyone_else(bridge, echo ? NULL : bridge_channel, frame);
918  } else if (video_src_priority == 2) {
919  softmix_pass_video_top_priority(bridge, frame);
920  }
921  break;
923  /* Nothing special to do here, the bridge channel stream map will ensure the
924  * video goes everywhere it needs to
925  */
926  ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
927  break;
928  }
929 }
930 
931 /*!
932  * \internal
933  * \brief Determine what to do with a voice frame.
934  * \since 12.0.0
935  *
936  * \param bridge Which bridge is getting the frame
937  * \param bridge_channel Which channel is writing the frame.
938  * \param frame What is being written.
939  *
940  * \return Nothing
941  */
942 static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
943 {
944  struct softmix_channel *sc = bridge_channel->tech_pvt;
945  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
946  int silent = 0;
947  int totalsilence = 0;
948  int cur_energy = 0;
949  int silence_threshold = bridge_channel->tech_args.silence_threshold ?
950  bridge_channel->tech_args.silence_threshold :
952  /*
953  * If update_talking is set to 0 or 1, tell the bridge that the channel
954  * has started or stopped talking.
955  */
956  char update_talking = -1;
957 
958  /* Write the frame into the conference */
959  ast_mutex_lock(&sc->lock);
960 
962  /*
963  * The incoming frame is not the expected format. Update
964  * the channel's translation path to get us slinear from
965  * the new format for the next frame.
966  *
967  * There is the possibility that this frame is an old slinear
968  * rate frame that was in flight when the softmix bridge
969  * changed rates. If so it will self correct on subsequent
970  * frames.
971  */
972  ast_channel_lock(bridge_channel->chan);
973  ast_debug(1, "Channel %s wrote unexpected format into bridge. Got %s, expected %s.\n",
974  ast_channel_name(bridge_channel->chan),
977  ast_set_read_format_path(bridge_channel->chan, frame->subclass.format,
978  sc->read_slin_format);
979  ast_channel_unlock(bridge_channel->chan);
980  }
981 
982  /* The channel will be leaving soon if there is no dsp. */
983  if (sc->dsp) {
984  silent = ast_dsp_silence_with_energy(sc->dsp, frame, &totalsilence, &cur_energy);
985  }
986 
988  int cur_slot = sc->video_talker.energy_history_cur_slot;
989 
991  sc->video_talker.energy_accum += cur_energy;
992  sc->video_talker.energy_history[cur_slot] = cur_energy;
996  sc->video_talker.energy_history_cur_slot = 0; /* wrap around */
997  }
998  }
999 
1000  if (totalsilence < silence_threshold) {
1001  if (!sc->talking && !silent) {
1002  /* Tell the write process we have audio to be mixed out */
1003  sc->talking = 1;
1004  update_talking = 1;
1005  }
1006  } else {
1007  if (sc->talking) {
1008  sc->talking = 0;
1009  update_talking = 0;
1010  }
1011  }
1012 
1013  /* Before adding audio in, make sure we haven't fallen behind. If audio has fallen
1014  * behind 4 times the amount of samples mixed on every iteration of the mixer, Re-sync
1015  * the audio by flushing the buffer before adding new audio in. */
1016  if (ast_slinfactory_available(&sc->factory) > (4 * SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval))) {
1018  }
1019 
1020  if (sc->talking || !bridge_channel->tech_args.drop_silence) {
1021  /* Add frame to the smoother for mixing with other channels. */
1022  ast_slinfactory_feed(&sc->factory, frame);
1023  }
1024 
1025  /* Alllll done */
1026  ast_mutex_unlock(&sc->lock);
1027 
1028  if (update_talking != -1) {
1029  ast_bridge_channel_notify_talking(bridge_channel, update_talking);
1030  }
1031 }
1032 
1033 /*!
1034  * \internal
1035  * \brief Clear talking flag, stop contributing to mixing and notify handlers.
1036  * \since 13.21.0, 15.4.0
1037  *
1038  * \param bridge_channel Which channel's talking to clear
1039  *
1040  * \return Nothing
1041  */
1042 static void clear_talking(struct ast_bridge_channel *bridge_channel)
1043 {
1044  struct softmix_channel *sc = bridge_channel->tech_pvt;
1045 
1046  if (sc->talking) {
1047  ast_mutex_lock(&sc->lock);
1049  sc->talking = 0;
1050  ast_mutex_unlock(&sc->lock);
1051 
1052  /* Notify that we are no longer talking. */
1053  ast_bridge_channel_notify_talking(bridge_channel, 0);
1054  }
1055 }
1056 
1057 /*!
1058  * \internal
1059  * \brief Check for voice status updates.
1060  * \since 13.20.0
1061  *
1062  * \param bridge Which bridge we are in
1063  * \param bridge_channel Which channel we are checking
1064  *
1065  * \return Nothing
1066  */
1067 static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
1068 {
1069  if (bridge_channel->features->mute) {
1070  /*
1071  * We were muted while we were talking.
1072  *
1073  * Immediately stop contributing to mixing
1074  * and report no longer talking.
1075  */
1076  clear_talking(bridge_channel);
1077  }
1078 }
1079 
1081  const struct ast_stream_topology *source,
1082  const struct ast_stream_topology *original)
1083 {
1084  int i;
1085 
1086  for (i = 0; i < ast_stream_topology_get_count(source); ++i) {
1087  struct ast_stream *stream;
1088  int original_index;
1089 
1090  stream = ast_stream_topology_get_stream(source, i);
1091 
1092  /* Mark the existing stream as removed so we get a new one, this will get
1093  * reused on a subsequent renegotiation.
1094  */
1095  for (original_index = 0; original_index < ast_stream_topology_get_count(original); ++original_index) {
1096  struct ast_stream *original_stream = ast_stream_topology_get_stream(original, original_index);
1097 
1098  if (!strcmp(ast_stream_get_name(stream), ast_stream_get_name(original_stream))) {
1099  struct ast_stream *removed;
1100 
1101  removed = ast_stream_clone(stream, NULL);
1102  if (!removed) {
1103  return -1;
1104  }
1105 
1107 
1108  /* The destination topology can only ever contain the same, or more,
1109  * streams than the original so this is safe.
1110  */
1111  if (ast_stream_topology_set_stream(dest, original_index, removed)) {
1112  ast_stream_free(removed);
1113  return -1;
1114  }
1115 
1116  break;
1117  }
1118  }
1119  }
1120 
1121  return 0;
1122 }
1123 
1124 static void sfu_topologies_on_source_change(struct ast_bridge *bridge,
1125  struct ast_bridge_channel *source)
1126 {
1127  struct ast_stream_topology *source_video = NULL;
1128  struct ast_bridge_channels_list *participants = &bridge->channels;
1129  struct ast_bridge_channel *participant;
1130  int res;
1131 
1132  source_video = ast_stream_topology_alloc();
1133  if (!source_video) {
1134  return;
1135  }
1136 
1137  ast_channel_lock(source->chan);
1138  res = append_source_streams(source_video, ast_channel_name(source->chan),
1139  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(source->chan) : NULL,
1141  ast_channel_unlock(source->chan);
1142  if (res) {
1143  goto cleanup;
1144  }
1145 
1146  AST_LIST_TRAVERSE(participants, participant, entry) {
1147  struct ast_stream_topology *original_topology;
1148  struct softmix_channel *sc;
1149 
1150  if (participant == source) {
1151  continue;
1152  }
1153 
1154  sc = participant->tech_pvt;
1155 
1156  original_topology = ast_stream_topology_clone(sc->topology);
1157  if (!original_topology) {
1158  goto cleanup;
1159  }
1160 
1161  /* We add all the source streams back in, if any removed streams are already present they will
1162  * get used first followed by appending new ones.
1163  */
1164  if (append_all_streams(sc->topology, source_video)) {
1165  ast_stream_topology_free(original_topology);
1166  goto cleanup;
1167  }
1168 
1169  /* And the original existing streams get marked as removed. This causes the remote side to see
1170  * a new stream for the source streams.
1171  */
1172  if (remove_all_original_streams(sc->topology, source_video, original_topology)) {
1173  ast_stream_topology_free(original_topology);
1174  goto cleanup;
1175  }
1176 
1178  ast_stream_topology_free(original_topology);
1179  }
1180 
1181 cleanup:
1182  ast_stream_topology_free(source_video);
1183 }
1184 
1185 /*!
1186  * \internal
1187  * \brief Determine what to do with a text frame.
1188  * \since 13.22.0
1189  * \since 15.5.0
1190  *
1191  * \param bridge Which bridge is getting the frame
1192  * \param bridge_channel Which channel is writing the frame.
1193  * \param frame What is being written.
1194  *
1195  * \return Nothing
1196  */
1197 static void softmix_bridge_write_text(struct ast_bridge *bridge,
1198  struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1199 {
1200  if (DEBUG_ATLEAST(1)) {
1201  struct ast_msg_data *msg = frame->data.ptr;
1202  char frame_type[64];
1203 
1204  ast_frame_type2str(frame->frametype, frame_type, sizeof(frame_type));
1205 
1206  if (frame->frametype == AST_FRAME_TEXT_DATA) {
1207  ast_log(LOG_DEBUG, "Received %s frame from '%s:%s': %s\n", frame_type,
1209  ast_channel_name(bridge_channel->chan),
1211  } else {
1212  ast_log(LOG_DEBUG, "Received %s frame from '%s': %.*s\n", frame_type,
1213  ast_channel_name(bridge_channel->chan), frame->datalen,
1214  (char *)frame->data.ptr);
1215  }
1216  }
1217 
1218  ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1219 }
1220 
1221 /*!
1222  * \internal
1223  * \brief Determine what to do with a control frame.
1224  * \since 12.0.0
1225  *
1226  * \param bridge Which bridge is getting the frame
1227  * \param bridge_channel Which channel is writing the frame.
1228  * \param frame What is being written.
1229  *
1230  * \retval 0 Frame accepted into the bridge.
1231  * \retval -1 Frame needs to be deferred.
1232  */
1233 static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1234 {
1235  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1236 
1237  /*
1238  * XXX Softmix needs to use channel roles to determine what to
1239  * do with control frames.
1240  */
1241 
1242  switch (frame->subclass.integer) {
1243  case AST_CONTROL_HOLD:
1244  /*
1245  * Doing anything for holds in a conference bridge could be considered a bit
1246  * odd. That being said, in most cases one would probably want the talking
1247  * flag cleared when 'hold' is pressed by the remote endpoint, so go ahead
1248  * and do that here. However, that is all we'll do. Meaning if for some reason
1249  * the endpoint continues to send audio frames despite pressing 'hold' talking
1250  * will once again be detected for that channel.
1251  */
1252  clear_talking(bridge_channel);
1253  break;
1254  case AST_CONTROL_VIDUPDATE:
1255  if (!bridge->softmix.video_mode.video_update_discard ||
1257  ast_bridge_queue_everyone_else(bridge, NULL, frame);
1258  softmix_data->last_video_update = ast_tvnow();
1259  }
1260  break;
1263  sfu_topologies_on_source_change(bridge, bridge_channel);
1264  }
1265  break;
1266  default:
1267  break;
1268  }
1269 
1270  return 0;
1271 }
1272 
1273 /*!
1274  * \internal
1275  * \brief Determine what to do with an RTCP frame.
1276  * \since 15.4.0
1277  *
1278  * \param bridge Which bridge is getting the frame
1279  * \param bridge_channel Which channel is writing the frame.
1280  * \param frame What is being written.
1281  */
1282 static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1283 {
1284  struct ast_rtp_rtcp_feedback *feedback = frame->data.ptr;
1285  struct softmix_channel *sc = bridge_channel->tech_pvt;
1286 
1287  /* We only care about REMB reports right now. In the future we may be able to use sender or
1288  * receiver reports to further tweak things, but not yet.
1289  */
1290  if (frame->subclass.integer != AST_RTP_RTCP_PSFB || feedback->fmt != AST_RTP_RTCP_FMT_REMB ||
1293  return;
1294  }
1295 
1296  /* REMB is the total estimated maximum bitrate across all streams within the session, so we store
1297  * only the latest report and use it everywhere.
1298  */
1299  ast_mutex_lock(&sc->lock);
1300  sc->remb = feedback->remb;
1301  ast_mutex_unlock(&sc->lock);
1302 
1303  return;
1304 }
1305 
1306 /*!
1307  * \internal
1308  * \brief Determine what to do with a frame written into the bridge.
1309  * \since 12.0.0
1310  *
1311  * \param bridge Which bridge is getting the frame
1312  * \param bridge_channel Which channel is writing the frame.
1313  * \param frame What is being written.
1314  *
1315  * \retval 0 Frame accepted into the bridge.
1316  * \retval -1 Frame needs to be deferred.
1317  *
1318  * \note On entry, bridge is already locked.
1319  */
1320 static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
1321 {
1322  int res = 0;
1323 
1324  if (!bridge->tech_pvt || !bridge_channel || !bridge_channel->tech_pvt) {
1325  /* "Accept" the frame and discard it. */
1326  return 0;
1327  }
1328 
1329  /*
1330  * XXX Softmix needs to use channel roles to determine who gets
1331  * what frame. Possible roles: announcer, recorder, agent,
1332  * supervisor.
1333  */
1334  switch (frame->frametype) {
1335  case AST_FRAME_NULL:
1336  /* "Accept" the frame and discard it. */
1337  softmix_bridge_check_voice(bridge, bridge_channel);
1338  break;
1339  case AST_FRAME_DTMF_BEGIN:
1340  case AST_FRAME_DTMF_END:
1341  res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1342  break;
1343  case AST_FRAME_VOICE:
1344  softmix_bridge_write_voice(bridge, bridge_channel, frame);
1345  break;
1346  case AST_FRAME_VIDEO:
1347  softmix_bridge_write_video(bridge, bridge_channel, frame);
1348  break;
1349  case AST_FRAME_TEXT:
1350  case AST_FRAME_TEXT_DATA:
1351  softmix_bridge_write_text(bridge, bridge_channel, frame);
1352  break;
1353  case AST_FRAME_CONTROL:
1354  res = softmix_bridge_write_control(bridge, bridge_channel, frame);
1355  break;
1356  case AST_FRAME_RTCP:
1357  softmix_bridge_write_rtcp(bridge, bridge_channel, frame);
1358  break;
1360  res = ast_bridge_queue_everyone_else(bridge, bridge_channel, frame);
1361  break;
1363  ast_log(LOG_ERROR, "Synchronous bridge action written to a softmix bridge.\n");
1364  ast_assert(0);
1365  default:
1366  ast_debug(3, "Frame type %u unsupported\n", frame->frametype);
1367  /* "Accept" the frame and discard it. */
1368  break;
1369  }
1370 
1371  return res;
1372 }
1373 
1374 static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data,
1375  float bitrate)
1376 {
1377  if (!softmix_data->bitrate) {
1378  softmix_data->bitrate = bitrate;
1379  return;
1380  }
1381 
1384  softmix_data->bitrate = (softmix_data->bitrate + bitrate) / 2;
1385  break;
1387  if (bitrate < softmix_data->bitrate) {
1388  softmix_data->bitrate = bitrate;
1389  }
1390  break;
1392  if (bitrate > softmix_data->bitrate) {
1393  softmix_data->bitrate = bitrate;
1394  }
1395  break;
1400  /* These will never actually get hit due to being handled by remb_collect_report below */
1401  break;
1402  }
1403 }
1404 
1405 static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
1406  struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
1407 {
1408  int i;
1409  float bitrate;
1410 
1411  /* If there are no video sources that we are a receiver of then we have noone to
1412  * report REMB to.
1413  */
1414  if (!AST_VECTOR_SIZE(&sc->video_sources)) {
1415  return;
1416  }
1417 
1418  /* We evenly divide the available maximum bitrate across the video sources
1419  * to this receiver so each source gets an equal slice.
1420  */
1421 
1424  return;
1425  }
1426 
1427  bitrate = (sc->remb.br_mantissa << sc->remb.br_exp) / AST_VECTOR_SIZE(&sc->video_sources);
1428 
1429  /* If this receiver has no bitrate yet ignore it */
1430  if (!bitrate) {
1431  return;
1432  }
1433 
1434  /* If we are using the "all" variants then we should use the bridge bitrate to store information */
1438  remb_collect_report_all(bridge, softmix_data, bitrate);
1439  return;
1440  }
1441 
1442  for (i = 0; i < AST_VECTOR_SIZE(&sc->video_sources); ++i) {
1443  struct softmix_remb_collector *collector;
1444 
1445  /* The collector will always exist if a video source is in our list */
1446  collector = AST_VECTOR_GET(&softmix_data->remb_collectors, AST_VECTOR_GET(&sc->video_sources, i));
1447 
1448  if (!collector->bitrate) {
1449  collector->bitrate = bitrate;
1450  continue;
1451  }
1452 
1455  collector->bitrate = (collector->bitrate + bitrate) / 2;
1456  break;
1458  if (bitrate < collector->bitrate) {
1459  collector->bitrate = bitrate;
1460  }
1461  break;
1463  if (bitrate > collector->bitrate) {
1464  collector->bitrate = bitrate;
1465  }
1466  break;
1470  /* These will never actually get hit due to being handled by remb_collect_report_all above */
1471  break;
1473  /* Don't do anything, we've already forced it */
1474  break;
1475  }
1476  }
1477 
1478  /* After the report is integrated we reset this to 0 in case they stop producing
1479  * REMB reports.
1480  */
1481  sc->remb.br_mantissa = 0;
1482  sc->remb.br_exp = 0;
1483 }
1484 
1485 static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data,
1486  struct softmix_channel *sc)
1487 {
1488  float bitrate = softmix_data->bitrate;
1489  int i;
1490  int exp;
1491 
1492  if (!sc->remb_collector) {
1493  return;
1494  }
1495 
1496  /* If there is no bridge level bitrate fall back to collector level */
1497  if (!bitrate) {
1498  bitrate = sc->remb_collector->bitrate;
1499  sc->remb_collector->bitrate = 0;
1500  }
1501 
1502  /* We always do this calculation as even when the bitrate is zero the browser
1503  * still prefers it to be accurate instead of lying.
1504  *
1505  * The mantissa only has 18 bits available, so make sure it fits. Adjust the
1506  * value and exponent for those values that don't.
1507  *
1508  * For example given the following:
1509  *
1510  * bitrate = 123456789.0
1511  * frexp(bitrate, &exp);
1512  *
1513  * 'exp' should now equal 27 (number of bits needed to represent the value). Since
1514  * the mantissa must fit into an 18-bit unsigned integer, and the given bitrate is
1515  * too large to fit, we must subtract 18 from the exponent in order to get the
1516  * number of times the bitrate will fit into that size integer.
1517  *
1518  * exp -= 18;
1519  *
1520  * 'exp' is now equal to 9. Now we can get the mantissa that fits into an 18-bit
1521  * unsigned integer by dividing the bitrate by 2^exp:
1522  *
1523  * mantissa = 123456789.0 / 2^9
1524  *
1525  * This makes the final mantissa equal to 241126 (implicitly cast), which is less
1526  * than 262143 (the max value that can be put into an unsigned 18-bit integer).
1527  * So now we have the following:
1528  *
1529  * exp = 9;
1530  * mantissa = 241126;
1531  *
1532  * If we multiply that back we should come up with something close to the original
1533  * bit rate:
1534  *
1535  * 241126 * 2^9 = 123456512
1536  *
1537  * Precision is lost due to the nature of floating point values. Easier to why from
1538  * the binary:
1539  *
1540  * 241126 * 2^9 = 241126 << 9 = 111010110111100110 << 9 = 111010110111100110000000000
1541  *
1542  * Precision on the "lower" end is lost due to zeros being shifted in. This loss is
1543  * both expected and acceptable.
1544  */
1545  frexp(bitrate, &exp);
1546  exp = exp > 18 ? exp - 18 : 0;
1547 
1548  sc->remb_collector->feedback.remb.br_mantissa = bitrate / (1 << exp);
1549  sc->remb_collector->feedback.remb.br_exp = exp;
1550 
1551  for (i = 0; i < AST_VECTOR_SIZE(&bridge_channel->stream_map.to_bridge); ++i) {
1552  int bridge_num = AST_VECTOR_GET(&bridge_channel->stream_map.to_bridge, i);
1553 
1554  /* If this stream is not being provided to the bridge there can be no receivers of it
1555  * so therefore no REMB reports.
1556  */
1557  if (bridge_num == -1) {
1558  continue;
1559  }
1560 
1561  /* We need to update the frame with this stream, or else it won't be
1562  * properly routed. We don't use the actual channel stream identifier as
1563  * the bridging core will do the translation from bridge stream identifier to
1564  * channel stream identifier.
1565  */
1566  sc->remb_collector->frame.stream_num = bridge_num;
1567  ast_bridge_channel_queue_frame(bridge_channel, &sc->remb_collector->frame);
1568  }
1569 }
1570 
1571 static void gather_softmix_stats(struct softmix_stats *stats,
1572  const struct softmix_bridge_data *softmix_data,
1573  struct ast_bridge_channel *bridge_channel)
1574 {
1575  int channel_native_rate;
1576 
1577  /* Gather stats about channel sample rates. */
1578  ast_channel_lock(bridge_channel->chan);
1579  channel_native_rate = MAX(SOFTMIX_MIN_SAMPLE_RATE,
1581  ast_channel_unlock(bridge_channel->chan);
1582 
1583  if (stats->highest_supported_rate < channel_native_rate) {
1584  stats->highest_supported_rate = channel_native_rate;
1585  }
1586  if (stats->maximum_rate && stats->maximum_rate < channel_native_rate) {
1587  stats->num_above_maximum_rate++;
1588  } else if (softmix_data->internal_rate < channel_native_rate) {
1589  int i;
1590 
1591  for (i = 0; i < ARRAY_LEN(stats->sample_rates); i++) {
1592  if (stats->sample_rates[i] == channel_native_rate) {
1593  stats->num_channels[i]++;
1594  break;
1595  } else if (!stats->sample_rates[i]) {
1596  stats->sample_rates[i] = channel_native_rate;
1597  stats->num_channels[i]++;
1598  break;
1599  }
1600  }
1601  stats->num_above_internal_rate++;
1602  } else if (softmix_data->internal_rate == channel_native_rate) {
1603  stats->num_at_internal_rate++;
1604  }
1605 }
1606 
1607 /*!
1608  * \internal
1609  * \brief Analyse mixing statistics and change bridges internal rate
1610  * if necessary.
1611  *
1612  * \retval 0, no changes to internal rate
1613  * \retval 1, internal rate was changed, update all the channels on the next mixing iteration.
1614  */
1615 static unsigned int analyse_softmix_stats(struct softmix_stats *stats,
1616  struct softmix_bridge_data *softmix_data, int binaural_active)
1617 {
1618  int i;
1619 
1620  if (binaural_active) {
1622  }
1623 
1624  /*
1625  * Re-adjust the internal bridge sample rate if
1626  * 1. The bridge's internal sample rate is locked in at a sample
1627  * rate other than the current sample rate being used.
1628  * 2. two or more channels support a higher sample rate
1629  * 3. no channels support the current sample rate or a higher rate
1630  */
1631  if (stats->locked_rate) {
1632  /* if the rate is locked by the bridge, only update it if it differs
1633  * from the current rate we are using. */
1634  if (softmix_data->internal_rate != stats->locked_rate) {
1635  ast_debug(1, "Locking at new rate. Bridge changed from %u to %u.\n",
1636  softmix_data->internal_rate, stats->locked_rate);
1637  softmix_data->internal_rate = stats->locked_rate;
1638  return 1;
1639  }
1640  } else if (stats->num_above_maximum_rate) {
1641  /* if the bridge has a maximum rate set and channels are above it only
1642  * update if it differs from the current rate we are using. */
1643  if (softmix_data->internal_rate != stats->maximum_rate) {
1644  ast_debug(1, "Locking at new maximum rate. Bridge changed from %u to %u.\n",
1645  softmix_data->internal_rate, stats->maximum_rate);
1646  softmix_data->internal_rate = stats->maximum_rate;
1647  return 1;
1648  }
1649  } else if (stats->num_above_internal_rate >= 2) {
1650  /* the highest rate is just used as a starting point */
1651  unsigned int best_rate = stats->highest_supported_rate;
1652  int best_index = -1;
1653 
1654  for (i = 0; i < ARRAY_LEN(stats->num_channels); i++) {
1655  if (stats->num_channels[i]) {
1656  break;
1657  }
1658  if (2 <= stats->num_channels[i]) {
1659  /* Two or more channels support this rate. */
1660  if (best_index == -1
1661  || stats->sample_rates[best_index] < stats->sample_rates[i]) {
1662  /*
1663  * best_rate starts out being the first sample rate
1664  * greater than the internal sample rate that two or
1665  * more channels support.
1666  *
1667  * or
1668  *
1669  * There are multiple rates above the internal rate
1670  * and this rate is higher than the previous rate two
1671  * or more channels support.
1672  */
1673  best_rate = stats->sample_rates[i];
1674  best_index = i;
1675  }
1676  } else if (best_index == -1) {
1677  /*
1678  * It is possible that multiple channels exist with native sample
1679  * rates above the internal sample rate, but none of those channels
1680  * have the same rate in common. In this case, the lowest sample
1681  * rate among those channels is picked. Over time as additional
1682  * statistic runs are made the internal sample rate number will
1683  * adjust to the most optimal sample rate, but it may take multiple
1684  * iterations.
1685  */
1686  best_rate = MIN(best_rate, stats->sample_rates[i]);
1687  }
1688  }
1689 
1690  ast_debug(1, "Multiple above internal rate. Bridge changed from %u to %u.\n",
1691  softmix_data->internal_rate, best_rate);
1692  softmix_data->internal_rate = best_rate;
1693  return 1;
1694  } else if (!stats->num_at_internal_rate && !stats->num_above_internal_rate) {
1695  /* In this case, the highest supported rate is actually lower than the internal rate */
1696  ast_debug(1, "All below internal rate. Bridge changed from %u to %u.\n",
1697  softmix_data->internal_rate, stats->highest_supported_rate);
1698  softmix_data->internal_rate = stats->highest_supported_rate;
1699  return 1;
1700  }
1701  return 0;
1702 }
1703 
1704 static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array,
1705  unsigned int starting_num_entries, unsigned int binaural_active)
1706 {
1707  memset(mixing_array, 0, sizeof(*mixing_array));
1708  mixing_array->max_num_entries = starting_num_entries;
1709  if (!(mixing_array->buffers = ast_calloc(mixing_array->max_num_entries, sizeof(int16_t *)))) {
1710  ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1711  return -1;
1712  }
1713  if (binaural_active) {
1714  if (!(mixing_array->chan_pairs = ast_calloc(mixing_array->max_num_entries,
1715  sizeof(struct convolve_channel_pair *)))) {
1716  ast_log(LOG_NOTICE, "Failed to allocate softmix mixing structure.\n");
1717  return -1;
1718  }
1719  }
1720  return 0;
1721 }
1722 
1723 static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array,
1724  unsigned int binaural_active)
1725 {
1726  ast_free(mixing_array->buffers);
1727  if (binaural_active) {
1728  ast_free(mixing_array->chan_pairs);
1729  }
1730 }
1731 
1732 static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array,
1733  unsigned int num_entries, unsigned int binaural_active)
1734 {
1735  int16_t **tmp;
1736 
1737  /* give it some room to grow since memory is cheap but allocations can be expensive */
1738  mixing_array->max_num_entries = num_entries;
1739  if (!(tmp = ast_realloc(mixing_array->buffers, (mixing_array->max_num_entries * sizeof(int16_t *))))) {
1740  ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1741  return -1;
1742  }
1743  mixing_array->buffers = tmp;
1744 
1745  if (binaural_active) {
1746  struct convolve_channel_pair **tmp2;
1747  if (!(tmp2 = ast_realloc(mixing_array->chan_pairs,
1748  (mixing_array->max_num_entries * sizeof(struct convolve_channel_pair *))))) {
1749  ast_log(LOG_NOTICE, "Failed to re-allocate softmix mixing structure.\n");
1750  return -1;
1751  }
1752  mixing_array->chan_pairs = tmp2;
1753  }
1754  return 0;
1755 }
1756 
1757 /*!
1758  * \brief Mixing loop.
1759  *
1760  * \retval 0 on success
1761  * \retval -1 on failure
1762  */
1763 static int softmix_mixing_loop(struct ast_bridge *bridge)
1764 {
1765  struct softmix_stats stats = { { 0 }, };
1766  struct softmix_mixing_array mixing_array;
1767  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
1768  struct ast_timer *timer;
1769  struct softmix_translate_helper trans_helper;
1771 #ifdef BINAURAL_RENDERING
1772  int16_t bin_buf[MAX_DATALEN];
1773  int16_t ann_buf[MAX_DATALEN];
1774 #endif
1775  unsigned int stat_iteration_counter = 0; /* counts down, gather stats at zero and reset. */
1776  int timingfd;
1777  int update_all_rates = 0; /* set this when the internal sample rate has changed */
1778  unsigned int idx;
1779  unsigned int x;
1780  int res = -1;
1781 
1782  timer = softmix_data->timer;
1783  timingfd = ast_timer_fd(timer);
1784  softmix_translate_helper_init(&trans_helper, softmix_data->internal_rate);
1785  ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1786 
1787  /* Give the mixing array room to grow, memory is cheap but allocations are expensive. */
1788  if (softmix_mixing_array_init(&mixing_array, bridge->num_channels + 10,
1789  bridge->softmix.binaural_active)) {
1790  goto softmix_cleanup;
1791  }
1792 
1793  /*
1794  * XXX Softmix needs to use channel roles to determine who gets
1795  * what audio mixed.
1796  */
1797  while (!softmix_data->stop && bridge->num_active) {
1798  struct ast_bridge_channel *bridge_channel;
1799  int timeout = -1;
1800  struct ast_format *cur_slin = ast_format_cache_get_slin_by_rate(softmix_data->internal_rate);
1801  unsigned int softmix_samples = SOFTMIX_SAMPLES(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1802  unsigned int softmix_datalen = SOFTMIX_DATALEN(softmix_data->internal_rate, softmix_data->internal_mixing_interval);
1803  int remb_update = 0;
1804 
1805  if (softmix_datalen > MAX_DATALEN) {
1806  /* This should NEVER happen, but if it does we need to know about it. Almost
1807  * all the memcpys used during this process depend on this assumption. Rather
1808  * than checking this over and over again through out the code, this single
1809  * verification is done on each iteration. */
1811  "Bridge %s: Conference mixing error, requested mixing length greater than mixing buffer.\n",
1812  bridge->uniqueid);
1813  goto softmix_cleanup;
1814  }
1815 
1816  /* Grow the mixing array buffer as participants are added. */
1817  if (mixing_array.max_num_entries < bridge->num_channels
1818  && softmix_mixing_array_grow(&mixing_array, bridge->num_channels + 5,
1819  bridge->softmix.binaural_active)) {
1820  goto softmix_cleanup;
1821  }
1822 
1823  /* init the number of buffers stored in the mixing array to 0.
1824  * As buffers are added for mixing, this number is incremented. */
1825  mixing_array.used_entries = 0;
1826 
1827  /* These variables help determine if a rate change is required */
1828  if (!stat_iteration_counter) {
1829  memset(&stats, 0, sizeof(stats));
1830  stats.locked_rate = bridge->softmix.internal_sample_rate;
1831  stats.maximum_rate = bridge->softmix.maximum_sample_rate;
1832  }
1833 
1834  /* If the sample rate has changed, update the translator helper */
1835  if (update_all_rates) {
1836  softmix_translate_helper_change_rate(&trans_helper, softmix_data->internal_rate);
1837  }
1838 
1839 #ifdef BINAURAL_RENDERING
1840  check_binaural_position_change(bridge, softmix_data);
1841 #endif
1842 
1843  /* If we need to do a REMB update to all video sources then do so */
1847  remb_update = 1;
1848  softmix_data->last_remb_update = ast_tvnow();
1849  }
1850 
1851  /* Go through pulling audio from each factory that has it available */
1852  AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1853  struct softmix_channel *sc = bridge_channel->tech_pvt;
1854 
1855  if (!sc) {
1856  /* This channel failed to join successfully. */
1857  continue;
1858  }
1859 
1860  /* Update the sample rate to match the bridge's native sample rate if necessary. */
1861  if (update_all_rates) {
1862  set_softmix_bridge_data(softmix_data->internal_rate,
1863  softmix_data->internal_mixing_interval, bridge_channel, 1, -1, -1, -1);
1864  }
1865 
1866  /* If stat_iteration_counter is 0, then collect statistics during this mixing interation */
1867  if (!stat_iteration_counter) {
1868  gather_softmix_stats(&stats, softmix_data, bridge_channel);
1869  }
1870 
1871  /* if the channel is suspended, don't check for audio, but still gather stats */
1872  if (bridge_channel->suspended) {
1873  continue;
1874  }
1875 
1876  /* Try to get audio from the factory if available */
1877  ast_mutex_lock(&sc->lock);
1878  if ((mixing_array.buffers[mixing_array.used_entries] = softmix_process_read_audio(sc, softmix_samples))) {
1879 #ifdef BINAURAL_RENDERING
1880  add_binaural_mixing(bridge, softmix_data, softmix_samples, &mixing_array, sc,
1881  ast_channel_name(bridge_channel->chan));
1882 #endif
1883  mixing_array.used_entries++;
1884  }
1885  if (remb_update) {
1886  remb_collect_report(bridge, bridge_channel, softmix_data, sc);
1887  }
1888  ast_mutex_unlock(&sc->lock);
1889  }
1890 
1891  /* mix it like crazy (non binaural channels)*/
1892  memset(buf, 0, softmix_datalen);
1893  for (idx = 0; idx < mixing_array.used_entries; ++idx) {
1894  for (x = 0; x < softmix_samples; ++x) {
1895  ast_slinear_saturated_add(buf + x, mixing_array.buffers[idx] + x);
1896  }
1897  }
1898 
1899 #ifdef BINAURAL_RENDERING
1900  binaural_mixing(bridge, softmix_data, &mixing_array, bin_buf, ann_buf);
1901 #endif
1902 
1903  /* Next step go through removing the channel's own audio and creating a good frame... */
1904  AST_LIST_TRAVERSE(&bridge->channels, bridge_channel, entry) {
1905  struct softmix_channel *sc = bridge_channel->tech_pvt;
1906 
1907  if (!sc || bridge_channel->suspended) {
1908  /* This channel failed to join successfully or is suspended. */
1909  continue;
1910  }
1911 
1912  ast_mutex_lock(&sc->lock);
1913 
1914  /* Make SLINEAR write frame from local buffer */
1915  ao2_t_replace(sc->write_frame.subclass.format, cur_slin,
1916  "Replace softmix channel slin format");
1917 #ifdef BINAURAL_RENDERING
1918  if (bridge->softmix.binaural_active && softmix_data->convolve.binaural_active
1919  && sc->binaural) {
1920  create_binaural_frame(bridge_channel, sc, bin_buf, ann_buf, softmix_datalen,
1921  softmix_samples, buf);
1922  } else
1923 #endif
1924  {
1925  sc->write_frame.datalen = softmix_datalen;
1926  sc->write_frame.samples = softmix_samples;
1927  memcpy(sc->final_buf, buf, softmix_datalen);
1928  }
1929  /* process the softmix channel's new write audio */
1930  softmix_process_write_audio(&trans_helper,
1931  ast_channel_rawwriteformat(bridge_channel->chan), sc,
1932  softmix_data->default_sample_size);
1933 
1934  ast_mutex_unlock(&sc->lock);
1935 
1936  /* A frame is now ready for the channel. */
1937  ast_bridge_channel_queue_frame(bridge_channel, &sc->write_frame);
1938 
1939  if (remb_update) {
1940  remb_send_report(bridge_channel, softmix_data, sc);
1941  }
1942  }
1943 
1944  if (remb_update) {
1945  /* In case we are doing bridge level REMB reset the bitrate so we start fresh */
1946  softmix_data->bitrate = 0;
1947  }
1948 
1949  update_all_rates = 0;
1950  if (!stat_iteration_counter) {
1951  update_all_rates = analyse_softmix_stats(&stats, softmix_data,
1952  bridge->softmix.binaural_active);
1953  stat_iteration_counter = SOFTMIX_STAT_INTERVAL;
1954  }
1955  stat_iteration_counter--;
1956 
1957  ast_bridge_unlock(bridge);
1958  /* cleanup any translation frame data from the previous mixing iteration. */
1959  softmix_translate_helper_cleanup(&trans_helper);
1960  /* Wait for the timing source to tell us to wake up and get things done */
1961  ast_waitfor_n_fd(&timingfd, 1, &timeout, NULL);
1962  if (ast_timer_ack(timer, 1) < 0) {
1963  ast_log(LOG_ERROR, "Bridge %s: Failed to acknowledge timer in softmix.\n",
1964  bridge->uniqueid);
1965  ast_bridge_lock(bridge);
1966  goto softmix_cleanup;
1967  }
1968  ast_bridge_lock(bridge);
1969 
1970  /* make sure to detect mixing interval changes if they occur. */
1971  if (bridge->softmix.internal_mixing_interval
1972  && (bridge->softmix.internal_mixing_interval != softmix_data->internal_mixing_interval)) {
1974  ast_timer_set_rate(timer, (1000 / softmix_data->internal_mixing_interval));
1975  update_all_rates = 1; /* if the interval changes, the rates must be adjusted as well just to be notified new interval.*/
1976  }
1977  }
1978 
1979  res = 0;
1980 
1981 softmix_cleanup:
1982  softmix_translate_helper_destroy(&trans_helper);
1983  softmix_mixing_array_destroy(&mixing_array, bridge->softmix.binaural_active);
1984  return res;
1985 }
1986 
1987 /*!
1988  * \internal
1989  * \brief Mixing thread.
1990  * \since 12.0.0
1991  *
1992  * \note The thread does not have its own reference to the
1993  * bridge. The lifetime of the thread is tied to the lifetime
1994  * of the mixing technology association with the bridge.
1995  */
1996 static void *softmix_mixing_thread(void *data)
1997 {
1998  struct softmix_bridge_data *softmix_data = data;
1999  struct ast_bridge *bridge = softmix_data->bridge;
2000 
2001  ast_bridge_lock(bridge);
2002  if (bridge->callid) {
2004  }
2005 
2006  ast_debug(1, "Bridge %s: starting mixing thread\n", bridge->uniqueid);
2007 
2008  while (!softmix_data->stop) {
2009  if (!bridge->num_active) {
2010  /* Wait for something to happen to the bridge. */
2011  ast_bridge_unlock(bridge);
2012  ast_mutex_lock(&softmix_data->lock);
2013  if (!softmix_data->stop) {
2014  ast_cond_wait(&softmix_data->cond, &softmix_data->lock);
2015  }
2016  ast_mutex_unlock(&softmix_data->lock);
2017  ast_bridge_lock(bridge);
2018  continue;
2019  }
2020 
2021  if (bridge->softmix.binaural_active && !softmix_data->binaural_init) {
2022 #ifndef BINAURAL_RENDERING
2023  ast_bridge_lock(bridge);
2024  bridge->softmix.binaural_active = 0;
2025  ast_bridge_unlock(bridge);
2026  ast_log(LOG_WARNING, "Bridge: %s: Binaural rendering active by config but not "
2027  "compiled.\n", bridge->uniqueid);
2028 #else
2029  /* Set and init binaural data if binaural is activated in the configuration. */
2031  softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2032  softmix_data->internal_mixing_interval);
2033  /* If init for binaural processing fails we will fall back to mono audio processing. */
2034  if (init_convolve_data(&softmix_data->convolve, softmix_data->default_sample_size)
2035  == -1) {
2036  ast_bridge_lock(bridge);
2037  bridge->softmix.binaural_active = 0;
2038  ast_bridge_unlock(bridge);
2039  ast_log(LOG_ERROR, "Bridge: %s: Unable to allocate memory for "
2040  "binaural processing, Will only process mono audio.\n",
2041  bridge->uniqueid);
2042  }
2043  softmix_data->binaural_init = 1;
2044 #endif
2045  }
2046 
2047  if (softmix_mixing_loop(bridge)) {
2048  /*
2049  * A mixing error occurred. Sleep and try again later so we
2050  * won't flood the logs.
2051  */
2052  ast_bridge_unlock(bridge);
2053  sleep(1);
2054  ast_bridge_lock(bridge);
2055  }
2056  }
2057 
2058  ast_bridge_unlock(bridge);
2059 
2060  ast_debug(1, "Bridge %s: stopping mixing thread\n", bridge->uniqueid);
2061 
2062  return NULL;
2063 }
2064 
2065 static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
2066 {
2067  if (softmix_data->timer) {
2068  ast_timer_close(softmix_data->timer);
2069  softmix_data->timer = NULL;
2070  }
2071  ast_mutex_destroy(&softmix_data->lock);
2072  ast_cond_destroy(&softmix_data->cond);
2073  AST_VECTOR_RESET(&softmix_data->remb_collectors, ao2_cleanup);
2074  AST_VECTOR_FREE(&softmix_data->remb_collectors);
2075  ast_free(softmix_data);
2076 }
2077 
2078 /*! \brief Function called when a bridge is created */
2079 static int softmix_bridge_create(struct ast_bridge *bridge)
2080 {
2081  struct softmix_bridge_data *softmix_data;
2082 
2083  softmix_data = ast_calloc(1, sizeof(*softmix_data));
2084  if (!softmix_data) {
2085  return -1;
2086  }
2087  softmix_data->bridge = bridge;
2088  ast_mutex_init(&softmix_data->lock);
2089  ast_cond_init(&softmix_data->cond, NULL);
2090  softmix_data->timer = ast_timer_open();
2091  if (!softmix_data->timer) {
2092  ast_log(AST_LOG_WARNING, "Failed to open timer for softmix bridge\n");
2093  softmix_bridge_data_destroy(softmix_data);
2094  return -1;
2095  }
2096  /* start at minimum rate, let it grow from there */
2097  softmix_data->internal_rate = SOFTMIX_MIN_SAMPLE_RATE;
2099 
2100 #ifdef BINAURAL_RENDERING
2101  softmix_data->default_sample_size = SOFTMIX_SAMPLES(softmix_data->internal_rate,
2102  softmix_data->internal_mixing_interval);
2103 #endif
2104 
2105  AST_VECTOR_INIT(&softmix_data->remb_collectors, 0);
2106 
2107  bridge->tech_pvt = softmix_data;
2108 
2109  /* Start the mixing thread. */
2110  if (ast_pthread_create(&softmix_data->thread, NULL, softmix_mixing_thread,
2111  softmix_data)) {
2112  softmix_data->thread = AST_PTHREADT_NULL;
2113  softmix_bridge_data_destroy(softmix_data);
2114  bridge->tech_pvt = NULL;
2115  return -1;
2116  }
2117 
2118  return 0;
2119 }
2120 
2121 /*!
2122  * \internal
2123  * \brief Request the softmix mixing thread stop.
2124  * \since 12.0.0
2125  *
2126  * \param bridge Which bridge is being stopped.
2127  *
2128  * \return Nothing
2129  */
2131 {
2132  struct softmix_bridge_data *softmix_data;
2133 
2134  softmix_data = bridge->tech_pvt;
2135  if (!softmix_data) {
2136  return;
2137  }
2138 
2139  ast_mutex_lock(&softmix_data->lock);
2140  softmix_data->stop = 1;
2141  ast_mutex_unlock(&softmix_data->lock);
2142 }
2143 
2144 /*! \brief Function called when a bridge is destroyed */
2146 {
2147  struct softmix_bridge_data *softmix_data;
2148  pthread_t thread;
2149 
2150  softmix_data = bridge->tech_pvt;
2151  if (!softmix_data) {
2152  return;
2153  }
2154 
2155  /* Stop the mixing thread. */
2156  ast_mutex_lock(&softmix_data->lock);
2157  softmix_data->stop = 1;
2158  ast_cond_signal(&softmix_data->cond);
2159  thread = softmix_data->thread;
2160  softmix_data->thread = AST_PTHREADT_NULL;
2161  ast_mutex_unlock(&softmix_data->lock);
2162  if (thread != AST_PTHREADT_NULL) {
2163  ast_debug(1, "Bridge %s: Waiting for mixing thread to die.\n", bridge->uniqueid);
2164  pthread_join(thread, NULL);
2165  }
2166 #ifdef BINAURAL_RENDERING
2167  free_convolve_data(&softmix_data->convolve);
2168 #endif
2169  softmix_bridge_data_destroy(softmix_data);
2170  bridge->tech_pvt = NULL;
2171 }
2172 
2173 /*!
2174  * \brief Map a source stream to all of its destination streams.
2175  *
2176  * \param source_channel_name Name of channel where the source stream originates
2177  * \param bridge_stream_position The slot in the bridge where source video will come from
2178  * \param participants The bridge_channels in the bridge
2179  * \param source_channel_stream_position The position of the stream on the source channel
2180  */
2181 static void map_source_to_destinations(const char *source_channel_name,
2182  size_t bridge_stream_position, struct ast_bridge_channels_list *participants, int source_channel_stream_position)
2183 {
2184  struct ast_bridge_channel *participant;
2185 
2186  AST_LIST_TRAVERSE(participants, participant, entry) {
2187  int i;
2188  struct ast_stream_topology *topology;
2189 
2190  if (!strcmp(source_channel_name, ast_channel_name(participant->chan))) {
2191  continue;
2192  }
2193 
2194  ast_bridge_channel_lock(participant);
2195  ast_channel_lock(participant->chan);
2196  topology = ast_channel_get_stream_topology(participant->chan);
2197 
2198  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2199  struct ast_stream *stream;
2200 
2201  stream = ast_stream_topology_get_stream(topology, i);
2202  if (is_video_dest(stream, source_channel_name, source_channel_stream_position)) {
2203  struct softmix_channel *sc = participant->tech_pvt;
2204 
2205  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, bridge_stream_position, i);
2206  AST_VECTOR_APPEND(&sc->video_sources, bridge_stream_position);
2207  break;
2208  }
2209  }
2210  ast_channel_unlock(participant->chan);
2211  ast_bridge_channel_unlock(participant);
2212  }
2213 }
2214 
2215 /*!
2216  * \brief Allocate a REMB collector
2217  *
2218  * \retval non-NULL success
2219  * \retval NULL failure
2220  */
2222 {
2223  struct softmix_remb_collector *collector;
2224 
2225  collector = ao2_alloc_options(sizeof(*collector), NULL, AO2_ALLOC_OPT_LOCK_NOLOCK);
2226  if (!collector) {
2227  return NULL;
2228  }
2229 
2230  collector->frame.frametype = AST_FRAME_RTCP;
2231  collector->frame.subclass.integer = AST_RTP_RTCP_PSFB;
2232  collector->feedback.fmt = AST_RTP_RTCP_FMT_REMB;
2233  collector->frame.data.ptr = &collector->feedback;
2234  collector->frame.datalen = sizeof(collector->feedback);
2235 
2236  return collector;
2237 }
2238 
2239 /*!
2240  * \brief Setup REMB collection for a particular bridge stream and channel.
2241  *
2242  * \param bridge The bridge
2243  * \param bridge_channel Channel that is collecting REMB information
2244  * \param bridge_stream_position The slot in the bridge where source video comes from
2245  */
2246 static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2247  size_t bridge_stream_position)
2248 {
2249  struct softmix_channel *sc = bridge_channel->tech_pvt;
2250  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2251 
2252  if (!sc->remb_collector) {
2254  if (!sc->remb_collector) {
2255  /* This is not fatal. Things will still continue to work but we won't
2256  * produce a REMB report to the sender.
2257  */
2258  return;
2259  }
2260  }
2261 
2262  ao2_ref(sc->remb_collector, +1);
2263  if (AST_VECTOR_REPLACE(&softmix_data->remb_collectors, bridge_stream_position,
2264  sc->remb_collector)) {
2265  ao2_ref(sc->remb_collector, -1);
2266  }
2267 }
2268 
2269 static void softmix_bridge_stream_sources_update(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel,
2270  struct softmix_channel *sc)
2271 {
2272  int index;
2273  struct ast_stream_topology *old_topology = sc->topology;
2274  struct ast_stream_topology *new_topology = ast_channel_get_stream_topology(bridge_channel->chan);
2275  int removed_streams[MAX(ast_stream_topology_get_count(sc->topology), ast_stream_topology_get_count(new_topology))];
2276  size_t removed_streams_count = 0;
2277  struct ast_stream_topology *added_streams;
2278  struct ast_bridge_channels_list *participants = &bridge->channels;
2279  struct ast_bridge_channel *participant;
2280  SCOPE_ENTER(3, "%s: OT: %s NT: %s\n", ast_channel_name(bridge_channel->chan),
2281  ast_str_tmp(256, ast_stream_topology_to_str(old_topology, &STR_TMP)),
2282  ast_str_tmp(256, ast_stream_topology_to_str(new_topology, &STR_TMP)));
2283 
2284  added_streams = ast_stream_topology_alloc();
2285  if (!added_streams) {
2286  SCOPE_EXIT_LOG(LOG_ERROR, "%s: Couldn't alloc topology\n", ast_channel_name(bridge_channel->chan));
2287  }
2288 
2289  /* We go through the old topology comparing it to the new topology to determine what streams
2290  * changed state. A state transition can result in the stream being considered a new source
2291  * (for example it was removed and is now present) or being removed (a stream became inactive).
2292  * Added streams are copied into a topology and added to each other participant while for
2293  * removed streams we merely store their position and mark them as removed later.
2294  */
2295  ast_trace(-1, "%s: Checking for state changes\n", ast_channel_name(bridge_channel->chan));
2296  for (index = 0; index < ast_stream_topology_get_count(sc->topology) && index < ast_stream_topology_get_count(new_topology); ++index) {
2297  struct ast_stream *old_stream = ast_stream_topology_get_stream(sc->topology, index);
2298  struct ast_stream *new_stream = ast_stream_topology_get_stream(new_topology, index);
2299  SCOPE_ENTER(4, "%s: Slot: %d Old stream: %s New stream: %s\n", ast_channel_name(bridge_channel->chan),
2300  index, ast_str_tmp(256, ast_stream_to_str(old_stream, &STR_TMP)),
2301  ast_str_tmp(256, ast_stream_to_str(new_stream, &STR_TMP)));
2302 
2303  /* Ignore all streams that don't carry video and streams that are strictly outgoing destination streams */
2304  if ((ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) != AST_MEDIA_TYPE_VIDEO) ||
2305  !strncmp(ast_stream_get_name(new_stream), SOFTBRIDGE_VIDEO_DEST_PREFIX,
2307  SCOPE_EXIT_EXPR(continue, "%s: Stream %d ignored\n", ast_channel_name(bridge_channel->chan), index);
2308  }
2309 
2311  /* If a stream renegotiates from video to non-video then we need to remove it as a source */
2312  ast_trace(-1, "%s: Stream %d added to remove list\n", ast_channel_name(bridge_channel->chan), index);
2313  removed_streams[removed_streams_count++] = index;
2314  } else if (ast_stream_get_type(old_stream) != AST_MEDIA_TYPE_VIDEO && ast_stream_get_type(new_stream) == AST_MEDIA_TYPE_VIDEO) {
2315  if (ast_stream_get_state(new_stream) != AST_STREAM_STATE_REMOVED) {
2316  /* If a stream renegotiates from non-video to video in a non-removed state we need to add it as a source */
2317  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2318  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2319  new_stream, index)) {
2320  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append source stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2321  index, ast_stream_get_name(new_stream));
2322  }
2323  ast_trace(-1, "%s: Stream %d changed from non-video to video\n", ast_channel_name(bridge_channel->chan), index);
2324  }
2325  } else if (ast_stream_get_state(old_stream) != AST_STREAM_STATE_REMOVED &&
2327  ast_trace(-1, "%s: Stream %d added to remove list\n", ast_channel_name(bridge_channel->chan), index);
2328  /* If a stream renegotiates and is removed then we remove it */
2329  removed_streams[removed_streams_count++] = index;
2334  /* If a stream renegotiates and is added then we add it */
2335  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2336  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2337  new_stream, index)) {
2338  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append source stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2339  index, ast_stream_get_name(new_stream));
2340  }
2341  ast_trace(-1, "%s: Stream %d:%s changed state from %s to %s\n", ast_channel_name(bridge_channel->chan),
2342  index, ast_stream_get_name(old_stream), ast_stream_state2str(ast_stream_get_state(old_stream)),
2344  } else {
2345  ast_trace(-1, "%s: Stream %d:%s didn't do anything\n", ast_channel_name(bridge_channel->chan),
2346  index, ast_stream_get_name(old_stream));
2347  }
2348  SCOPE_EXIT();
2349  }
2350 
2351  /* Any newly added streams that did not take the position of a removed stream
2352  * will be present at the end of the new topology. Since streams are never
2353  * removed from the topology but merely marked as removed we can pick up where we
2354  * left off when comparing the old and new topologies.
2355  */
2356  ast_trace(-1, "%s: Checking for newly added streams\n", ast_channel_name(bridge_channel->chan));
2357 
2358  for (; index < ast_stream_topology_get_count(new_topology); ++index) {
2359  struct ast_stream *stream = ast_stream_topology_get_stream(new_topology, index);
2360  SCOPE_ENTER(4, "%s: Checking stream %d:%s\n", ast_channel_name(bridge_channel->chan), index,
2361  ast_stream_get_name(stream));
2362 
2363  if (!is_video_source(stream)) {
2364  SCOPE_EXIT_EXPR(continue, "%s: Stream %d:%s is not video source\n", ast_channel_name(bridge_channel->chan),
2365  index, ast_stream_get_name(stream));
2366  }
2367 
2368  if (append_source_stream(added_streams, ast_channel_name(bridge_channel->chan),
2369  bridge->softmix.send_sdp_label ? ast_channel_uniqueid(bridge_channel->chan) : NULL,
2370  stream, index)) {
2371  SCOPE_EXIT_EXPR(goto cleanup, "%s: Couldn't append stream %d:%s\n", ast_channel_name(bridge_channel->chan),
2372  index, ast_stream_get_name(stream));
2373  }
2374  SCOPE_EXIT("%s: Added new stream %s\n", ast_channel_name(bridge_channel->chan),
2375  ast_str_tmp(256, ast_stream_to_str(stream, &STR_TMP)));
2376  }
2377 
2378  /* We always update the stored topology if we can to reflect what is currently negotiated */
2379  sc->topology = ast_stream_topology_clone(new_topology);
2380  if (!sc->topology) {
2381  sc->topology = old_topology;
2382  } else {
2383  ast_stream_topology_free(old_topology);
2384  }
2385 
2386  /* If there are no removed sources and no added sources we don't need to renegotiate the
2387  * other participants.
2388  */
2389  if (!removed_streams_count && !ast_stream_topology_get_count(added_streams)) {
2390  ast_trace(-1, "%s: Nothing added or removed\n", ast_channel_name(bridge_channel->chan));
2391  goto cleanup;
2392  }
2393 
2394  ast_trace(-1, "%s: Processing adds and removes\n", ast_channel_name(bridge_channel->chan));
2395  /* Go through each participant adding in the new streams and removing the old ones */
2396  AST_LIST_TRAVERSE(participants, participant, entry)
2397  {
2398  struct softmix_channel *participant_sc = participant->tech_pvt;
2399  SCOPE_ENTER(4, "%s/%s: Old participant topology %s\n",
2400  ast_channel_name(bridge_channel->chan),
2401  ast_channel_name(participant->chan),
2402  ast_str_tmp(256, ast_stream_topology_to_str(participant_sc->topology, &STR_TMP)));
2403 
2404  if (participant == bridge_channel) {
2405  SCOPE_EXIT_EXPR(continue, "%s/%s: Same channel. Skipping\n",
2406  ast_channel_name(bridge_channel->chan),
2407  ast_channel_name(participant->chan));
2408  }
2409 
2410  /* We add in all the new streams first so that they do not take the place
2411  * of any of our removed streams, allowing the remote side to reset the state
2412  * for each removed stream. */
2413  if (append_all_streams(participant_sc->topology, added_streams)) {
2414  SCOPE_EXIT_EXPR(goto cleanup, "%s/%s: Couldn't append streams\n", ast_channel_name(bridge_channel->chan),
2415  ast_channel_name(participant->chan));
2416  }
2417  ast_trace(-1, "%s/%s: Adding streams %s\n", ast_channel_name(bridge_channel->chan),
2418  ast_channel_name(participant->chan),
2419  ast_str_tmp(256, ast_stream_topology_to_str(added_streams, &STR_TMP)));
2420 
2421  /* Then we go through and remove any ones that were removed */
2422  for (index = 0;
2423  removed_streams_count && index < ast_stream_topology_get_count(sc->topology); ++index) {
2424  struct ast_stream *stream = ast_stream_topology_get_stream(sc->topology, index);
2425  int removed_stream;
2426 
2427  for (removed_stream = 0; removed_stream < removed_streams_count; ++removed_stream) {
2428  if (is_video_dest(stream, ast_channel_name(bridge_channel->chan),
2429  removed_streams[removed_stream])) {
2430  ast_trace(-1, "%s/%s: Removing stream %s\n",
2431  ast_channel_name(bridge_channel->chan),
2432  ast_channel_name(participant->chan),
2433  ast_str_tmp(256, ast_stream_to_str(stream, &STR_TMP)));
2435  }
2436  }
2437  }
2438  ast_channel_request_stream_topology_change(participant->chan, participant_sc->topology, NULL);
2439  SCOPE_EXIT("%s/%s: New participant topology %s\n",
2440  ast_channel_name(bridge_channel->chan),
2441  ast_channel_name(participant->chan),
2442  ast_str_tmp(256, ast_stream_topology_to_str(participant_sc->topology, &STR_TMP)));
2443  }
2444 
2445  ast_trace(-1, "%s: New topology %s\n", ast_channel_name(bridge_channel->chan),
2446  ast_str_tmp(256, ast_stream_topology_to_str(sc->topology, &STR_TMP)));
2447 
2448 cleanup:
2449  ast_stream_topology_free(added_streams);
2450  SCOPE_EXIT();
2451 }
2452 
2453 /*!
2454  * \brief stream_topology_changed callback
2455  *
2456  * For most video modes, nothing beyond the ordinary is required.
2457  * For the SFU case, though, we need to completely remap the streams
2458  * in order to ensure video gets directed where it is expected to go.
2459  *
2460  * \param bridge The bridge
2461  * \param bridge_channel Channel whose topology has changed
2462  */
2463 static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
2464 {
2465  struct softmix_bridge_data *softmix_data = bridge->tech_pvt;
2466  struct softmix_channel *sc = bridge_channel->tech_pvt;
2467  struct ast_bridge_channel *participant;
2468  struct ast_vector_int media_types;
2469  int nths[AST_MEDIA_TYPE_END] = {0};
2470  int idx;
2471  SCOPE_ENTER(3, "%s: \n", ast_channel_name(bridge_channel->chan));
2472 
2473  switch (bridge->softmix.video_mode.mode) {
2477  default:
2478  ast_bridge_channel_stream_map(bridge_channel);
2479  SCOPE_EXIT_RTN("%s: Not in SFU mode\n", ast_channel_name(bridge_channel->chan));
2481  break;
2482  }
2483 
2484  ast_channel_lock(bridge_channel->chan);
2485  softmix_bridge_stream_sources_update(bridge, bridge_channel, sc);
2486  ast_channel_unlock(bridge_channel->chan);
2487 
2488  AST_VECTOR_INIT(&media_types, AST_MEDIA_TYPE_END);
2489 
2490  /* The bridge stream identifiers may change, so reset the mapping for them.
2491  * When channels end up getting added back in they'll reuse their existing
2492  * collector and won't need to allocate a new one (unless they were just added).
2493  */
2494  for (idx = 0; idx < AST_VECTOR_SIZE(&softmix_data->remb_collectors); ++idx) {
2495  ao2_cleanup(AST_VECTOR_GET(&softmix_data->remb_collectors, idx));
2496  AST_VECTOR_REPLACE(&softmix_data->remb_collectors, idx, NULL);
2497  }
2498 
2499  /* First traversal: re-initialize all of the participants' stream maps */
2500  AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2501  ast_bridge_channel_lock(participant);
2502 
2505 
2506  sc = participant->tech_pvt;
2507  AST_VECTOR_RESET(&sc->video_sources, AST_VECTOR_ELEM_CLEANUP_NOOP);
2508 
2509  ast_bridge_channel_unlock(participant);
2510  }
2511 
2512  /* Second traversal: Map specific video channels from their source to their destinations.
2513  *
2514  * This is similar to what is done in ast_stream_topology_map(),
2515  * except that video channels are handled differently. Each video
2516  * source has it's own unique index on the bridge. This way, a
2517  * particular channel's source video can be distributed to the
2518  * appropriate destination streams on the other channels.
2519  */
2520  AST_LIST_TRAVERSE(&bridge->channels, participant, entry) {
2521  int i;
2522  struct ast_stream_topology *topology;
2523 
2524  ast_bridge_channel_lock(participant);
2525  ast_channel_lock(participant->chan);
2526 
2527  topology = ao2_bump(ast_channel_get_stream_topology(participant->chan));
2528  if (!topology) {
2529  /* Oh, my, we are in trouble. */
2530  ast_channel_unlock(participant->chan);
2531  ast_bridge_channel_unlock(participant);
2532  continue;
2533  }
2534 
2535  for (i = 0; i < ast_stream_topology_get_count(topology); ++i) {
2536  struct ast_stream *stream = ast_stream_topology_get_stream(topology, i);
2537 
2538  if (is_video_source(stream)) {
2539  AST_VECTOR_APPEND(&media_types, AST_MEDIA_TYPE_VIDEO);
2540  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, AST_VECTOR_SIZE(&media_types) - 1);
2541  /*
2542  * There are cases where we need to bidirectionally send frames, such as for REMB reports
2543  * so we also map back to the channel.
2544  */
2545  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, AST_VECTOR_SIZE(&media_types) - 1, i);
2546  remb_enable_collection(bridge, participant, AST_VECTOR_SIZE(&media_types) - 1);
2547  /*
2548  * Unlock the channel and participant to prevent
2549  * potential deadlock in map_source_to_destinations().
2550  */
2551  ast_channel_unlock(participant->chan);
2552  ast_bridge_channel_unlock(participant);
2553 
2555  AST_VECTOR_SIZE(&media_types) - 1, &bridge->channels, i);
2556  ast_bridge_channel_lock(participant);
2557  ast_channel_lock(participant->chan);
2558  } else if (ast_stream_get_type(stream) == AST_MEDIA_TYPE_VIDEO) {
2559  /* Video stream mapping occurs directly when a video source stream
2560  * is found on a channel. Video streams should otherwise remain
2561  * unmapped.
2562  */
2563  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, -1);
2564  } else if (ast_stream_get_state(stream) != AST_STREAM_STATE_REMOVED) {
2565  /* XXX This is copied from ast_stream_topology_map(). This likely could
2566  * be factored out in some way
2567  */
2568  enum ast_media_type type = ast_stream_get_type(stream);
2569  int index = AST_VECTOR_GET_INDEX_NTH(&media_types, ++nths[type],
2571 
2572  if (index == -1) {
2573  AST_VECTOR_APPEND(&media_types, type);
2574  index = AST_VECTOR_SIZE(&media_types) - 1;
2575  }
2576 
2577  AST_VECTOR_REPLACE(&participant->stream_map.to_bridge, i, index);
2578  AST_VECTOR_REPLACE(&participant->stream_map.to_channel, index, i);
2579  }
2580  }
2581 
2582  ast_stream_topology_free(topology);
2583 
2584  ast_channel_unlock(participant->chan);
2585  ast_bridge_channel_unlock(participant);
2586  }
2587 
2588  AST_VECTOR_FREE(&media_types);
2589  SCOPE_EXIT_RTN("%s\n", ast_channel_name(bridge_channel->chan));
2590 }
2591 
2593  .name = "softmix",
2594  .capabilities = AST_BRIDGE_CAPABILITY_MULTIMIX,
2596  .create = softmix_bridge_create,
2597  .stop = softmix_bridge_stop,
2598  .destroy = softmix_bridge_destroy,
2599  .join = softmix_bridge_join,
2600  .leave = softmix_bridge_leave,
2601  .unsuspend = softmix_bridge_unsuspend,
2602  .write = softmix_bridge_write,
2603  .stream_topology_changed = softmix_bridge_stream_topology_changed,
2604 };
2605 
2606 #ifdef TEST_FRAMEWORK
2608  const char *name;
2609  const char *formats;
2611 };
2612 
2613 static struct ast_stream_topology *build_topology(const struct stream_parameters *params, size_t num_streams)
2614 {
2615  struct ast_stream_topology *topology;
2616  size_t i;
2617 
2618  topology = ast_stream_topology_alloc();
2619  if (!topology) {
2620  return NULL;
2621  }
2622 
2623  for (i = 0; i < num_streams; ++i) {
2624  RAII_VAR(struct ast_format_cap *, caps, NULL, ao2_cleanup);
2625  struct ast_stream *stream;
2626 
2628  if (!caps) {
2629  goto fail;
2630  }
2631  if (ast_format_cap_update_by_allow_disallow(caps, params[i].formats, 1) < 0) {
2632  goto fail;
2633  }
2634  stream = ast_stream_alloc(params[i].name, params[i].type);
2635  if (!stream) {
2636  goto fail;
2637  }
2638  ast_stream_set_formats(stream, caps);
2639  if (ast_stream_topology_append_stream(topology, stream) < 0) {
2640  ast_stream_free(stream);
2641  goto fail;
2642  }
2643  }
2644 
2645  return topology;
2646 
2647 fail:
2648  ast_stream_topology_free(topology);
2649  return NULL;
2650 }
2651 
2652 static int validate_stream(struct ast_test *test, struct ast_stream *stream,
2653  const struct stream_parameters *params)
2654 {
2655  const struct ast_format_cap *stream_caps;
2656  struct ast_format_cap *params_caps;
2657 
2658  if (ast_stream_get_type(stream) != params->type) {
2659  ast_test_status_update(test, "Expected stream type '%s' but got type '%s'\n",
2660  ast_codec_media_type2str(params->type),
2662  return -1;
2663  }
2664  if (strcmp(ast_stream_get_name(stream), params->name)) {
2665  ast_test_status_update(test, "Expected stream name '%s' but got type '%s'\n",
2666  params->name, ast_stream_get_name(stream));
2667  return -1;
2668  }
2669 
2670  stream_caps = ast_stream_get_formats(stream);
2672  if (!params_caps) {
2673  ast_test_status_update(test, "Allocation error on capabilities\n");
2674  return -1;
2675  }
2676  ast_format_cap_update_by_allow_disallow(params_caps, params->formats, 1);
2677 
2678  if (!ast_format_cap_identical(stream_caps, params_caps)) {
2679  ast_test_status_update(test, "Formats are not as expected on stream '%s'\n",
2680  ast_stream_get_name(stream));
2681  ao2_cleanup(params_caps);
2682  return -1;
2683  }
2684 
2685  ao2_cleanup(params_caps);
2686  return 0;
2687 }
2688 
2689 static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology,
2690  const struct stream_parameters *params, size_t num_streams)
2691 {
2692  int i;
2693 
2694  if (ast_stream_topology_get_count(topology) < num_streams) {
2695  ast_test_status_update(test, "Topology only has %d streams. Needs to have at least %zu\n",
2696  ast_stream_topology_get_count(topology), num_streams);
2697  return -1;
2698  }
2699 
2700  for (i = 0; i < num_streams; ++i) {
2701  if (validate_stream(test, ast_stream_topology_get_stream(topology, i), &params[i])) {
2702  return -1;
2703  }
2704  }
2705 
2706  return 0;
2707 }
2708 
2709 AST_TEST_DEFINE(sfu_append_source_streams)
2710 {
2712  static const struct stream_parameters bob_streams[] = {
2713  { "bob_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2714  { "bob_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2715  };
2716  static const struct stream_parameters alice_streams[] = {
2717  { "alice_audio", "ulaw,opus", AST_MEDIA_TYPE_AUDIO, },
2718  { "alice_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2719  };
2720  static const struct stream_parameters alice_dest_stream = {
2721  "softbridge_dest_PJSIP/Bob-00000001_1", "h264,vp8", AST_MEDIA_TYPE_VIDEO,
2722  };
2723  static const struct stream_parameters bob_dest_stream = {
2724  "softbridge_dest_PJSIP/Alice-00000000_1", "vp8", AST_MEDIA_TYPE_VIDEO,
2725  };
2726  struct ast_stream_topology *topology_alice = NULL;
2727  struct ast_stream_topology *topology_bob = NULL;
2728 
2729  switch (cmd) {
2730  case TEST_INIT:
2731  info->name = "sfu_append_source_streams";
2732  info->category = "/bridges/bridge_softmix/";
2733  info->summary = "Test appending of video streams";
2734  info->description =
2735  "This tests does stuff.";
2736  return AST_TEST_NOT_RUN;
2737  case TEST_EXECUTE:
2738  break;
2739  }
2740 
2741  topology_alice = build_topology(alice_streams, ARRAY_LEN(alice_streams));
2742  if (!topology_alice) {
2743  goto end;
2744  }
2745 
2746  topology_bob = build_topology(bob_streams, ARRAY_LEN(bob_streams));
2747  if (!topology_bob) {
2748  goto end;
2749  }
2750 
2751  if (append_source_streams(topology_alice, "PJSIP/Bob-00000001", NULL, topology_bob)) {
2752  ast_test_status_update(test, "Failed to append Bob's streams to Alice\n");
2753  goto end;
2754  }
2755 
2756  if (ast_stream_topology_get_count(topology_alice) != 3) {
2757  ast_test_status_update(test, "Alice's topology isn't large enough! It's %d but needs to be %d\n",
2758  ast_stream_topology_get_count(topology_alice), 3);
2759  goto end;
2760  }
2761 
2762  if (validate_original_streams(test, topology_alice, alice_streams, ARRAY_LEN(alice_streams))) {
2763  goto end;
2764  }
2765 
2766  if (validate_stream(test, ast_stream_topology_get_stream(topology_alice, 2), &alice_dest_stream)) {
2767  goto end;
2768  }
2769 
2770  if (append_source_streams(topology_bob, "PJSIP/Alice-00000000", NULL, topology_alice)) {
2771  ast_test_status_update(test, "Failed to append Alice's streams to Bob\n");
2772  goto end;
2773  }
2774 
2775  if (ast_stream_topology_get_count(topology_bob) != 3) {
2776  ast_test_status_update(test, "Bob's topology isn't large enough! It's %d but needs to be %d\n",
2777  ast_stream_topology_get_count(topology_bob), 3);
2778  goto end;
2779  }
2780 
2781  if (validate_original_streams(test, topology_bob, bob_streams, ARRAY_LEN(bob_streams))) {
2782  goto end;
2783  }
2784 
2785  if (validate_stream(test, ast_stream_topology_get_stream(topology_bob, 2), &bob_dest_stream)) {
2786  goto end;
2787  }
2788 
2789  res = AST_TEST_PASS;
2790 
2791 end:
2792  ast_stream_topology_free(topology_alice);
2793  ast_stream_topology_free(topology_bob);
2794  return res;
2795 }
2796 
2797 AST_TEST_DEFINE(sfu_remove_destination_streams)
2798 {
2800  static const struct stream_parameters params[] = {
2801  { "alice_audio", "ulaw,alaw,g722,opus", AST_MEDIA_TYPE_AUDIO, },
2802  { "alice_video", "h264,vp8", AST_MEDIA_TYPE_VIDEO, },
2803  { "softbridge_dest_PJSIP/Bob-00000001_video", "vp8", AST_MEDIA_TYPE_VIDEO, },
2804  { "softbridge_dest_PJSIP/Carol-00000002_video", "h264", AST_MEDIA_TYPE_VIDEO, },
2805  };
2806  static const struct {
2807  const char *channel_name;
2808  int num_streams;
2809  int params_index[4];
2810  } removal_results[] = {
2811  { "PJSIP/Bob-00000001", 4, { 0, 1, 2, 3 }, },
2812  { "PJSIP/Edward-00000004", 4, { 0, 1, 2, 3 }, },
2813  { "", 4, { 0, 1, 2, 3 }, },
2814  };
2815  struct ast_stream_topology *orig = NULL;
2816  int i;
2817 
2818  switch (cmd) {
2819  case TEST_INIT:
2820  info->name = "sfu_remove_destination_streams";
2821  info->category = "/bridges/bridge_softmix/";
2822  info->summary = "Test removal of destination video streams";
2823  info->description =
2824  "This tests does stuff.";
2825  return AST_TEST_NOT_RUN;
2826  case TEST_EXECUTE:
2827  break;
2828  }
2829 
2830  orig = build_topology(params, ARRAY_LEN(params));
2831  if (!orig) {
2832  ast_test_status_update(test, "Unable to build initial stream topology\n");
2833  goto end;
2834  }
2835 
2836  for (i = 0; i < ARRAY_LEN(removal_results); ++i) {
2837  int j;
2838 
2839  remove_destination_streams(orig, removal_results[i].channel_name);
2840 
2841  if (ast_stream_topology_get_count(orig) != removal_results[i].num_streams) {
2842  ast_test_status_update(test, "Resulting topology has %d streams, when %d are expected\n",
2843  ast_stream_topology_get_count(orig), removal_results[i].num_streams);
2844  goto end;
2845  }
2846 
2847  for (j = 0; j < removal_results[i].num_streams; ++j) {
2848  struct ast_stream *actual;
2849  struct ast_stream *expected;
2850  int orig_index;
2851 
2852  actual = ast_stream_topology_get_stream(orig, j);
2853 
2854  orig_index = removal_results[i].params_index[j];
2855  expected = ast_stream_topology_get_stream(orig, orig_index);
2856 
2858  ast_stream_get_formats(expected))) {
2859  struct ast_str *expected_str;
2860  struct ast_str *actual_str;
2861 
2862  expected_str = ast_str_alloca(64);
2863  actual_str = ast_str_alloca(64);
2864 
2865  ast_test_status_update(test, "Mismatch between expected (%s) and actual (%s) stream formats\n",
2866  ast_format_cap_get_names(ast_stream_get_formats(expected), &expected_str),
2867  ast_format_cap_get_names(ast_stream_get_formats(actual), &actual_str));
2868  goto end;
2869  }
2870 
2871  if (is_video_dest(actual, removal_results[i].channel_name, -1) &&
2873  ast_test_status_update(test, "Removed stream %s does not have a state of removed\n", ast_stream_get_name(actual));
2874  goto end;
2875  }
2876  }
2877  }
2878 
2879  res = AST_TEST_PASS;
2880 
2881 end:
2883  return res;
2884 }
2885 
2886 #endif
2887 
2888 static int unload_module(void)
2889 {
2890  ast_bridge_technology_unregister(&softmix_bridge);
2891  AST_TEST_UNREGISTER(sfu_append_source_streams);
2892  AST_TEST_UNREGISTER(sfu_remove_destination_streams);
2893  return 0;
2894 }
2895 
2896 static int load_module(void)
2897 {
2898  if (ast_bridge_technology_register(&softmix_bridge)) {
2899  unload_module();
2900  return AST_MODULE_LOAD_DECLINE;
2901  }
2902  AST_TEST_REGISTER(sfu_append_source_streams);
2903  AST_TEST_REGISTER(sfu_remove_destination_streams);
2904  return AST_MODULE_LOAD_SUCCESS;
2905 }
2906 
2907 AST_MODULE_INFO_STANDARD(ASTERISK_GPL_KEY, "Multi-party software based channel mixing");
void check_binaural_position_change(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data)
Checks if a position change in the virual enviroment is requested by one of the participants.
#define AST_VECTOR_FREE(vec)
Deallocates this vector.
Definition: vector.h:174
unsigned int num_above_maximum_rate
static void softmix_bridge_write_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
static const char type[]
Definition: chan_ooh323.c:109
#define SCOPE_EXIT_LOG(__log_level,...)
Definition: logger.h:928
pthread_t thread
Definition: app_meetme.c:1089
#define ast_channel_lock(chan)
Definition: channel.h:2945
Multi-party software based channel mixing (header)
const char * ast_stream_to_str(const struct ast_stream *stream, struct ast_str **buf)
Get a string representing the stream for debugging/display purposes.
Definition: stream.c:337
#define AST_MODULE_INFO_STANDARD(keystr, desc)
Definition: module.h:567
int ast_channel_hold_state(const struct ast_channel *chan)
static void * softmix_translate_helper_free_entry(struct softmix_translate_helper_entry *entry)
struct convolve_channel_pair ** chan_pairs
#define AST_VECTOR_GET_INDEX_NTH(vec, nth, value, cmp)
Get the nth index from a vector that matches the given comparison.
Definition: vector.h:698
Asterisk main include file. File version handling, generic pbx functions.
unsigned int ast_format_get_channel_count(const struct ast_format *format)
Get the channel count on a format.
Definition: format.c:135
const ast_string_field uniqueid
Definition: bridge.h:409
#define ast_realloc(p, len)
A wrapper for realloc()
Definition: astmm.h:228
struct ast_bridge_features * features
int ast_dsp_silence_with_energy(struct ast_dsp *dsp, struct ast_frame *f, int *totalsilence, int *frames_energy)
Process the audio frame for silence.
Definition: dsp.c:1478
#define ARRAY_LEN(a)
Definition: isdn_lib.c:42
#define SOFTBRIDGE_VIDEO_DEST_PREFIX
static void softmix_bridge_stop(struct ast_bridge *bridge)
unsigned int num_active
Definition: bridge.h:383
static int softmix_mixing_array_grow(struct softmix_mixing_array *mixing_array, unsigned int num_entries, unsigned int binaural_active)
int init_convolve_data(struct convolve_data *data, unsigned int default_sample_size)
Preinits a specific number of channels (CONVOVLE_CHANNEL_PREALLOC) at the beginning of a conference...
struct ast_stream_topology * ast_channel_get_stream_topology(const struct ast_channel *chan)
Retrieve the topology of streams on a channel.
static struct ast_stream_topology * build_topology(const struct stream_parameters *params, size_t num_streams)
static int validate_original_streams(struct ast_test *test, struct ast_stream_topology *topology, const struct stream_parameters *params, size_t num_streams)
void ast_slinfactory_flush(struct ast_slinfactory *sf)
Flush the contents of a slinfactory.
Definition: slinfactory.c:204
static int softmix_bridge_write(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
unsigned int remb_send_interval
Definition: bridge.h:151
struct ast_stream_topology * topology
short int16_t
Definition: db.h:59
static void softmix_translate_helper_change_rate(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
int ast_indicate(struct ast_channel *chan, int condition)
Indicates condition of channel.
Definition: channel.c:4322
static int softmix_bridge_create(struct ast_bridge *bridge)
Function called when a bridge is created.
void ast_dsp_free(struct ast_dsp *dsp)
Definition: dsp.c:1770
char buf[BUFSIZE]
Definition: eagi_proxy.c:66
unsigned int internal_mixing_interval
The mixing interval indicates how quickly softmix mixing should occur to mix audio.
Definition: bridge.h:301
unsigned int locked_rate
#define AST_VECTOR_ELEM_DEFAULT_CMP(elem, value)
Default comparator for AST_VECTOR_REMOVE_ELEM_UNORDERED()
Definition: vector.h:564
enum ast_media_type ast_stream_get_type(const struct ast_stream *stream)
Get the media type of a stream.
Definition: stream.c:316
static void sfu_topologies_on_join(struct ast_bridge *bridge, struct ast_bridge_channel *joiner)
Issue channel stream topology change requests.
#define LOG_WARNING
Definition: logger.h:274
ast_callid callid
Definition: bridge.h:369
struct ast_bridge_video_mode video_mode
Definition: bridge.h:287
static int timeout
Definition: cdr_mysql.c:86
static int tmp()
Definition: bt_open.c:389
const char * ast_codec_media_type2str(enum ast_media_type type)
Conversion function to take a media type and turn it into a string.
Definition: codec.c:347
#define AST_VECTOR_APPEND(vec, elem)
Append an element to a vector, growing the vector if needed.
Definition: vector.h:256
static void softmix_translate_helper_destroy(struct softmix_translate_helper *trans_helper)
#define AST_LOG_WARNING
Definition: logger.h:279
unsigned int suspended
static int remove_destination_streams(struct ast_stream_topology *topology, const char *channel_name)
An object that represents data received in a feedback report.
Definition: rtp_engine.h:358
Set when the stream has been removed/declined.
Definition: stream.h:78
Test Framework API.
int ast_slinfactory_feed(struct ast_slinfactory *sf, struct ast_frame *f)
Feed audio into a slinfactory.
Definition: slinfactory.c:77
#define SOFTBRIDGE_VIDEO_DEST_SEPARATOR
#define AST_TEST_REGISTER(cb)
Definition: test.h:127
void ast_timer_close(struct ast_timer *handle)
Close an opened timing handle.
Definition: timing.c:154
#define ao2_t_replace(dst, src, tag)
Replace one object reference with another cleaning up the original.
Definition: astobj2.h:503
Structure used to transport a message through the frame core.
Definition: message.c:1406
#define ast_trace(level,...)
Print a basic trace message.
Definition: logger.h:692
static void softmix_bridge_write_rtcp(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
short our_buf[MAX_DATALEN]
struct ast_timer * ast_timer_open(void)
Open a timer.
Definition: timing.c:122
static struct softmix_remb_collector * remb_collector_alloc(void)
Allocate a REMB collector.
Definition of a media format.
Definition: format.c:43
const char * formats
#define SOFTMIX_MIN_SAMPLE_RATE
#define ast_cond_wait(cond, mutex)
Definition: lock.h:203
#define ast_cond_init(cond, attr)
Definition: lock.h:199
static void * softmix_mixing_thread(void *data)
struct timeval ast_tvnow(void)
Returns current timeval. Meant to replace calls to gettimeofday().
Definition: time.h:150
static void remb_collect_report_all(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, float bitrate)
#define ao2_alloc_options(data_size, destructor_fn, options)
Definition: astobj2.h:406
#define ast_assert(a)
Definition: utils.h:695
#define ast_mutex_lock(a)
Definition: lock.h:187
struct ast_frame * ast_translate(struct ast_trans_pvt *tr, struct ast_frame *f, int consume)
translates one or more frames Apply an input frame into the translator and receive zero or one output...
Definition: translate.c:565
Set when the stream is not sending OR receiving media.
Definition: stream.h:94
#define ast_str_alloca(init_len)
Definition: strings.h:800
const char * ast_format_get_name(const struct ast_format *format)
Get the name associated with a format.
Definition: format.c:334
int64_t ast_tvdiff_ms(struct timeval end, struct timeval start)
Computes the difference (in milliseconds) between two struct timeval instances.
Definition: time.h:98
static void softmix_bridge_write_text(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
void softmix_process_write_binaural_audio(struct softmix_channel *sc, unsigned int default_sample_size)
Writes the binaural audio to a channel.
unsigned int fmt
Definition: rtp_engine.h:359
struct ast_rtp_rtcp_feedback_remb remb
Definition: rtp_engine.h:361
int ast_channel_request_stream_topology_change(struct ast_channel *chan, struct ast_stream_topology *topology, void *change_source)
Request that the stream topology of a channel change.
Definition: channel.c:11167
struct ast_bridge_channel::@236 stream_map
struct ast_stream * ast_stream_topology_get_stream(const struct ast_stream_topology *topology, unsigned int position)
Get a specific stream from the topology.
Definition: stream.c:788
#define NULL
Definition: resample.c:96
int ast_stream_topology_append_stream(struct ast_stream_topology *topology, struct ast_stream *stream)
Append a stream to the topology.
Definition: stream.c:748
char * end
Definition: eagi_proxy.c:73
static void map_source_to_destinations(const char *source_channel_name, size_t bridge_stream_position, struct ast_bridge_channels_list *participants, int source_channel_stream_position)
Map a source stream to all of its destination streams.
#define DEFAULT_SOFTMIX_SILENCE_THRESHOLD
Default time in ms of silence necessary to declare talking stopped by the bridge. ...
Out-of-call text message support.
#define LOG_DEBUG
Definition: logger.h:241
int ast_bridge_queue_everyone_else(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
Queue the given frame to everyone else.
void ast_stream_set_formats(struct ast_stream *stream, struct ast_format_cap *caps)
Set the current negotiated formats of a stream.
Definition: stream.c:365
short final_buf[MAX_DATALEN]
#define AST_LIST_TRAVERSE_SAFE_END
Closes a safe loop traversal block.
Definition: linkedlists.h:614
static void softmix_mixing_array_destroy(struct softmix_mixing_array *mixing_array, unsigned int binaural_active)
#define ast_cond_signal(cond)
Definition: lock.h:201
struct softmix_translate_helper::@96 entries
int ast_stream_topology_set_stream(struct ast_stream_topology *topology, unsigned int position, struct ast_stream *stream)
Set a specific position in a topology.
Definition: stream.c:796
struct ast_format * dst_format
struct ast_bridge_softmix softmix
Definition: bridge.h:375
union ast_bridge_video_mode::@227 mode_data
static int softmix_bridge_write_control(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
unsigned int ast_slinfactory_available(const struct ast_slinfactory *sf)
Retrieve number of samples currently in a slinfactory.
Definition: slinfactory.c:199
enum ast_bridge_video_sfu_remb_behavior remb_behavior
Definition: bridge.h:153
static void softmix_bridge_destroy(struct ast_bridge *bridge)
Function called when a bridge is destroyed.
struct ast_vector_int to_bridge
struct ast_bridge * bridge
Bridge this channel is participating in.
struct ast_frame_subclass subclass
Media Stream API.
int ast_set_write_format_interleaved_stereo(struct ast_channel *chan, struct ast_format *format)
Sets write format for a channel. All internal data will than be handled in an interleaved format...
Definition: channel.c:5872
#define ast_asprintf(ret, fmt,...)
A wrapper for asprintf()
Definition: astmm.h:269
int ast_set_read_format_path(struct ast_channel *chan, struct ast_format *raw_format, struct ast_format *core_format)
Set specific read path on channel.
Definition: channel.c:5575
#define ast_strlen_zero(foo)
Definition: strings.h:52
unsigned int sample_rates[16]
int ast_format_cap_update_by_allow_disallow(struct ast_format_cap *cap, const char *list, int allowing)
Parse an "allow" or "deny" list and modify a format capabilities structure accordingly.
Definition: format_cap.c:320
#define DEFAULT_SOFTMIX_INTERVAL
Interval at which mixing will take place. Valid options are 10, 20, and 40.
#define ao2_bump(obj)
Definition: astobj2.h:491
int ast_callid_threadassoc_add(ast_callid callid)
Adds a known callid to thread storage of the calling thread.
Definition: logger.c:1984
#define MIN(a, b)
Definition: utils.h:226
#define SOFTMIX_SAMPLES(rate, interval)
Number of samples we are dealing with.
#define SOFTMIX_BINAURAL_SAMPLE_RATE
static void softmix_bridge_write_video(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct ast_frame *frame)
void ast_bridge_update_talker_src_video_mode(struct ast_bridge *bridge, struct ast_channel *chan, int talker_energy, int is_keyfame)
Update information about talker energy for talker src video mode.
Definition: bridge.c:3883
struct softmix_translate_helper_entry::@95 entry
#define ast_str_tmp(init_len, __expr)
Definition: strings.h:1136
#define ast_debug(level,...)
Log a DEBUG message.
Definition: logger.h:452
#define ast_log
Definition: astobj2.c:42
#define MAX(a, b)
Definition: utils.h:228
const char * ast_stream_topology_to_str(const struct ast_stream_topology *topology, struct ast_str **buf)
Get a string representing the topology for debugging/display purposes.
Definition: stream.c:936
enum ast_format_cmp_res ast_format_cmp(const struct ast_format *format1, const struct ast_format *format2)
Compare two formats.
Definition: format.c:201
#define ast_bridge_channel_lock(bridge_channel)
Lock the bridge_channel.
#define AST_VECTOR_INIT(vec, size)
Initialize a vector.
Definition: vector.h:113
struct ast_trans_pvt * ast_translator_build_path(struct ast_format *dest, struct ast_format *source)
Builds a translator path Build a path (possibly NULL) from source to dest.
Definition: translate.c:485
int ast_waitfor_n_fd(int *fds, int n, int *ms, int *exception)
Waits for input on an fd.
Definition: channel.c:2989
#define BINAURAL_MIXING_INTERVAL
struct ast_frame frame
#define AST_VECTOR_ELEM_CLEANUP_NOOP(elem)
Vector element cleanup that does nothing.
Definition: vector.h:573
#define RAII_VAR(vartype, varname, initval, dtor)
Declare a variable that will call a destructor function when it goes out of scope.
Definition: utils.h:911
#define ast_test_status_update(a, b, c...)
Definition: test.h:129
static void softmix_bridge_unsuspend(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel is unsuspended from the bridge.
#define AST_PTHREADT_NULL
Definition: lock.h:66
struct ast_format * read_slin_format
int ast_bridge_channel_notify_talking(struct ast_bridge_channel *bridge_channel, int started_talking)
Lets the bridging indicate when a bridge channel has stopped or started talking.
#define AST_LIST_REMOVE_CURRENT(field)
Removes the current entry from a list during a traversal.
Definition: linkedlists.h:556
#define ao2_ref(o, delta)
Definition: astobj2.h:464
unsigned int maximum_rate
static force_inline void ast_slinear_saturated_add(short *input, short *value)
Definition: utils.h:425
static void sfu_topologies_on_source_change(struct ast_bridge *bridge, struct ast_bridge_channel *source)
static void softmix_bridge_data_destroy(struct softmix_bridge_data *softmix_data)
AST_TEST_DEFINE(sfu_append_source_streams)
int ast_set_write_format(struct ast_channel *chan, struct ast_format *format)
Sets write format on channel chan.
Definition: channel.c:5890
#define SCOPE_EXIT(...)
Scope Exit.
Definition: logger.h:805
#define AST_LIST_REMOVE_HEAD(head, field)
Removes and returns the head entry from a list.
Definition: linkedlists.h:832
unsigned int num_at_internal_rate
const struct ast_format_cap * ast_stream_get_formats(const struct ast_stream *stream)
Get the current negotiated formats of a stream.
Definition: stream.c:330
void set_binaural_data_leave(struct convolve_data *data, unsigned int pos, unsigned int default_sample_size)
Removes a channel from the binaural conference bridge. Marks the position in the virtual room as unus...
AST_LIST_HEAD_NOLOCK(contactliststruct, contact)
#define ast_format_cap_alloc(flags)
Definition: format_cap.h:52
static void softmix_bridge_check_voice(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
#define SCOPE_EXIT_EXPR(__expr,...)
Definition: logger.h:834
enum ast_media_type type
void ast_slinfactory_destroy(struct ast_slinfactory *sf)
Destroy the contents of a slinfactory.
Definition: slinfactory.c:58
struct ast_format * write_format
void ast_dsp_set_threshold(struct ast_dsp *dsp, int threshold)
Set the minimum average magnitude threshold to determine talking by the DSP.
Definition: dsp.c:1775
#define ast_bridge_technology_register(technology)
See __ast_bridge_technology_register()
#define ast_alloca(size)
call __builtin_alloca to ensure we get gcc builtin semantics
Definition: astmm.h:290
const char * ast_channel_uniqueid(const struct ast_channel *chan)
static int is_video_dest(const struct ast_stream *stream, const char *source_channel_name, int source_channel_stream_position)
Determine if a stream is a video destination stream.
#define ast_bridge_channel_unlock(bridge_channel)
Unlock the bridge_channel.
struct ast_vector_int to_channel
int ast_slinfactory_init_with_format(struct ast_slinfactory *sf, struct ast_format *slin_out)
Initialize a slinfactory.
Definition: slinfactory.c:46
void free_convolve_data(struct convolve_data *data)
Frees all channels and data needed for binaural audio processing.
static int unload_module(void)
#define SCOPE_EXIT_LOG_RTN(__log_level,...)
Definition: logger.h:934
Structure that contains information about a bridge.
Definition: bridge.h:357
int ast_timer_ack(const struct ast_timer *handle, unsigned int quantity)
Acknowledge a timer event.
Definition: timing.c:171
struct ast_format * ast_channel_rawreadformat(struct ast_channel *chan)
static void remb_send_report(struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
Set when the stream is sending and receiving media.
Definition: stream.h:82
int ast_bridge_technology_unregister(struct ast_bridge_technology *technology)
Unregister a bridge technology from use.
Definition: bridge.c:265
static int append_source_stream(struct ast_stream_topology *dest, const char *channel_name, const char *sdp_label, struct ast_stream *stream, int index)
static struct softmix_translate_helper_entry * softmix_translate_helper_entry_alloc(struct ast_format *dst)
#define LOG_ERROR
Definition: logger.h:285
struct ast_frame write_frame
struct ast_format * slin_src
The descriptor of a dynamic string XXX storage will be optimized later if needed We use the ts field ...
Definition: strings.h:584
Format capabilities structure, holds formats + preference order + etc.
Definition: format_cap.c:54
Default structure for translators, with the basic fields and buffers, all allocated as part of the sa...
Definition: translate.h:213
struct ast_bridge_video_sfu_data sfu_data
Definition: bridge.h:165
struct ast_stream_topology * ast_stream_topology_alloc(void)
Create a stream topology.
Definition: stream.c:650
static int softmix_bridge_join(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel is joined into the bridge.
Structure which contains per-channel mixing information.
static int16_t * softmix_process_read_audio(struct softmix_channel *sc, unsigned int num_samples)
#define AST_TEST_UNREGISTER(cb)
Definition: test.h:128
void create_binaural_frame(struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc, int16_t *bin_buf, int16_t *ann_buf, unsigned int softmix_datalen, unsigned int softmix_samples, int16_t *buf)
Creates a frame out of binaural audio data.
static void softmix_translate_helper_init(struct softmix_translate_helper *trans_helper, unsigned int sample_rate)
struct ast_bridge_tech_optimizations tech_args
int ast_bridge_is_video_src(struct ast_bridge *bridge, struct ast_channel *chan)
Determine if a channel is a video src for the bridge.
Definition: bridge.c:3958
def info(msg)
static int append_all_streams(struct ast_stream_topology *dest, const struct ast_stream_topology *source)
const char * ast_msg_data_get_attribute(struct ast_msg_data *msg, enum ast_msg_data_attribute_type attribute_type)
Get attribute from ast_msg_data.
Definition: message.c:1533
frame_type
Definition: codec_builtin.c:44
Definition: file.c:69
static void softmix_bridge_stream_sources_update(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct softmix_channel *sc)
#define ast_cond_destroy(cond)
Definition: lock.h:200
#define LOG_NOTICE
Definition: logger.h:263
void ast_bridge_channel_leave_bridge(struct ast_bridge_channel *bridge_channel, enum bridge_channel_state new_state, int cause)
Set bridge channel state to leave bridge (if not leaving already).
const char * ast_format_cap_get_names(const struct ast_format_cap *cap, struct ast_str **buf)
Get the names of codecs of a set of formats.
Definition: format_cap.c:736
static void clear_talking(struct ast_bridge_channel *bridge_channel)
#define SCOPE_EXIT_RTN(...)
Scope Exit with return.
Definition: logger.h:854
#define AST_LIST_TRAVERSE(head, var, field)
Loops over (traverses) the entries in a list.
Definition: linkedlists.h:490
#define AST_LIST_ENTRY(type)
Declare a forward link structure inside a list entry.
Definition: linkedlists.h:409
#define AST_LIST_INSERT_HEAD(head, elm, field)
Inserts a list entry at the head of a list.
Definition: linkedlists.h:710
struct ast_slinfactory factory
static void remb_collect_report(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, struct softmix_bridge_data *softmix_data, struct softmix_channel *sc)
#define ast_channel_unlock(chan)
Definition: channel.h:2946
#define ast_bridge_unlock(bridge)
Unlock the bridge.
Definition: bridge.h:493
Set when the stream is sending media only.
Definition: stream.h:86
static const char name[]
Definition: cdr_mysql.c:74
#define ast_free(a)
Definition: astmm.h:182
#define ast_calloc(num, len)
A wrapper for calloc()
Definition: astmm.h:204
#define ast_pthread_create(a, b, c, d)
Definition: utils.h:559
struct ast_bridge * bridge
Bridge pointer passed to the softmix mixing thread.
#define AST_VECTOR_RESET(vec, cleanup)
Reset vector.
Definition: vector.h:627
struct ast_frame * out_frame
struct video_follow_talker_data video_talker
int ast_format_cap_identical(const struct ast_format_cap *cap1, const struct ast_format_cap *cap2)
Determine if two capabilities structures are identical.
Definition: format_cap.c:689
struct ast_rtp_rtcp_feedback_remb remb
void ast_stream_set_state(struct ast_stream *stream, enum ast_stream_state state)
Set the state of a stream.
Definition: stream.c:380
static struct ast_bridge_technology softmix_bridge
#define SCOPE_ENTER(level,...)
Non RAII_VAR Scope Trace macros The advantage of these macros is that the EXITs will have the actual ...
Definition: logger.h:780
int num_times_requested
static void gather_softmix_stats(struct softmix_stats *stats, const struct softmix_bridge_data *softmix_data, struct ast_bridge_channel *bridge_channel)
static int is_video_source(const struct ast_stream *stream)
Determine if a stream is a video source stream.
unsigned int video_update_discard
Definition: bridge.h:168
static void softmix_pass_video_top_priority(struct ast_bridge *bridge, struct ast_frame *frame)
Module has failed to load, may be in an inconsistent state.
Definition: module.h:78
struct ast_trans_pvt * trans_pvt
Vector container support.
unsigned int talking
TRUE if a channel is talking.
unsigned int num_channels[16]
static int validate_stream(struct ast_test *test, struct ast_stream *stream, const struct stream_parameters *params)
static void softmix_process_write_audio(struct softmix_translate_helper *trans_helper, struct ast_format *raw_write_fmt, struct softmix_channel *sc, unsigned int default_sample_size)
int set_binaural_data_join(struct convolve_data *data, unsigned int default_sample_size)
Joins a channel into a virtual enviroment build with the help of binaural sythesis.
void * tech_pvt
Private information unique to the bridge technology.
int ast_timer_fd(const struct ast_timer *handle)
Get a poll()-able file descriptor for a timer.
Definition: timing.c:161
#define MAX_DATALEN
int ast_timer_set_rate(const struct ast_timer *handle, unsigned int rate)
Set the timing tick rate.
Definition: timing.c:166
int ast_stream_topology_get_count(const struct ast_stream_topology *topology)
Get the number of streams in a topology.
Definition: stream.c:765
static void * cleanup(void *unused)
Definition: pbx_realtime.c:124
char * ast_frame_type2str(enum ast_frame_type frame_type, char *ftype, size_t len)
Copy the discription of a frame type into the provided string.
Definition: main/frame.c:671
unsigned int send_sdp_label
Definition: bridge.h:308
static void remb_enable_collection(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel, size_t bridge_stream_position)
Setup REMB collection for a particular bridge stream and channel.
static void softmix_translate_helper_cleanup(struct softmix_translate_helper *trans_helper)
struct ast_stream_topology * ast_stream_topology_clone(const struct ast_stream_topology *topology)
Create a deep clone of an existing stream topology.
Definition: stream.c:667
#define ast_bridge_lock(bridge)
Lock the bridge.
Definition: bridge.h:480
#define SCOPE_EXIT_RTN_VALUE(__return_value,...)
Scope Exit with return value.
Definition: logger.h:875
int ast_bridge_channel_queue_frame(struct ast_bridge_channel *bridge_channel, struct ast_frame *fr)
Write a frame to the specified bridge_channel.
void ast_stream_free(struct ast_stream *stream)
Destroy a media stream representation.
Definition: stream.c:292
struct ast_bridge_channels_list channels
Definition: bridge.h:371
#define AST_VECTOR_GET(vec, idx)
Get an element from a vector.
Definition: vector.h:682
struct ast_channel * chan
Structure that contains information regarding a channel in a bridge.
static void set_softmix_bridge_data(int rate, int interval, struct ast_bridge_channel *bridge_channel, int reset, int set_binaural, int binaural_pos_id, int is_announcement)
struct ast_stream * ast_stream_alloc(const char *name, enum ast_media_type type)
Create a new media stream representation.
Definition: stream.c:233
#define ao2_replace(dst, src)
Definition: astobj2.h:517
void add_binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, unsigned int softmix_samples, struct softmix_mixing_array *mixing_array, struct softmix_channel *sc, const char *channel_name)
Processes audio data with the binaural synthesis and adds the result to the mixing array...
Definition: test.c:65
#define ao2_cleanup(obj)
Definition: astobj2.h:1958
struct ast_stream * ast_stream_clone(const struct ast_stream *stream, const char *name)
Create a deep clone of an existing stream.
Definition: stream.c:257
struct convolve_data convolve
const char * ast_channel_name(const struct ast_channel *chan)
Structure that is the essence of a bridge technology.
unsigned int num_above_internal_rate
struct ast_rtp_rtcp_feedback feedback
static int remove_all_original_streams(struct ast_stream_topology *dest, const struct ast_stream_topology *source, const struct ast_stream_topology *original)
const char * ast_stream_state2str(enum ast_stream_state state)
Convert the state of a stream into a string.
Definition: stream.c:388
int ast_stream_set_metadata(struct ast_stream *stream, const char *m_key, const char *value)
Set a stream metadata value.
Definition: stream.c:460
unsigned int ast_format_get_sample_rate(const struct ast_format *format)
Get the sample rate of a media format.
Definition: format.c:379
#define ast_frfree(fr)
#define AST_VECTOR_REPLACE(vec, idx, elem)
Replace an element at a specific position in a vector, growing the vector if needed.
Definition: vector.h:284
void ast_bridge_channel_stream_map(struct ast_bridge_channel *bridge_channel)
Maps a channel&#39;s stream topology to and from the bridge.
Data structure associated with a single frame of data.
#define AST_RTP_RTCP_PSFB
Definition: rtp_engine.h:299
static int sfu_topologies_on_leave(struct ast_bridge_channel *leaver, struct ast_bridge_channels_list *participants)
int ast_slinfactory_read(struct ast_slinfactory *sf, short *buf, size_t samples)
Read samples from a slinfactory.
Definition: slinfactory.c:145
void * tech_pvt
Definition: bridge.h:365
struct ast_dsp * ast_dsp_new_with_rate(unsigned int sample_rate)
Allocates a new dsp with a specific internal sample rate used during processing.
Definition: dsp.c:1750
Definition: search.h:40
static force_inline void ast_slinear_saturated_subtract(short *input, short *value)
Definition: utils.h:438
union ast_frame::@263 data
ast_media_type
Types of media.
Definition: codec.h:30
int energy_history[DEFAULT_ENERGY_HISTORY_LEN]
#define AST_LIST_TRAVERSE_SAFE_BEGIN(head, var, field)
Loops safely over (traverses) the entries in a list.
Definition: linkedlists.h:528
enum ast_frame_type frametype
#define ast_mutex_init(pmutex)
Definition: lock.h:184
#define ast_mutex_destroy(a)
Definition: lock.h:186
static void softmix_bridge_leave(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
Function called when a channel leaves the bridge.
static unsigned int analyse_softmix_stats(struct softmix_stats *stats, struct softmix_bridge_data *softmix_data, int binaural_active)
struct ast_format * format
static void softmix_bridge_stream_topology_changed(struct ast_bridge *bridge, struct ast_bridge_channel *bridge_channel)
stream_topology_changed callback
void ast_stream_topology_free(struct ast_stream_topology *topology)
Unreference and destroy a stream topology.
Definition: stream.c:743
const char * ast_stream_get_name(const struct ast_stream *stream)
Get the name of a stream.
Definition: stream.c:309
#define ASTERISK_GPL_KEY
The text the key() function should return.
Definition: module.h:46
#define DEBUG_ATLEAST(level)
Definition: logger.h:441
static struct ast_timer * timer
Definition: chan_iax2.c:360
static int load_module(void)
unsigned int internal_sample_rate
The internal sample rate softmix uses to mix channels.
Definition: bridge.h:293
#define DEFAULT_SOFTMIX_TALKING_THRESHOLD
unsigned int highest_supported_rate
static int softmix_mixing_loop(struct ast_bridge *bridge)
Mixing loop.
struct ast_format * ast_channel_rawwriteformat(struct ast_channel *chan)
static int append_source_streams(struct ast_stream_topology *dest, const char *channel_name, const char *sdp_label, const struct ast_stream_topology *source)
unsigned int maximum_sample_rate
The maximum sample rate softmix uses to mix channels.
Definition: bridge.h:314
unsigned int num_channels
Definition: bridge.h:381
Set when the stream is receiving media only.
Definition: stream.h:90
int ast_bridge_number_video_src(struct ast_bridge *bridge)
Returns the number of video sources currently active in the bridge.
Definition: bridge.c:3931
enum ast_bridge_video_mode_type mode
Definition: bridge.h:160
enum ast_stream_state ast_stream_get_state(const struct ast_stream *stream)
Get the current state of a stream.
Definition: stream.c:373
#define AST_VECTOR_SIZE(vec)
Get the number of elements in a vector.
Definition: vector.h:611
struct softmix_remb_collector * remb_collector
struct ast_format * ast_format_cache_get_slin_by_rate(unsigned int rate)
Retrieve the best signed linear format given a sample rate.
Definition: format_cache.c:520
#define DEFAULT_ENERGY_HISTORY_LEN
ast_test_result_state
Definition: test.h:200
void ast_translator_free_path(struct ast_trans_pvt *tr)
Frees a translator path Frees the given translator path structure.
Definition: translate.c:475
static int softmix_mixing_array_init(struct softmix_mixing_array *mixing_array, unsigned int starting_num_entries, unsigned int binaural_active)
void binaural_mixing(struct ast_bridge *bridge, struct softmix_bridge_data *softmix_data, struct softmix_mixing_array *mixing_array, int16_t *bin_buf, int16_t *ann_buf)
Mixes all binaural audio data contained in the mixing array.
#define ast_mutex_unlock(a)
Definition: lock.h:188
#define SOFTBRIDGE_VIDEO_DEST_LEN
#define SOFTMIX_DATALEN(rate, interval)
Size of the buffer used for sample manipulation.
unsigned int binaural_active
Definition: bridge.h:303
#define SOFTMIX_STAT_INTERVAL
Number of mixing iterations to perform between gathering statistics.
#define AST_RTP_RTCP_FMT_REMB
Definition: rtp_engine.h:309
static void softmix_poke_thread(struct softmix_bridge_data *softmix_data)