media_index.c

Go to the documentation of this file.

/*
 * Asterisk -- An open source telephony toolkit.
 *
 * Copyright (C) 2013, Digium, Inc.
 *
 * Kinsey Moore <[email protected]>
 *
 * See http://www.asterisk.org for more information about
 * the Asterisk project. Please do not directly contact
 * any of the maintainers of this project for assistance;
 * the project provides a web site, mailing lists and IRC
 * channels for your use.
 *
 * This program is free software, distributed under the terms of
 * the GNU General Public License Version 2. See the LICENSE file
 * at the top of the source tree.
 */

/*! \file
 * \brief Sound file format and description indexer.
 */

#include "asterisk.h"

#include <dirent.h>
#include <sys/stat.h>

#include "asterisk/utils.h"
#include "asterisk/lock.h"
#include "asterisk/format.h"
#include "asterisk/format_cap.h"
#include "asterisk/media_index.h"
#include "asterisk/file.h"

/*** MODULEINFO
   <support_level>core</support_level>
 ***/

/*! \brief The number of buckets to be used for storing variant-keyed objects */
#define VARIANT_BUCKETS 7

/*! \brief The number of buckets to be used for storing media filename-keyed objects */
#define INDEX_BUCKETS 157

/*! \brief Structure to hold a list of the format variations for a media file for a specific variant */
struct media_variant {
   AST_DECLARE_STRING_FIELDS(
      AST_STRING_FIELD(description);   /*!< The description of the media */
   );
   struct ast_format_cap *formats;  /*!< The formats this media is available in for this variant */
   char variant[0];           /*!< The variant this media is available in */
};

static void media_variant_destroy(void *obj)
{
   struct media_variant *variant = obj;

   ast_string_field_free_memory(variant);
   ao2_cleanup(variant->formats);
}

static struct media_variant *media_variant_alloc(const char *variant_str)
{
   size_t str_sz = strlen(variant_str) + 1;
   struct media_variant *variant;

   variant = ao2_alloc_options(sizeof(*variant) + str_sz, media_variant_destroy,
      AO2_ALLOC_OPT_LOCK_NOLOCK);
   if (!variant) {
      return NULL;
   }

   memcpy(variant->variant, variant_str, str_sz);

   variant->formats = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
   if (!variant->formats || ast_string_field_init(variant, 8)) {
      ao2_ref(variant, -1);

      return NULL;
   }

   return variant;
}

static int media_variant_hash(const void *obj, const int flags)
{
   const char *variant = (flags & OBJ_KEY) ? obj : ((struct media_variant*) obj)->variant;
   return ast_str_case_hash(variant);
}

static int media_variant_cmp(void *obj, void *arg, int flags)
{
   struct media_variant *opt1 = obj, *opt2 = arg;
   const char *variant = (flags & OBJ_KEY) ? arg : opt2->variant;
   return strcasecmp(opt1->variant, variant) ? 0 : CMP_MATCH | CMP_STOP;
}

/*! \brief Structure to hold information about a media file */
struct media_info {
   struct ao2_container *variants;  /*!< The variants for which this media is available */
   char name[0];                    /*!< The file name of the media */
};

static void media_info_destroy(void *obj)
{
   struct media_info *info = obj;

   ao2_cleanup(info->variants);
}

static struct media_info *media_info_alloc(const char *name)
{
   size_t name_sz = strlen(name) + 1;
   struct media_info *info;

   info = ao2_alloc_options(sizeof(*info) + name_sz, media_info_destroy,
      AO2_ALLOC_OPT_LOCK_NOLOCK);
   if (!info) {
      return NULL;
   }

   memcpy(info->name, name, name_sz);

   info->variants = ao2_container_alloc_hash(AO2_ALLOC_OPT_LOCK_MUTEX, 0,
      VARIANT_BUCKETS, media_variant_hash, NULL, media_variant_cmp);
   if (!info->variants) {
      ao2_ref(info, -1);

      return NULL;
   }

   return info;
}

static int media_info_hash(const void *obj, const int flags)
{
   const char *name = (flags & OBJ_KEY) ? obj : ((struct media_info*) obj)->name;
   return ast_str_case_hash(name);
}

static int media_info_cmp(void *obj, void *arg, int flags)
{
   struct media_info *opt1 = obj, *opt2 = arg;
   const char *name = (flags & OBJ_KEY) ? arg : opt2->name;
   return strcasecmp(opt1->name, name) ? 0 : CMP_MATCH | CMP_STOP;
}

struct ast_media_index {
   struct ao2_container *index;            /*!< The index of media that has requested */
   struct ao2_container *media_list_cache; /*!< Cache of filenames to prevent them from being regenerated so often */
   char base_dir[0];                       /*!< Base directory for indexing */
};

static void media_index_dtor(void *obj)
{
   struct ast_media_index *index = obj;

   ao2_cleanup(index->index);
   ao2_cleanup(index->media_list_cache);
}

struct ast_media_index *ast_media_index_create(const char *base_dir)
{
   size_t base_dir_sz = strlen(base_dir) + 1;
   struct ast_media_index *index = ao2_alloc(sizeof(*index) + base_dir_sz, media_index_dtor);

   if (!index) {
      return NULL;
   }

   memcpy(index->base_dir, base_dir, base_dir_sz);

   index->index = ao2_container_alloc_hash(AO2_ALLOC_OPT_LOCK_MUTEX, 0, INDEX_BUCKETS,
      media_info_hash, NULL, media_info_cmp);
   if (!index->index) {
      ao2_ref(index, -1);

      return NULL;
   }

   return index;
}

static struct media_variant *find_variant(struct ast_media_index *index, const char *filename, const char *variant)
{
   RAII_VAR(struct media_info *, info, NULL, ao2_cleanup);

   info = ao2_find(index->index, filename, OBJ_KEY);
   if (!info) {
      return NULL;
   }

   return ao2_find(info->variants, variant, OBJ_KEY);
}

/*! \brief create the appropriate media_variant and any necessary structures */
static struct media_variant *alloc_variant(struct ast_media_index *index, const char *filename, const char *variant_str)
{
   struct media_info *info;
   struct media_variant *variant = NULL;

   info = ao2_find(index->index, filename, OBJ_KEY);
   if (!info) {
      /* This is the first time the index has seen this filename,
       * allocate and link */
      info = media_info_alloc(filename);
      if (!info) {
         return NULL;
      }

      ao2_link(index->index, info);
   } else {
      variant = ao2_find(info->variants, variant_str, OBJ_KEY);
   }

   if (!variant) {
      /* This is the first time the index has seen this variant for
       * this filename, allocate and link */
      variant = media_variant_alloc(variant_str);
      if (variant) {
         ao2_link(info->variants, variant);
      }
   }

   ao2_ref(info, -1);

   return variant;
}

const char *ast_media_get_description(struct ast_media_index *index, const char *filename, const char *variant_str)
{
   RAII_VAR(struct media_variant *, variant, NULL, ao2_cleanup);
   if (ast_strlen_zero(filename) || ast_strlen_zero(variant_str)) {
      return NULL;
   }

   variant = find_variant(index, filename, variant_str);
   if (!variant) {
      return NULL;
   }

   return variant->description;
}

struct ast_format_cap *ast_media_get_format_cap(struct ast_media_index *index, const char *filename, const char *variant_str)
{
   struct ast_format_cap *dupcap;
   RAII_VAR(struct media_variant *, variant, NULL, ao2_cleanup);
   if (ast_strlen_zero(filename) || ast_strlen_zero(variant_str)) {
      return NULL;
   }

   variant = find_variant(index, filename, variant_str);
   if (!variant) {
      return NULL;
   }

   dupcap = ast_format_cap_alloc(AST_FORMAT_CAP_FLAG_DEFAULT);
   if (dupcap) {
      ast_format_cap_append_from_cap(dupcap, variant->formats, AST_MEDIA_TYPE_UNKNOWN);
   }
   return dupcap;
}

/*! \brief Add the variant to the list of variants requested */
static int add_variant_cb(void *obj, void *arg, int flags)
{
   struct media_variant *variant = obj;
   struct ao2_container *variants= arg;
   ast_str_container_add(variants, variant->variant);
   return 0;
}

struct ao2_container *ast_media_get_variants(struct ast_media_index *index, const char *filename)
{
   RAII_VAR(struct media_info *, info, NULL, ao2_cleanup);
   RAII_VAR(struct ao2_container *, variants, NULL, ao2_cleanup);
   if (!filename) {
      return NULL;
   }

   variants = ast_str_container_alloc(VARIANT_BUCKETS);
   if (!variants) {
      return NULL;
   }

   info = ao2_find(index->index, filename, OBJ_KEY);
   if (!info) {
      return NULL;
   }

   ao2_callback(info->variants, OBJ_NODATA, add_variant_cb, variants);

   ao2_ref(variants, +1);
   return variants;
}

/*! \brief Add the media_info's filename to the container of filenames requested */
static int add_media_cb(void *obj, void *arg, int flags)
{
   struct media_info *info = obj;
   struct ao2_container *media = arg;
   ast_str_container_add(media, info->name);
   return 0;
}

struct ao2_container *ast_media_get_media(struct ast_media_index *index)
{
   RAII_VAR(struct ao2_container *, media, NULL, ao2_cleanup);

   if (!index->media_list_cache) {
      media = ast_str_container_alloc(INDEX_BUCKETS);
      if (!media) {
         return NULL;
      }

      ao2_callback(index->index, OBJ_NODATA, add_media_cb, media);

      /* Ref to the cache */
      ao2_ref(media, +1);
      index->media_list_cache = media;
   }

   /* Ref to the caller */
   ao2_ref(index->media_list_cache, +1);
   return index->media_list_cache;
}

/*! \brief Update an index with new format/variant information */
static int update_file_format_info(struct ast_media_index *index, const char *filename, const char *variant_str, struct ast_format *file_format)
{
   struct media_variant *variant;

   variant = alloc_variant(index, filename, variant_str);
   if (!variant) {
      return -1;
   }

   ast_format_cap_append(variant->formats, file_format, 0);
   ao2_ref(variant, -1);

   return 0;
}

/*! \brief Process a media file into the index */
static int process_media_file(struct ast_media_index *index, const char *variant, const char *subdir, const char *filename_stripped, const char *ext)
{
   struct ast_format *file_format;
   const char *file_identifier = filename_stripped;
   char *file_id_str = NULL;
   int res;

   file_format = ast_get_format_for_file_ext(ext);
   if (!file_format) {
      /* extension not registered */
      return 0;
   }

   /* handle updating the file information */
   if (subdir) {
      if (ast_asprintf(&file_id_str, "%s/%s", subdir, filename_stripped) == -1) {
         return -1;
      }

      file_identifier = file_id_str;
   }

   res = update_file_format_info(index, file_identifier, variant, file_format);
   ast_free(file_id_str);

   return res;
}

/*!
 * \brief Process a media description text file
 *
 * This currently processes core-sounds-*.txt and extra-sounds-*.txt, but will
 * process others if present.
 */
static int process_description_file(struct ast_media_index *index,
   const char *subdir,
   const char *variant_str,
   const char *filename,
   const char *match_filename)
{
   RAII_VAR(struct ast_str *, description_file_path, ast_str_create(64), ast_free);
   RAII_VAR(struct ast_str *, cumulative_description, ast_str_create(64), ast_free);
   char *file_id_persist = NULL;
   int res = 0;
   FILE *f = NULL;
#if defined(LOW_MEMORY)
   char buf[256];
#else
   char buf[2048];
#endif

   if (!description_file_path || !cumulative_description) {
      return -1;
   }

   if (ast_strlen_zero(subdir)) {
      ast_str_set(&description_file_path, 0, "%s/%s/%s", index->base_dir, variant_str, filename);
   } else {
      ast_str_set(&description_file_path, 0, "%s/%s/%s/%s", index->base_dir, variant_str, subdir, filename);
   }
   f = fopen(ast_str_buffer(description_file_path), "r");
   if (!f) {
      ast_log(LOG_WARNING, "Could not open media description file '%s': %s\n", ast_str_buffer(description_file_path), strerror(errno));
      return -1;
   }

   while (!feof(f)) {
      char *file_identifier, *description;
      if (!fgets(buf, sizeof(buf), f)) {
         if (ferror(f)) {
            ast_log(LOG_ERROR, "Error reading from file %s: %s\n", ast_str_buffer(description_file_path), strerror(errno));
         }
         continue;
      }

      /* Skip lines that are too long */
      if (strlen(buf) == sizeof(buf) - 1 && buf[sizeof(buf) - 1] != '\n') {
         ast_log(LOG_WARNING, "Line too long, skipping. It begins with: %.32s...\n", buf);
         while (fgets(buf, sizeof(buf), f)) {
            if (strlen(buf) != sizeof(buf) - 1 || buf[sizeof(buf) - 1] == '\n') {
               break;
            }
         }
         if (ferror(f)) {
            ast_log(LOG_ERROR, "Error reading from file %s: %s\n", ast_str_buffer(description_file_path), strerror(errno));
         }
         continue;
      }

      if (buf[0] == ';') {
         /* ignore comments */
         continue;
      }

      ast_trim_blanks(buf);
      description = buf;
      file_identifier = strsep(&description, ":");
      if (!description) {
         /* no ':' means this is a continuation */
         if (file_id_persist) {
            ast_str_append(&cumulative_description, 0, "\n%s", file_identifier);
         }
         continue;
      } else {
         /* if there's text in cumulative_description, archive it and start anew */
         if (file_id_persist && !ast_strlen_zero(ast_str_buffer(cumulative_description))) {
            struct media_variant *variant;

            /*
             * If we were only searching for a specific sound filename
             * don't include others.
             */
            if (ast_strlen_zero(match_filename) || strcmp(match_filename, file_id_persist) == 0) {
               variant = alloc_variant(index, file_id_persist, variant_str);
               if (!variant) {
                  res = -1;
                  break;
               }
               ast_string_field_set(variant, description, ast_str_buffer(cumulative_description));
               ao2_ref(variant, -1);
            }

            ast_str_reset(cumulative_description);
         }

         ast_free(file_id_persist);
         file_id_persist = ast_strdup(file_identifier);
         description = ast_skip_blanks(description);
         ast_str_set(&cumulative_description, 0, "%s", description);
      }
   }

   /* handle the last one */
   if (file_id_persist && !ast_strlen_zero(ast_str_buffer(cumulative_description))) {
      struct media_variant *variant;

      /*
       * If we were only searching for a specific sound filename
       * don't include others.
       */
      if (ast_strlen_zero(match_filename) || strcmp(match_filename, file_id_persist) == 0) {
         variant = alloc_variant(index, file_id_persist, variant_str);
         if (variant) {
            ast_string_field_set(variant, description, ast_str_buffer(cumulative_description));
            ao2_ref(variant, -1);
         } else {
            res = -1;
         }
      }
   }

   ast_free(file_id_persist);
   fclose(f);
   return res;
}

struct read_dirs_data {
   const char *search_filename;
   size_t search_filename_len;
   const char *search_variant;
   struct ast_media_index *index;
   size_t dirname_len;
};

static int read_dirs_cb(const char *dir_name, const char *filename, void *obj)
{
   struct read_dirs_data *data = obj;
   char *ext;
   size_t match_len;
   char *match;
   size_t match_base_len;
   char *subdirs = (char *)dir_name + data->dirname_len;

   /*
    * Example:
    * From the filesystem:
    *      index's base_dir = "/var/lib/asterisk/sounds"
    *      search_variant = "en"
    *      search directory base = "/var/lib/asterisk/sounds/en"
    *      dirname_len = 27
    *    current dir_name = "/var/lib/asterisk/sounds/en/digits"
    *    subdirs =                                     "/digits"
    *    filename = "1.ulaw"
    *
    * From the search criteria:
    *    search_filename = "digits/1"
    *    search_filename_len = 8
    */

   if (*subdirs == '/') {
      subdirs++;
   }

   /* subdirs = "digits" */

   match_len = strlen(subdirs) + strlen(filename) + 2;
   match = ast_alloca(match_len);
   snprintf(match, match_len, "%s%s%s", subdirs,
      ast_strlen_zero(subdirs) ? "" : "/", filename);

   /* match = discovered filename relative to language = "digits/1.ulaw" */

   ext = strrchr(match, '.');
   if (!ext) {
      return 0;
   }

   /* ext = ".ulaw" */

   if (data->search_filename_len > 0) {
      match_base_len = ext - match;
      /*
       * match_base_len = length of "digits/1" = 8 which
       * happens to match the length of search_filename.
       * However if the discovered filename was 11.ulaw
       * it would be length of "digits/11" = 9.
       * We need to use the larger during the compare to
       * make sure we don't match just search_filename
       * as a substring of the discovered filename.
       */
      if (data->search_filename_len > match_base_len) {
         match_base_len = data->search_filename_len;
      }
   }

   /* We always process txt files because they should contain description. */
   if (strcmp(ext, ".txt") == 0) {
      if (process_description_file(data->index, NULL, data->search_variant,
         match, data->search_filename)) {
         return -1;
      }
   } else if (data->search_filename_len == 0
      || strncmp(data->search_filename, match, match_base_len  ) == 0) {
      *ext = '\0';
      ext++;
      process_media_file(data->index, data->search_variant, NULL, match, ext);
   }

   return 0;
}

int ast_media_index_update_for_file(struct ast_media_index *index,
   const char *variant, const char *filename)
{
   struct timeval start;
   struct timeval end;
   int64_t elapsed;
   int rc;
   size_t dirname_len = strlen(index->base_dir) + strlen(S_OR(variant, "")) + 1;
   struct read_dirs_data data = {
      .search_filename = S_OR(filename, ""),
      .search_filename_len = strlen(S_OR(filename, "")),
      .search_variant = S_OR(variant, ""),
      .index = index,
      .dirname_len = dirname_len,
   };
   char *search_dir = ast_alloca(dirname_len + 1);

   sprintf(search_dir, "%s%s%s", index->base_dir, ast_strlen_zero(variant) ? "" : "/",
      data.search_variant);

   gettimeofday(&start, NULL);
   rc = ast_file_read_dirs(search_dir, read_dirs_cb, &data, -1);
   gettimeofday(&end, NULL);
   elapsed = ast_tvdiff_us(end, start);
   ast_debug(1, "Media for language '%s' indexed in %8.6f seconds\n", data.search_variant, elapsed / 1E6);

   return rc;
}