utf8.c File Reference

UTF-8 information and validation functions. More...

#include "asterisk.h"
#include "asterisk/utils.h"
#include "asterisk/utf8.h"
#include "asterisk/test.h"

Include dependency graph for utf8.c:

Go to the source code of this file.

Data Structures
struct	ast_utf8_validator

Macros
#define	UTF8_ACCEPT   0
#define	UTF8_REJECT   12

Functions
	AST_TEST_DEFINE (test_utf8_is_valid)
	AST_TEST_DEFINE (test_utf8_copy_string)
	AST_TEST_DEFINE (test_utf8_validator)
void	ast_utf8_copy_string (char *dst, const char *src, size_t size)
	Copy a string safely ensuring valid UTF-8. More...
int	ast_utf8_init (void)
	Register UTF-8 tests. More...
int	ast_utf8_is_valid (const char *src)
	Check if a zero-terminated string is valid UTF-8. More...
int	ast_utf8_is_validn (const char *src, size_t size)
	Check if the first size bytes of a string are valid UTF-8. More...
void	ast_utf8_validator_destroy (struct ast_utf8_validator *validator)
	Destroy a UTF-8 validator. More...
enum ast_utf8_validation_result	ast_utf8_validator_feed (struct ast_utf8_validator *validator, const char *data)
	Feed a zero-terminated string into the UTF-8 validator. More...
enum ast_utf8_validation_result	ast_utf8_validator_feedn (struct ast_utf8_validator *validator, const char *data, size_t size)
	Feed a string into the UTF-8 validator. More...
int	ast_utf8_validator_new (struct ast_utf8_validator **validator)
	Create a new UTF-8 validator. More...
void	ast_utf8_validator_reset (struct ast_utf8_validator *validator)
	Reset the state of a UTF-8 validator. More...
enum ast_utf8_validation_result	ast_utf8_validator_state (struct ast_utf8_validator *validator)
	Get the current UTF-8 validator state. More...
static uint32_t	decode (uint32_t *state, uint32_t byte)
static int	test_copy_and_compare (const char *src, size_t dst_len, const char *cmp)
static void	test_utf8_shutdown (void)

Variables
static const uint8_t	utf8d []

Detailed Description

UTF-8 information and validation functions.

Definition in file utf8.c.

Macro Definition Documentation

UTF8_ACCEPT

#define UTF8_ACCEPT   0

Definition at line 60 of file utf8.c.

Referenced by ast_utf8_copy_string(), ast_utf8_is_valid(), ast_utf8_is_validn(), ast_utf8_validator_new(), ast_utf8_validator_reset(), and ast_utf8_validator_state().

UTF8_REJECT

#define UTF8_REJECT   12

Definition at line 61 of file utf8.c.

Referenced by ast_utf8_copy_string(), and ast_utf8_validator_state().

Function Documentation

AST_TEST_DEFINE() [1/3]

AST_TEST_DEFINE

(

test_utf8_is_valid

)

Definition at line 222 of file utf8.c.

References AST_TEST_NOT_RUN, AST_TEST_PASS, ast_utf8_is_valid(), ast_utf8_is_validn(), sip_to_pjsip::info(), TEST_EXECUTE, and TEST_INIT.

{
   switch (cmd) {
   case TEST_INIT:
      info->name = "is_valid";
      info->category = "/main/utf8/";
      info->summary = "Test ast_utf8_is_valid and ast_utf8_is_validn";
      info->description =
         "Tests UTF-8 string validation code.";
      return AST_TEST_NOT_RUN;
   case TEST_EXECUTE:
      break;
   }

   /* Valid UTF-8 */
   ast_test_validate(test, ast_utf8_is_valid("Asterisk"));
   ast_test_validate(test, ast_utf8_is_valid("\xce\xbb"));
   ast_test_validate(test, ast_utf8_is_valid("\xe2\x8a\x9b"));
   ast_test_validate(test, ast_utf8_is_valid("\xf0\x9f\x93\x9e"));

   /* Valid with leading */
   ast_test_validate(test, ast_utf8_is_valid("aaa Asterisk"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xce\xbb"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xe2\x8a\x9b"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xf0\x9f\x93\x9e"));

   /* Valid with trailing */
   ast_test_validate(test, ast_utf8_is_valid("Asterisk aaa"));
   ast_test_validate(test, ast_utf8_is_valid("\xce\xbb aaa"));
   ast_test_validate(test, ast_utf8_is_valid("\xe2\x8a\x9b aaa"));
   ast_test_validate(test, ast_utf8_is_valid("\xf0\x9f\x93\x9e aaa"));

   /* Valid with leading and trailing */
   ast_test_validate(test, ast_utf8_is_valid("aaa Asterisk aaa"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xce\xbb aaa"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xe2\x8a\x9b aaa"));
   ast_test_validate(test, ast_utf8_is_valid("aaa \xf0\x9f\x93\x9e aaa"));

   /* Valid if limited by number of bytes */
   ast_test_validate(test, ast_utf8_is_validn("Asterisk" "\xff", strlen("Asterisk")));
   ast_test_validate(test, ast_utf8_is_validn("\xce\xbb" "\xff", strlen("\xce\xbb")));
   ast_test_validate(test, ast_utf8_is_validn("\xe2\x8a\x9b" "\xff", strlen("\xe2\x8a\x9b")));
   ast_test_validate(test, ast_utf8_is_validn("\xf0\x9f\x93\x9e" "\xff", strlen("\xf0\x9f\x93\x9e")));

   /* Invalid */
   ast_test_validate(test, !ast_utf8_is_valid("\xc0\x8a")); /* Overlong */
   ast_test_validate(test, !ast_utf8_is_valid("98.6\xa7")); /* 'High ASCII' */
   ast_test_validate(test, !ast_utf8_is_valid("\xc3\x28"));
   ast_test_validate(test, !ast_utf8_is_valid("\xa0\xa1"));
   ast_test_validate(test, !ast_utf8_is_valid("\xe2\x28\xa1"));
   ast_test_validate(test, !ast_utf8_is_valid("\xe2\x82\x28"));
   ast_test_validate(test, !ast_utf8_is_valid("\xf0\x28\x8c\xbc"));
   ast_test_validate(test, !ast_utf8_is_valid("\xf0\x90\x28\xbc"));
   ast_test_validate(test, !ast_utf8_is_valid("\xf0\x28\x8c\x28"));

   return AST_TEST_PASS;
}

AST_TEST_DEFINE() [2/3]

AST_TEST_DEFINE

(

test_utf8_copy_string

)

Definition at line 287 of file utf8.c.

References AST_TEST_NOT_RUN, AST_TEST_PASS, sip_to_pjsip::info(), test_copy_and_compare(), TEST_EXECUTE, and TEST_INIT.

{
   switch (cmd) {
   case TEST_INIT:
      info->name = "copy_string";
      info->category = "/main/utf8/";
      info->summary = "Test ast_utf8_copy_string";
      info->description =
         "Tests UTF-8 string copying code.";
      return AST_TEST_NOT_RUN;
   case TEST_EXECUTE:
      break;
   }

   ast_test_validate(test, test_copy_and_compare("Asterisk",           6, "Aster"));
   ast_test_validate(test, test_copy_and_compare("Asterisk \xc2\xae", 11, "Asterisk "));
   ast_test_validate(test, test_copy_and_compare("Asterisk \xc2\xae", 12, "Asterisk \xc2\xae"));
   ast_test_validate(test, test_copy_and_compare("Asterisk \xc0\x8a", 12, "Asterisk "));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 1, ""));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 2, ""));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 3, "\xce\xbb"));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 4, "\xce\xbb "));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 5, "\xce\xbb x"));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 6, "\xce\xbb xy"));
   ast_test_validate(test, test_copy_and_compare("\xce\xbb xyz", 7, "\xce\xbb xyz"));

   return AST_TEST_PASS;
}

AST_TEST_DEFINE() [3/3]

AST_TEST_DEFINE

(

test_utf8_validator

)

Definition at line 316 of file utf8.c.

References AST_TEST_FAIL, AST_TEST_NOT_RUN, AST_TEST_PASS, AST_UTF8_INVALID, AST_UTF8_UNKNOWN, AST_UTF8_VALID, ast_utf8_validator_destroy(), ast_utf8_validator_feed(), ast_utf8_validator_new(), sip_to_pjsip::info(), TEST_EXECUTE, and TEST_INIT.

{
   struct ast_utf8_validator *validator;

   switch (cmd) {
   case TEST_INIT:
      info->name = "utf8_validator";
      info->category = "/main/utf8/";
      info->summary = "Test ast_utf8_validator";
      info->description =
         "Tests UTF-8 progressive validator code.";
      return AST_TEST_NOT_RUN;
   case TEST_EXECUTE:
      break;
   }

   if (ast_utf8_validator_new(&validator)) {
      return AST_TEST_FAIL;
   }

   ast_test_validate(test, ast_utf8_validator_feed(validator, "Asterisk") == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\xc2")     == AST_UTF8_UNKNOWN);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\xae")     == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "Private")  == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "Branch")   == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "Exchange") == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\xe2")     == AST_UTF8_UNKNOWN);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\x84")     == AST_UTF8_UNKNOWN);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\xbb")     == AST_UTF8_VALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "\xc0\x8a") == AST_UTF8_INVALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "valid")    == AST_UTF8_INVALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "valid")    == AST_UTF8_INVALID);
   ast_test_validate(test, ast_utf8_validator_feed(validator, "valid")    == AST_UTF8_INVALID);

   ast_utf8_validator_destroy(validator);

   return AST_TEST_PASS;
}

ast_utf8_copy_string()

void ast_utf8_copy_string	(	char *	dst,
		const char *	src,
		size_t	size
	)

Copy a string safely ensuring valid UTF-8.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

This is similar to ast_copy_string, but it will only copy valid UTF-8 sequences from the source string into the destination buffer. If an invalid UTF-8 sequence is encountered, or the available space in the destination buffer is exhausted in the middle of an otherwise valid UTF-8 sequence, the destination buffer will be truncated to ensure that it only contains valid UTF-8.

Parameters

dst	The destination buffer.
src	The source string
size	The size of the destination buffer

Returns

Nothing.

Definition at line 133 of file utf8.c.

References ast_assert, decode(), UTF8_ACCEPT, and UTF8_REJECT.

Referenced by test_copy_and_compare().

{
   uint32_t state = UTF8_ACCEPT;
   char *last_good = dst;

   ast_assert(size > 0);

   while (size && *src) {
      if (decode(&state, (uint8_t) *src) == UTF8_REJECT) {
         /* We _could_ replace with U+FFFD and try to recover, but for now
          * we treat this the same as if we had run out of space */
         break;
      }

      *dst++ = *src++;
      size--;

      if (size && state == UTF8_ACCEPT) {
         /* last_good is where we will ultimately write the 0 byte */
         last_good = dst;
      }
   }

   *last_good = '\0';
}

ast_utf8_init()

int ast_utf8_init

(

void

)

Register UTF-8 tests.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Does nothing unless TEST_FRAMEWORK is defined.

Returns

Always returns 0

Definition at line 362 of file utf8.c.

References ast_register_cleanup(), AST_TEST_REGISTER, and test_utf8_shutdown().

Referenced by asterisk_daemon().

{
   AST_TEST_REGISTER(test_utf8_is_valid);
   AST_TEST_REGISTER(test_utf8_copy_string);
   AST_TEST_REGISTER(test_utf8_validator);

   ast_register_cleanup(test_utf8_shutdown);

   return 0;
}

ast_utf8_is_valid()

int ast_utf8_is_valid

(

const char *

str

)

Check if a zero-terminated string is valid UTF-8.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Parameters

str	The zero-terminated string to check

Return values

0	if the string is not valid UTF-8
Non-zero	if the string is valid UTF-8

Definition at line 110 of file utf8.c.

References decode(), and UTF8_ACCEPT.

Referenced by AST_TEST_DEFINE().

{
   uint32_t state = UTF8_ACCEPT;

   while (*src) {
      decode(&state, (uint8_t) *src++);
   }

   return state == UTF8_ACCEPT;
}

ast_utf8_is_validn()

int ast_utf8_is_validn	(	const char *	str,
		size_t	size
	)

Check if the first size bytes of a string are valid UTF-8.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Similar to ast_utf8_is_valid() but checks the first size bytes or until a zero byte is reached, whichever comes first.

Parameters

str	The string to check
size	The number of bytes to evaluate

Return values

0	if the string is not valid UTF-8
Non-zero	if the string is valid UTF-8

Definition at line 121 of file utf8.c.

References decode(), and UTF8_ACCEPT.

Referenced by AST_TEST_DEFINE().

{
   uint32_t state = UTF8_ACCEPT;

   while (size && *src) {
      decode(&state, (uint8_t) *src++);
      size--;
   }

   return state == UTF8_ACCEPT;
}

ast_utf8_validator_destroy()

void ast_utf8_validator_destroy

(

struct ast_utf8_validator *

validator

)

Destroy a UTF-8 validator.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Parameters

validator

The validator instance to destroy

Definition at line 215 of file utf8.c.

References ast_free.

Referenced by AST_TEST_DEFINE().

{
   ast_free(validator);
}

ast_utf8_validator_feed()

enum ast_utf8_validation_result ast_utf8_validator_feed	(	struct ast_utf8_validator *	validator,
		const char *	data
	)

Feed a zero-terminated string into the UTF-8 validator.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Parameters

validator	The validator instance
data	The zero-terminated string to feed into the validator

Returns

The ast_utf8_validation_result indicating the current state of the validator.

Definition at line 189 of file utf8.c.

References ast_utf8_validator_state(), decode(), and ast_utf8_validator::state.

Referenced by AST_TEST_DEFINE().

{
   while (*data) {
      decode(&validator->state, (uint8_t) *data++);
   }

   return ast_utf8_validator_state(validator);
}

ast_utf8_validator_feedn()

enum ast_utf8_validation_result ast_utf8_validator_feedn	(	struct ast_utf8_validator *	validator,
		const char *	data,
		size_t	size
	)

Feed a string into the UTF-8 validator.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Similar to ast_utf8_validator_feed but will stop feeding in data if a zero byte is encountered or size bytes have been read.

Parameters

validator	The validator instance
data	The string to feed into the validator
size	The number of bytes to feed into the validator

Returns

The ast_utf8_validation_result indicating the current state of the validator.

Definition at line 199 of file utf8.c.

References ast_utf8_validator_state(), decode(), and ast_utf8_validator::state.

{
   while (size && *data) {
      decode(&validator->state, (uint8_t) *data++);
      size--;
   }

   return ast_utf8_validator_state(validator);
}

ast_utf8_validator_new()

int ast_utf8_validator_new

(

struct ast_utf8_validator **

validator

)

Create a new UTF-8 validator.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Parameters

[out]

validator

The validator instance

Return values

0	on success
-1	on failure

Definition at line 163 of file utf8.c.

References ast_malloc, ast_utf8_validator::state, tmp(), and UTF8_ACCEPT.

Referenced by AST_TEST_DEFINE().

{
   struct ast_utf8_validator *tmp = ast_malloc(sizeof(*tmp));

   if (!tmp) {
      return 1;
   }

   tmp->state = UTF8_ACCEPT;
   *validator = tmp;
   return 0;
}

ast_utf8_validator_reset()

void ast_utf8_validator_reset

(

struct ast_utf8_validator *

validator

)

Reset the state of a UTF-8 validator.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Resets the provided UTF-8 validator to its initial state so that it can be reused.

Parameters

validator

The validator instance to reset

Definition at line 210 of file utf8.c.

References ast_utf8_validator::state, and UTF8_ACCEPT.

{
   validator->state = UTF8_ACCEPT;
}

ast_utf8_validator_state()

enum ast_utf8_validation_result ast_utf8_validator_state

(

struct ast_utf8_validator *

validator

)

Get the current UTF-8 validator state.

Since

13.36.0, 16.13.0, 17.7.0, 18.0.0

Parameters

validator

The validator instance

Returns

The ast_utf8_validation_result indicating the current state of the validator.

Definition at line 176 of file utf8.c.

References AST_UTF8_INVALID, AST_UTF8_UNKNOWN, AST_UTF8_VALID, ast_utf8_validator::state, UTF8_ACCEPT, and UTF8_REJECT.

Referenced by ast_utf8_validator_feed(), and ast_utf8_validator_feedn().

{
   switch (validator->state) {
   case UTF8_ACCEPT:
      return AST_UTF8_VALID;
   case UTF8_REJECT:
      return AST_UTF8_INVALID;
   default:
      return AST_UTF8_UNKNOWN;
   }
}

decode()

static uint32_t decode ( uint32_t *  state, uint32_t  byte  )

inlinestatic

Definition at line 98 of file utf8.c.

References state, type, and utf8d.

Referenced by ast_utf8_copy_string(), ast_utf8_is_valid(), ast_utf8_is_validn(), ast_utf8_validator_feed(), and ast_utf8_validator_feedn().

                                                              {
   uint32_t type = utf8d[byte];
   *state = utf8d[256 + *state + type];
   return *state;
}

test_copy_and_compare()

static int test_copy_and_compare ( const char *  src, size_t  dst_len, const char *  cmp  )

static

Definition at line 280 of file utf8.c.

References ast_utf8_copy_string().

Referenced by AST_TEST_DEFINE().

{
   char dst[dst_len];
   ast_utf8_copy_string(dst, src, dst_len);
   return strcmp(dst, cmp) == 0;
}

test_utf8_shutdown()

static void test_utf8_shutdown ( void  )

static

Definition at line 355 of file utf8.c.

References AST_TEST_UNREGISTER.

Referenced by ast_utf8_init().

{
   AST_TEST_UNREGISTER(test_utf8_is_valid);
   AST_TEST_UNREGISTER(test_utf8_copy_string);
   AST_TEST_UNREGISTER(test_utf8_validator);
}

Variable Documentation

utf8d

const uint8_t utf8d[]

static

Definition at line 63 of file utf8.c.

Referenced by decode().