fc_utf8.c

Go to the documentation of this file.

/***********************************************************************
 Freeciv - Copyright (C) 1996 - A Kjeldberg, L Gregersen, P Unold
   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2, or (at your option)
   any later version.
 
   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.
***********************************************************************/
 
#ifdef HAVE_CONFIG_H
#include <fc_config.h>
#endif
 
#include <stdarg.h>
#include <string.h>
 
/* utility */
#include "log.h"
#include "mem.h"
#include "support.h"
 
#include "fc_utf8.h"
 
 
/* The length of a character for external use (at least 1 to avoid infinite
 * loops). See also fc_ut8_next_char(). */
const char fc_utf8_skip[256] = {
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00000000 to 00001111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00010000 to 00011111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00100000 to 00101111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00110000 to 00111111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01000000 to 01001111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01010000 to 01011111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01100000 to 01101111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01110000 to 01111111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10000000 to 10001111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10010000 to 10011111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10100000 to 10101111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 10110000 to 10111111. */
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 11000000 to 11001111. */
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 11010000 to 11011111. */
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* 11100000 to 11101111. */
#ifdef USE_6_BYTES_CHAR
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 1, 1  /* 11110000 to 11111111. */
#else
  4, 4, 4, 4, 4, 4, 4, 4, 1, 1, 1, 1, 1, 1, 1, 1  /* 11110000 to 11111111. */
#endif /* USE_6_BYTES_CHAR */
};
 
/* The length of a character for internal use (0 means an invalid start of
 * a character). */
static const char fc_utf8_char_size[256] = {
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00000000 to 00001111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00010000 to 00011111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00100000 to 00101111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 00110000 to 00111111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01000000 to 01001111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01010000 to 01011111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01100000 to 01101111. */
  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 01110000 to 01111111. */
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10000000 to 10001111. */
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10010000 to 10011111. */
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10100000 to 10101111. */
  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 10110000 to 10111111. */
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 11000000 to 11001111. */
  2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, /* 11010000 to 11011111. */
  3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, /* 11100000 to 11101111. */
#ifdef USE_6_BYTES_CHAR
  4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 0, 0  /* 11110000 to 11111111. */
#else
  4, 4, 4, 4, 4, 4, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0  /* 11110000 to 11111111. */
#endif /* USE_6_BYTES_CHAR */
};
 
#define FC_UTF8_CHAR_SIZE(utf8_char) \
  fc_utf8_char_size[*(unsigned char *) utf8_char]
 
#define FC_UTF8_REP_CHAR "\xef\xbf\xbd" /* U+FFFD. */
 
 
/************************************************************************/
static inline bool base_fc_utf8_char_validate(const char *utf8_char,
                                              char size)
{
  if (1 < size) {
    do {
      utf8_char++;
      if (0x80 != (0xC0 & *(const unsigned char *)utf8_char)) {
        /* Not a valid byte of the sequence. */
        return FALSE;
      }
      size--;
    } while (1 < size);
    return TRUE;
  } else {
    return (1 == size);
  }
}
 
/************************************************************************/
static inline size_t base_fc_utf8_strlcpy_trunc(char *dest, const char *src,
                                                size_t n)
{
  const char *end;
  size_t len;
 
  (void) fc_utf8_validate_len(src, n, &end);
  len = end - src;
  fc_assert(len < n);
  if (0 < len) {
    memcpy(dest, src, len);
  }
  dest[len] = '\0';
  return strlen(src);
}
 
/************************************************************************/
static inline size_t base_fc_utf8_strlcpy_rep(char *dest, const char *src,
                                              size_t n)
{
  const char *end;
  size_t src_len, len;
 
  fc_assert_ret_val(src != nullptr, 0);
 
  src_len = strlen(src);
  while (TRUE) {
    if (fc_utf8_validate_len(src, n, &end)) {
      /* Valid UTF-8. */
      len = end - src;
 
      fc_assert(len < n);
 
      if (0 < len) {
        memcpy(dest, src, len);
      }
      dest[len] = '\0'; /* Valid UTF-8 string part. */
      return src_len;
    } else {
      /* '*end' is not a valid UTF-8 character. */
      len = end - src;
 
      fc_assert(len < n);
 
      if (0 < len) {
        memcpy(dest, src, len);
      }
 
      n -= len;
      dest += len;
 
      /* Try to insert the replacement character. */
      len = sizeof(FC_UTF8_REP_CHAR);
      if (n > len) {
        memcpy(dest, FC_UTF8_REP_CHAR, len);
        n -= len;
        dest += len;
      }
 
      if (1 == n) {
        *dest = '\0';
        return src_len; /* End of 'dest' reached. */
      }
 
      /* Jump to next character in src. */
      src = fc_utf8_find_next_char(end);
      if (src == nullptr || *src == '\0') {
        *dest = '\0';
        return src_len; /* End of 'src' reached. */
      }
    }
  }
  fc_assert(FALSE);     /* Shouldn't occur! */
  return src_len;
}
 
 
/************************************************************************/
bool fc_utf8_char_validate(const char *utf8_char)
{
  return base_fc_utf8_char_validate(utf8_char, FC_UTF8_CHAR_SIZE(utf8_char));
}
 
/************************************************************************/
const char *fc_utf8_find_next_char(const char *utf8_char)
{
  do {
    utf8_char++;
  } while (0 == FC_UTF8_CHAR_SIZE(utf8_char));
 
  return utf8_char;
}
 
/************************************************************************/
const char *fc_utf8_find_prev_char(const char *utf8_char,
                                   const char *utf8_string)
{
  for (utf8_char--; utf8_char > utf8_string; utf8_char--) {
    if (0 != FC_UTF8_CHAR_SIZE(utf8_char)) {
      return utf8_char;
    }
  }
 
  return utf8_string;
}
 
/************************************************************************/
bool fc_utf8_validate(const char *utf8_string, const char **end)
{
  char size;
 
  while ('\0' != *utf8_string) {
    size = FC_UTF8_CHAR_SIZE(utf8_string);
    if (!base_fc_utf8_char_validate(utf8_string, size)) {
      if (end != nullptr) {
        *end = utf8_string;
      }
      return FALSE;
    }
    utf8_string += size;
  }
 
  if (end != nullptr) {
    *end = utf8_string;
  }
 
  return TRUE;
}
 
/************************************************************************/
bool fc_utf8_validate_len(const char *utf8_string, size_t byte_len,
                          const char **end)
{
  unsigned char size;
 
  while ('\0' != *utf8_string) {
    size = FC_UTF8_CHAR_SIZE(utf8_string);
 
    if (!base_fc_utf8_char_validate(utf8_string, size)) {
      if (end != nullptr) {
        *end = utf8_string;
      }
      return FALSE;
    }
 
    if (size > byte_len) {
      if (end != nullptr) {
        *end = utf8_string;
      }
      return FALSE;
    } else {
      byte_len -= size;
    }
 
    utf8_string += size;
  }
 
  if (end != nullptr) {
    *end = utf8_string;
  }
 
  return TRUE;
}
 
/************************************************************************/
char *fc_utf8_validate_trunc(char *utf8_string)
{
  char *end;
 
  if (!fc_utf8_validate(utf8_string, (const char **) &end)) {
    *end = '\0';
  }
 
  return utf8_string;
}
 
/************************************************************************/
char *fc_utf8_validate_trunc_len(char *utf8_string, size_t byte_len)
{
  char *end;
 
  if (!fc_utf8_validate_len(utf8_string, byte_len, (const char **) &end)) {
    *end = '\0';
  }
 
  return utf8_string;
}
 
/************************************************************************/
char *fc_utf8_validate_trunc_dup(const char *utf8_string)
{
  const char *end;
  size_t size;
  char *ret;
 
  (void) fc_utf8_validate(utf8_string, &end);
  size = end - utf8_string;
  ret = fc_malloc(size + 1);    /* Keep a spot for '\0'. */
  memcpy(ret, utf8_string, size);
  ret[size] = '\0';
 
  return ret;
}
 
/************************************************************************/
char *fc_utf8_validate_rep_len(char *utf8_string, size_t byte_len)
{
  if (0 < byte_len) {
    char copy[byte_len];
 
    fc_strlcpy(copy, utf8_string, byte_len);
    base_fc_utf8_strlcpy_rep(utf8_string, copy, byte_len);
  }
 
  return utf8_string;
}
 
/************************************************************************/
char *fc_utf8_validate_rep_dup(const char *utf8_string)
{
  char *ret;
  const char *utf8_char;
  size_t size = 1;      /* '\0'. */
  char char_size;
 
  /* Check needed size. */
  utf8_char = utf8_string;
  while ('\0' != *utf8_char) {
    char_size = FC_UTF8_CHAR_SIZE(utf8_char);
    if (base_fc_utf8_char_validate(utf8_char, char_size)) {
      /* Normal valid character. */
      size += char_size;
      utf8_char += char_size;
    } else {
      /* Replacement character. */
      size += sizeof(FC_UTF8_REP_CHAR);
      /* Find next character. */
      do {
        utf8_char++;
      } while (0 == FC_UTF8_CHAR_SIZE(utf8_char));
    }
  }
 
  /* Do the allocation. */
  ret = fc_malloc(size);
  base_fc_utf8_strlcpy_rep(ret, utf8_string, size);
 
  return ret;
}
 
/************************************************************************/
size_t fc_utf8_strlen(const char *utf8_string)
{
  size_t len;
 
  for (len = 0; '\0' != *utf8_string; len++) {
    utf8_string = fc_ut8_next_char(utf8_string);
  }
 
  return len;
}
 
/************************************************************************/
size_t fc_utf8_strlcpy_trunc(char *dest, const char *src, size_t n)
{
  fc_assert_ret_val(0 < n, -1);
 
  return base_fc_utf8_strlcpy_trunc(dest, src, n);
}
 
/************************************************************************/
size_t fc_utf8_strlcpy_rep(char *dest, const char *src, size_t n)
{
  fc_assert_ret_val(0 < n, -1);
 
  return base_fc_utf8_strlcpy_rep(dest, src, n);
}
 
/************************************************************************/
size_t fc_utf8_strlcat_trunc(char *dest, const char *src, size_t n)
{
  size_t len;
 
  fc_assert_ret_val(0 < n, -1);
 
  len = strlen(dest);
  fc_assert_ret_val(len < n, -1);
 
  return len + base_fc_utf8_strlcpy_trunc(dest + len, src, n - len);
}
 
/************************************************************************/
size_t fc_utf8_strlcat_rep(char *dest, const char *src, size_t n)
{
  size_t len;
 
  fc_assert_ret_val(0 < n, -1);
 
  len = strlen(dest);
  fc_assert_ret_val(len < n, -1);
 
  return len + base_fc_utf8_strlcpy_rep(dest + len, src, n - len);
}
 
/************************************************************************/
int fc_utf8_snprintf_trunc(char *str, size_t n, const char *format, ...)
{
  int ret;
  va_list args;
 
  va_start(args, format);
  ret = fc_utf8_vsnprintf_trunc(str, n, format, args);
  va_end(args);
 
  return ret;
}
 
/************************************************************************/
int fc_utf8_snprintf_rep(char *str, size_t n, const char *format, ...)
{
  int ret;
  va_list args;
 
  va_start(args, format);
  ret = fc_utf8_vsnprintf_rep(str, n, format, args);
  va_end(args);
 
  return ret;
}
 
/************************************************************************/
int fc_utf8_vsnprintf_trunc(char *str, size_t n, const char *format,
                            va_list args)
{
  char *end;
  int ret;
 
  fc_assert_ret_val(0 < n, -1);
 
  ret = fc_vsnprintf(str, n, format, args);
  if (fc_utf8_validate(str, (const char **) &end)) {
    /* Already valid UTF-8. */
    return ret;
  } else {
    /* Truncate at last valid UTF-8 character. */
    *end = '\0';
    return (-1 == ret ? -1 : end - str);
  }
}
 
/************************************************************************/
int fc_utf8_vsnprintf_rep(char *str, size_t n, const char *format,
                          va_list args)
{
  char *end;
  int ret;
 
  fc_assert_ret_val(0 < n, -1);
 
  ret = fc_vsnprintf(str, n, format, args);
  if (fc_utf8_validate(str, (const char **) &end)) {
    /* Already valid UTF-8. */
    return ret;
  } else {
    (void) fc_utf8_validate_rep_len(end, n - (end - str));
    return ((-1 == ret) ? -1 : (int)strlen(str));
  }
}
 
/************************************************************************/
int cat_utf8_snprintf_trunc(char *str, size_t n, const char *format, ...)
{
  size_t len;
  int ret;
  va_list args;
 
  fc_assert_ret_val(0 < n, -1);
 
  len = strlen(str);
  fc_assert_ret_val(len < n, -1);
 
  va_start(args, format);
  ret = fc_utf8_vsnprintf_trunc(str + len, n - len, format, args);
  va_end(args);
 
  return ((-1 == ret) ? -1 : (int)(ret + len));
}
 
/************************************************************************/
int cat_utf8_snprintf_rep(char *str, size_t n, const char *format, ...)
{
  size_t len;
  int ret;
  va_list args;
 
  fc_assert_ret_val(0 < n, -1);
 
  len = strlen(str);
  fc_assert_ret_val(len < n, -1);
 
  va_start(args, format);
  ret = fc_utf8_vsnprintf_rep(str + len, n - len, format, args);
  va_end(args);
 
  return ((-1 == ret) ? -1 : (int)(ret + len));
}