fl_utf.c

Go to the documentation of this file.

/*
 * "$Id: fl_utf.c 8214 2011-01-07 17:23:02Z AlbrechtS $"
 *
 * This is the utf.c file from fltk2 adapted for use in my fltk1.1 port
 */
/* Copyright 2006-2010 by Bill Spitzak and others.
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Library General Public
 * License as published by the Free Software Foundation; either
 * version 2 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Library General Public License for more details.
 *
 * You should have received a copy of the GNU Library General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
 * USA.
 *
 * Please report all bugs and problems on the following page:
 *
 *     http://www.fltk.org/str.php
 */

/* Modified to obey rfc3629, which limits unicode to 0-0x10ffff */

#include <FL/fl_utf8.h>
#include <string.h>
#include <stdlib.h>

#if 0

  /* FL_EXPORT int fl_unichar_to_utf8(unsigned int uc, char *text); */
  
  /* FL_EXPORT int fl_utf8_size(unsigned int uc); */
  
#endif /* 0 */
  
#define ERRORS_TO_ISO8859_1 1

#define ERRORS_TO_CP1252 1

#define STRICT_RFC3629 0

#if ERRORS_TO_CP1252
/* Codes 0x80..0x9f from the Microsoft CP1252 character set, translated
 * to Unicode:
 */
static unsigned short cp1252[32] = {
  0x20ac, 0x0081, 0x201a, 0x0192, 0x201e, 0x2026, 0x2020, 0x2021,
  0x02c6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008d, 0x017d, 0x008f,
  0x0090, 0x2018, 0x2019, 0x201c, 0x201d, 0x2022, 0x2013, 0x2014,
  0x02dc, 0x2122, 0x0161, 0x203a, 0x0153, 0x009d, 0x017e, 0x0178
};
#endif

unsigned fl_utf8decode(const char* p, const char* end, int* len)
{
  unsigned char c = *(unsigned char*)p;
  if (c < 0x80) {
    if (len) *len = 1;
    return c;
#if ERRORS_TO_CP1252
  } else if (c < 0xa0) {
    if (len) *len = 1;
    return cp1252[c-0x80];
#endif
  } else if (c < 0xc2) {
    goto FAIL;
  }
  if ( (end && p+1 >= end) || (p[1]&0xc0) != 0x80) goto FAIL;
  if (c < 0xe0) {
    if (len) *len = 2;
    return
      ((p[0] & 0x1f) << 6) +
      ((p[1] & 0x3f));
  } else if (c == 0xe0) {
    if (((unsigned char*)p)[1] < 0xa0) goto FAIL;
    goto UTF8_3;
#if STRICT_RFC3629
  } else if (c == 0xed) {
    /* RFC 3629 says surrogate chars are illegal. */
    if (((unsigned char*)p)[1] >= 0xa0) goto FAIL;
    goto UTF8_3;
  } else if (c == 0xef) {
    /* 0xfffe and 0xffff are also illegal characters */
    if (((unsigned char*)p)[1]==0xbf &&
        ((unsigned char*)p)[2]>=0xbe) goto FAIL;
    goto UTF8_3;
#endif
  } else if (c < 0xf0) {
  UTF8_3:
    if ( (end && p+2 >= end) || (p[2]&0xc0) != 0x80) goto FAIL;
    if (len) *len = 3;
    return
      ((p[0] & 0x0f) << 12) +
      ((p[1] & 0x3f) << 6) +
      ((p[2] & 0x3f));
  } else if (c == 0xf0) {
    if (((unsigned char*)p)[1] < 0x90) goto FAIL;
    goto UTF8_4;
  } else if (c < 0xf4) {
  UTF8_4:
    if ( (end && p+3 >= end) || (p[2]&0xc0) != 0x80 || (p[3]&0xc0) != 0x80) goto FAIL;
    if (len) *len = 4;
#if STRICT_RFC3629
    /* RFC 3629 says all codes ending in fffe or ffff are illegal: */
    if ((p[1]&0xf)==0xf &&
        ((unsigned char*)p)[2] == 0xbf &&
        ((unsigned char*)p)[3] >= 0xbe) goto FAIL;
#endif
    return
      ((p[0] & 0x07) << 18) +
      ((p[1] & 0x3f) << 12) +
      ((p[2] & 0x3f) << 6) +
      ((p[3] & 0x3f));
  } else if (c == 0xf4) {
    if (((unsigned char*)p)[1] > 0x8f) goto FAIL; /* after 0x10ffff */
    goto UTF8_4;
  } else {
  FAIL:
    if (len) *len = 1;
#if ERRORS_TO_ISO8859_1
    return c;
#else
    return 0xfffd; /* Unicode REPLACEMENT CHARACTER */
#endif
  }
}

const char* fl_utf8fwd(const char* p, const char* start, const char* end)
{
  const char* a;
  int len;
  /* if we are not pointing at a continuation character, we are done: */
  if ((*p&0xc0) != 0x80) return p;
  /* search backwards for a 0xc0 starting the character: */
  for (a = p-1; ; --a) {
    if (a < start) return p;
    if (!(a[0]&0x80)) return p;
    if ((a[0]&0x40)) break;
  }
  fl_utf8decode(a,end,&len);
  a += len;
  if (a > p) return a;
  return p;
}

const char* fl_utf8back(const char* p, const char* start, const char* end)
{
  const char* a;
  int len;
  /* if we are not pointing at a continuation character, we are done: */
  if ((*p&0xc0) != 0x80) return p;
  /* search backwards for a 0xc0 starting the character: */
  for (a = p-1; ; --a) {
    if (a < start) return p;
    if (!(a[0]&0x80)) return p;
    if ((a[0]&0x40)) break;
  }
  fl_utf8decode(a,end,&len);
  if (a+len > p) return a;
  return p;
}

int fl_utf8bytes(unsigned ucs) {
  if (ucs < 0x000080U) {
    return 1;
  } else if (ucs < 0x000800U) {
    return 2;
  } else if (ucs < 0x010000U) {
    return 3;
  } else if (ucs <= 0x10ffffU) {
    return 4;
  } else {
    return 3; /* length of the illegal character encoding */
  }
}

int fl_utf8encode(unsigned ucs, char* buf) {
  if (ucs < 0x000080U) {
    buf[0] = ucs;
    return 1;
  } else if (ucs < 0x000800U) {
    buf[0] = 0xc0 | (ucs >> 6);
    buf[1] = 0x80 | (ucs & 0x3F);
    return 2;
  } else if (ucs < 0x010000U) {
    buf[0] = 0xe0 | (ucs >> 12);
    buf[1] = 0x80 | ((ucs >> 6) & 0x3F);
    buf[2] = 0x80 | (ucs & 0x3F);
    return 3;
  } else if (ucs <= 0x0010ffffU) {
    buf[0] = 0xf0 | (ucs >> 18);
    buf[1] = 0x80 | ((ucs >> 12) & 0x3F);
    buf[2] = 0x80 | ((ucs >> 6) & 0x3F);
    buf[3] = 0x80 | (ucs & 0x3F);
    return 4;
  } else {
    /* encode 0xfffd: */
    buf[0] = 0xefU;
    buf[1] = 0xbfU;
    buf[2] = 0xbdU;
    return 3;
  }
}

unsigned fl_utf8toUtf16(const char* src, unsigned srclen,
                  unsigned short* dst, unsigned dstlen)
{
  const char* p = src;
  const char* e = src+srclen;
  unsigned count = 0;
  if (dstlen) for (;;) {
    if (p >= e) {dst[count] = 0; return count;}
    if (!(*p & 0x80)) { /* ascii */
      dst[count] = *p++;
    } else {
      int len; unsigned ucs = fl_utf8decode(p,e,&len);
      p += len;
      if (ucs < 0x10000) {
        dst[count] = ucs;
      } else {
        /* make a surrogate pair: */
        if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
        dst[count] = (((ucs-0x10000u)>>10)&0x3ff) | 0xd800;
        dst[++count] = (ucs&0x3ff) | 0xdc00;
      }
    }
    if (++count == dstlen) {dst[count-1] = 0; break;}
  }
  /* we filled dst, measure the rest: */
  while (p < e) {
    if (!(*p & 0x80)) p++;
    else {
      int len; unsigned ucs = fl_utf8decode(p,e,&len);
      p += len;
      if (ucs >= 0x10000) ++count;
    }
    ++count;
  }
  return count;
}


unsigned fl_utf8towc(const char* src, unsigned srclen,
                  wchar_t* dst, unsigned dstlen)
{
#if defined(WIN32) || defined(__CYGWIN__)
  return fl_utf8toUtf16(src, srclen, (unsigned short*)dst, dstlen);
#else
  const char* p = src;
  const char* e = src+srclen;
  unsigned count = 0;
  if (dstlen) for (;;) {
    if (p >= e) {
      dst[count] = 0;
      return count;
    }
    if (!(*p & 0x80)) { /* ascii */
      dst[count] = *p++;
    } else {
      int len; unsigned ucs = fl_utf8decode(p,e,&len);
      p += len;
      dst[count] = (wchar_t)ucs;
    }
    if (++count == dstlen) {dst[count-1] = 0; break;}
  }
  /* we filled dst, measure the rest: */
  while (p < e) {
    if (!(*p & 0x80)) p++;
    else {
      int len; fl_utf8decode(p,e,&len);
      p += len;
    }
    ++count;
  }
  return count;
#endif
}

unsigned fl_utf8toa(const char* src, unsigned srclen,
                 char* dst, unsigned dstlen)
{
  const char* p = src;
  const char* e = src+srclen;
  unsigned count = 0;
  if (dstlen) for (;;) {
    unsigned char c;
    if (p >= e) {dst[count] = 0; return count;}
    c = *(unsigned char*)p;
    if (c < 0xC2) { /* ascii or bad code */
      dst[count] = c;
      p++;
    } else {
      int len; unsigned ucs = fl_utf8decode(p,e,&len);
      p += len;
      if (ucs < 0x100) dst[count] = ucs;
      else dst[count] = '?';
    }
    if (++count >= dstlen) {dst[count-1] = 0; break;}
  }
  /* we filled dst, measure the rest: */
  while (p < e) {
    if (!(*p & 0x80)) p++;
    else {
      int len;
      fl_utf8decode(p,e,&len);
      p += len;
    }
    ++count;
  }
  return count;
}

unsigned fl_utf8fromwc(char* dst, unsigned dstlen,
                    const wchar_t* src, unsigned srclen) {
  unsigned i = 0;
  unsigned count = 0;
  if (dstlen) for (;;) {
    unsigned ucs;
    if (i >= srclen) {dst[count] = 0; return count;}
    ucs = src[i++];
    if (ucs < 0x80U) {
      dst[count++] = ucs;
      if (count >= dstlen) {dst[count-1] = 0; break;}
    } else if (ucs < 0x800U) { /* 2 bytes */
      if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
      dst[count++] = 0xc0 | (ucs >> 6);
      dst[count++] = 0x80 | (ucs & 0x3F);
#if defined(WIN32) || defined(__CYGWIN__)
    } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen &&
               src[i] >= 0xdc00 && src[i] <= 0xdfff) {
      /* surrogate pair */
      unsigned ucs2 = src[i++];
      ucs = 0x10000U + ((ucs&0x3ff)<<10) + (ucs2&0x3ff);
      /* all surrogate pairs turn into 4-byte utf8 */
#else
    } else if (ucs >= 0x10000) {
      if (ucs > 0x10ffff) {
        ucs = 0xfffd;
        goto J1;
      }
#endif
      if (count+4 >= dstlen) {dst[count] = 0; count += 4; break;}
      dst[count++] = 0xf0 | (ucs >> 18);
      dst[count++] = 0x80 | ((ucs >> 12) & 0x3F);
      dst[count++] = 0x80 | ((ucs >> 6) & 0x3F);
      dst[count++] = 0x80 | (ucs & 0x3F);
    } else {
#if !(defined(WIN32) || defined(__CYGWIN__))
    J1:
#endif
      /* all others are 3 bytes: */
      if (count+3 >= dstlen) {dst[count] = 0; count += 3; break;}
      dst[count++] = 0xe0 | (ucs >> 12);
      dst[count++] = 0x80 | ((ucs >> 6) & 0x3F);
      dst[count++] = 0x80 | (ucs & 0x3F);
    }
  }
  /* we filled dst, measure the rest: */
  while (i < srclen) {
    unsigned ucs = src[i++];
    if (ucs < 0x80U) {
      count++;
    } else if (ucs < 0x800U) { /* 2 bytes */
      count += 2;
#if defined(WIN32) || defined(__CYGWIN__)
    } else if (ucs >= 0xd800 && ucs <= 0xdbff && i < srclen-1 &&
               src[i+1] >= 0xdc00 && src[i+1] <= 0xdfff) {
      /* surrogate pair */
      ++i;
#else
    } else if (ucs >= 0x10000 && ucs <= 0x10ffff) {
#endif
      count += 4;
    } else {
      count += 3;
    }
  }
  return count;
}

unsigned fl_utf8froma(char* dst, unsigned dstlen,
                   const char* src, unsigned srclen) {
  const char* p = src;
  const char* e = src+srclen;
  unsigned count = 0;
  if (dstlen) for (;;) {
    unsigned char ucs;
    if (p >= e) {dst[count] = 0; return count;}
    ucs = *(unsigned char*)p++;
    if (ucs < 0x80U) {
      dst[count++] = ucs;
      if (count >= dstlen) {dst[count-1] = 0; break;}
    } else { /* 2 bytes (note that CP1252 translate could make 3 bytes!) */
      if (count+2 >= dstlen) {dst[count] = 0; count += 2; break;}
      dst[count++] = 0xc0 | (ucs >> 6);
      dst[count++] = 0x80 | (ucs & 0x3F);
    }
  }
  /* we filled dst, measure the rest: */
  while (p < e) {
    unsigned char ucs = *(unsigned char*)p++;
    if (ucs < 0x80U) {
      count++;
    } else {
      count += 2;
    }
  }
  return count;
}

#ifdef WIN32
# include <windows.h>
#endif

int fl_utf8locale(void) {
  static int ret = 2;
  if (ret == 2) {
#ifdef WIN32
    ret = GetACP() == CP_UTF8;
#else
    char* s;
    ret = 1; /* assume UTF-8 if no locale */
    if (((s = getenv("LC_CTYPE")) && *s) ||
        ((s = getenv("LC_ALL"))   && *s) ||
        ((s = getenv("LANG"))     && *s)) {
      ret = (strstr(s,"utf") || strstr(s,"UTF"));
    }
#endif
  }
  return ret;
}

unsigned fl_utf8to_mb(const char* src, unsigned srclen,
                  char* dst, unsigned dstlen)
{
  if (!fl_utf8locale()) {
#ifdef WIN32
    wchar_t lbuf[1024];
    wchar_t* buf = lbuf;
    unsigned length = fl_utf8towc(src, srclen, buf, 1024);
    unsigned ret;
    if (length >= 1024) {
      buf = (wchar_t*)(malloc((length+1)*sizeof(wchar_t)));
      fl_utf8towc(src, srclen, buf, length+1);
    }
    if (dstlen) {
      /* apparently this does not null-terminate, even though msdn
       * documentation claims it does:
       */
      ret =
        WideCharToMultiByte(GetACP(), 0, buf, length, dst, dstlen, 0, 0);
      dst[ret] = 0;
    }
    /* if it overflows or measuring length, get the actual length: */
    if (dstlen==0 || ret >= dstlen-1)
      ret =
        WideCharToMultiByte(GetACP(), 0, buf, length, 0, 0, 0, 0);
    if (buf != lbuf) free((void*)buf);
    return ret;
#else
    wchar_t lbuf[1024];
    wchar_t* buf = lbuf;
    unsigned length = fl_utf8towc(src, srclen, buf, 1024);
    int ret;
    if (length >= 1024) {
      buf = (wchar_t*)(malloc((length+1)*sizeof(wchar_t)));
      fl_utf8towc(src, srclen, buf, length+1);
    }
    if (dstlen) {
      ret = wcstombs(dst, buf, dstlen);
      if (ret >= dstlen-1) ret = wcstombs(0,buf,0);
    } else {
      ret = wcstombs(0,buf,0);
    }
    if (buf != lbuf) free((void*)buf);
    if (ret >= 0) return (unsigned)ret;
    /* on any errors we return the UTF-8 as raw text...*/
#endif
  }
  /* identity transform: */
  if (srclen < dstlen) {
    memcpy(dst, src, srclen);
    dst[srclen] = 0;
  } else {
    // Buffer insufficent or buffer query
  }
  return srclen;
}

unsigned fl_utf8from_mb(char* dst, unsigned dstlen,
                    const char* src, unsigned srclen)
{
  if (!fl_utf8locale()) {
#ifdef WIN32
    wchar_t lbuf[1024];
    wchar_t* buf = lbuf;
    unsigned length;
    unsigned ret;
    length = MultiByteToWideChar(GetACP(), 0, src, srclen, buf, 1024);
    if ((length == 0)&&(GetLastError()==ERROR_INSUFFICIENT_BUFFER)) {
      length = MultiByteToWideChar(GetACP(), 0, src, srclen, 0, 0);
      buf = (wchar_t*)(malloc(length*sizeof(wchar_t)));
      MultiByteToWideChar(GetACP(), 0, src, srclen, buf, length);
    }
    ret = fl_utf8fromwc(dst, dstlen, buf, length);
    if (buf != lbuf) free((void*)buf);
    return ret;
#else
    wchar_t lbuf[1024];
    wchar_t* buf = lbuf;
    int length;
    unsigned ret;
    length = mbstowcs(buf, src, 1024);
    if (length >= 1024) {
      length = mbstowcs(0, src, 0)+1;
      buf = (wchar_t*)(malloc(length*sizeof(wchar_t)));
      mbstowcs(buf, src, length);
    }
    if (length >= 0) {
      ret = fl_utf8fromwc(dst, dstlen, buf, length);
      if (buf != lbuf) free((void*)buf);
      return ret;
    }
    /* errors in conversion return the UTF-8 unchanged */
#endif
  }
  /* identity transform: */
  if (srclen < dstlen) {
    memcpy(dst, src, srclen);
    dst[srclen] = 0;
  } else {
    // Buffer insufficent or buffer query
  }
  return srclen;
}

int fl_utf8test(const char* src, unsigned srclen) {
  int ret = 1;
  const char* p = src;
  const char* e = src+srclen;
  while (p < e) {
    if (*p & 0x80) {
      int len; fl_utf8decode(p,e,&len);
      if (len < 2) return 0;
      if (len > ret) ret = len;
      p += len;
    } else {
      p++;
    }
  }
  return ret;
}

/* forward declare mk_wcwidth() as static so the name is not visible.
 */
 static int mk_wcwidth(unsigned int ucs);

 /* include the c source directly so it's contents are only visible here
  */
#include "xutf8/mk_wcwidth.c"

int fl_wcwidth_(unsigned int ucs) {
  return mk_wcwidth(ucs);
}

int fl_wcwidth(const char* src) {
  int len = fl_utf8len(*src);
  int ret = 0;
  unsigned int ucs = fl_utf8decode(src, src+len, &ret);
  int width = fl_wcwidth_(ucs);
  return width;
}

/*
 * End of "$Id: fl_utf.c 8214 2011-01-07 17:23:02Z AlbrechtS $".
 */