physfs_unicode.c
changeset 787 81da113b878f
child 808 7656f86a9a2c
equal deleted inserted replaced
786:ff536b53a9a8 787:81da113b878f
       
     1 #if HAVE_CONFIG_H
       
     2 #  include <config.h>
       
     3 #endif
       
     4 
       
     5 #include "physfs.h"
       
     6 
       
     7 #define __PHYSICSFS_INTERNAL__
       
     8 #include "physfs_internal.h"
       
     9 
       
    10 
       
    11 /*
       
    12  * From rfc3629, the UTF-8 spec:
       
    13  *  http://www.ietf.org/rfc/rfc3629.txt
       
    14  *
       
    15  *   Char. number range  |        UTF-8 octet sequence
       
    16  *      (hexadecimal)    |              (binary)
       
    17  *   --------------------+---------------------------------------------
       
    18  *   0000 0000-0000 007F | 0xxxxxxx
       
    19  *   0000 0080-0000 07FF | 110xxxxx 10xxxxxx
       
    20  *   0000 0800-0000 FFFF | 1110xxxx 10xxxxxx 10xxxxxx
       
    21  *   0001 0000-0010 FFFF | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
       
    22  */
       
    23 
       
    24 
       
    25 /*
       
    26  * This may not be the best value, but it's one that isn't represented
       
    27  *  in Unicode (0x10FFFF is the largest codepoint value). We return this
       
    28  *  value from utf8codepoint() if there's bogus bits in the
       
    29  *  stream. utf8codepoint() will turn this value into something
       
    30  *  reasonable (like a question mark), for text that wants to try to recover,
       
    31  *  whereas utf8valid() will use the value to determine if a string has bad
       
    32  *  bits.
       
    33  */
       
    34 #define UNICODE_BOGUS_CHAR_VALUE 0xFFFFFFFF
       
    35 
       
    36 /*
       
    37  * This is the codepoint we currently return when there was bogus bits in a
       
    38  *  UTF-8 string. May not fly in Asian locales?
       
    39  */
       
    40 #define UNICODE_BOGUS_CHAR_CODEPOINT '?'
       
    41 
       
    42 static PHYSFS_uint32 utf8codepoint(const char **_str)
       
    43 {
       
    44     const char *str = *_str;
       
    45     PHYSFS_uint32 retval = 0;
       
    46     PHYSFS_uint32 octet = (PHYSFS_uint32) ((PHYSFS_uint8) *str);
       
    47     PHYSFS_uint32 octet2, octet3, octet4;
       
    48 
       
    49     if (octet == 0)  /* null terminator, end of string. */
       
    50         return 0;
       
    51 
       
    52     else if (octet < 128)  /* one octet char: 0 to 127 */
       
    53     {
       
    54         (*_str)++;  /* skip to next possible start of codepoint. */
       
    55         return(octet);
       
    56     } /* else if */
       
    57 
       
    58     else if ((octet > 127) && (octet < 192))  /* bad (starts with 10xxxxxx). */
       
    59     {
       
    60         /*
       
    61          * Apparently each of these is supposed to be flagged as a bogus
       
    62          *  char, instead of just resyncing to the next valid codepoint.
       
    63          */
       
    64         (*_str)++;  /* skip to next possible start of codepoint. */
       
    65         return UNICODE_BOGUS_CHAR_VALUE;
       
    66     } /* else if */
       
    67 
       
    68     else if (octet < 224)  /* two octets */
       
    69     {
       
    70         octet -= (128+64);
       
    71         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
    72         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
    73             return UNICODE_BOGUS_CHAR_VALUE;
       
    74 
       
    75         *_str += 2;  /* skip to next possible start of codepoint. */
       
    76         retval = ((octet << 6) | (octet2 - 128));
       
    77         if ((retval >= 0x80) && (retval <= 0x7FF))
       
    78             return retval;
       
    79     } /* else if */
       
    80 
       
    81     else if (octet < 240)  /* three octets */
       
    82     {
       
    83         octet -= (128+64+32);
       
    84         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
    85         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
    86             return UNICODE_BOGUS_CHAR_VALUE;
       
    87 
       
    88         octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
    89         if ((octet3 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
    90             return UNICODE_BOGUS_CHAR_VALUE;
       
    91 
       
    92         *_str += 3;  /* skip to next possible start of codepoint. */
       
    93         retval = ( ((octet << 12)) | ((octet2-128) << 6) | ((octet3-128)) );
       
    94 
       
    95         /* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */
       
    96         switch (retval)
       
    97         {
       
    98             case 0xD800:
       
    99             case 0xDB7F:
       
   100             case 0xDB80:
       
   101             case 0xDBFF:
       
   102             case 0xDC00:
       
   103             case 0xDF80:
       
   104             case 0xDFFF:
       
   105                 return UNICODE_BOGUS_CHAR_VALUE;
       
   106         } /* switch */
       
   107 
       
   108         /* 0xFFFE and 0xFFFF are illegal, too, so we check them at the edge. */
       
   109         if ((retval >= 0x800) && (retval <= 0xFFFD))
       
   110             return retval;
       
   111     } /* else if */
       
   112 
       
   113     else if (octet < 248)  /* four octets */
       
   114     {
       
   115         octet -= (128+64+32+16);
       
   116         octet2 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   117         if ((octet2 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   118             return UNICODE_BOGUS_CHAR_VALUE;
       
   119 
       
   120         octet3 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   121         if ((octet3 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   122             return UNICODE_BOGUS_CHAR_VALUE;
       
   123 
       
   124         octet4 = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   125         if ((octet4 & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   126             return UNICODE_BOGUS_CHAR_VALUE;
       
   127 
       
   128         *_str += 4;  /* skip to next possible start of codepoint. */
       
   129         retval = ( ((octet << 18)) | ((octet2 - 128) << 12) |
       
   130                    ((octet3 - 128) << 6) | ((octet4 - 128)) );
       
   131         if ((retval >= 0x10000) && (retval <= 0x10FFFF))
       
   132             return retval;
       
   133     } /* else if */
       
   134 
       
   135     /*
       
   136      * Five and six octet sequences became illegal in rfc3629.
       
   137      *  We throw the codepoint away, but parse them to make sure we move
       
   138      *  ahead the right number of bytes and don't overflow the buffer.
       
   139      */
       
   140 
       
   141     else if (octet < 252)  /* five octets */
       
   142     {
       
   143         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   144         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   145             return UNICODE_BOGUS_CHAR_VALUE;
       
   146 
       
   147         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   148         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   149             return UNICODE_BOGUS_CHAR_VALUE;
       
   150 
       
   151         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   152         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   153             return UNICODE_BOGUS_CHAR_VALUE;
       
   154 
       
   155         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   156         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   157             return UNICODE_BOGUS_CHAR_VALUE;
       
   158 
       
   159         *_str += 5;  /* skip to next possible start of codepoint. */
       
   160         return UNICODE_BOGUS_CHAR_VALUE;
       
   161     } /* else if */
       
   162 
       
   163     else  /* six octets */
       
   164     {
       
   165         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   166         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   167             return UNICODE_BOGUS_CHAR_VALUE;
       
   168 
       
   169         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   170         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   171             return UNICODE_BOGUS_CHAR_VALUE;
       
   172 
       
   173         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   174         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   175             return UNICODE_BOGUS_CHAR_VALUE;
       
   176 
       
   177         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   178         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   179             return UNICODE_BOGUS_CHAR_VALUE;
       
   180 
       
   181         octet = (PHYSFS_uint32) ((PHYSFS_uint8) *(++str));
       
   182         if ((octet & (128+64)) != 128)  /* Format isn't 10xxxxxx? */
       
   183             return UNICODE_BOGUS_CHAR_VALUE;
       
   184 
       
   185         *_str += 6;  /* skip to next possible start of codepoint. */
       
   186         return UNICODE_BOGUS_CHAR_VALUE;
       
   187     } /* else if */
       
   188 
       
   189     return UNICODE_BOGUS_CHAR_VALUE;
       
   190 } /* utf8codepoint */
       
   191 
       
   192 void PHYSFS_utf8toucs4(const char *src, PHYSFS_uint32 *dst, PHYSFS_uint64 len)
       
   193 {
       
   194     len -= sizeof (PHYSFS_uint32);   /* save room for null char. */
       
   195     while (len >= sizeof (PHYSFS_uint32))
       
   196     {
       
   197         PHYSFS_uint32 cp = utf8codepoint(&src);
       
   198         if (cp == 0)
       
   199             break;
       
   200         else if (cp == UNICODE_BOGUS_CHAR_VALUE)
       
   201             cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   202         *(dst++) = cp;
       
   203         len -= sizeof (PHYSFS_uint32);
       
   204     } /* while */
       
   205 
       
   206     *dst = 0;
       
   207 } /* PHYSFS_utf8toucs4 */
       
   208 
       
   209 void PHYSFS_utf8toucs2(const char *src, PHYSFS_uint16 *dst, PHYSFS_uint64 len)
       
   210 {
       
   211     len -= sizeof (PHYSFS_uint16);   /* save room for null char. */
       
   212     while (len >= sizeof (PHYSFS_uint16))
       
   213     {
       
   214         PHYSFS_uint32 cp = utf8codepoint(&src);
       
   215         if (cp == 0)
       
   216             break;
       
   217         else if (cp == UNICODE_BOGUS_CHAR_VALUE)
       
   218             cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   219 
       
   220         /* !!! BLUESKY: UTF-16 surrogates? */
       
   221         if (cp > 0xFFFF)
       
   222             cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   223 
       
   224         *(dst++) = cp;
       
   225         len -= sizeof (PHYSFS_uint16);
       
   226     } /* while */
       
   227 
       
   228     *dst = 0;
       
   229 } /* PHYSFS_utf8toucs2 */
       
   230 
       
   231 static void utf8fromcodepoint(PHYSFS_uint32 cp, char **_dst, PHYSFS_uint64 *_len)
       
   232 {
       
   233     char *dst = *_dst;
       
   234     PHYSFS_uint64 len = *_len;
       
   235 
       
   236     if (len == 0)
       
   237         return;
       
   238 
       
   239     if (cp > 0x10FFFF)
       
   240         cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   241     else if ((cp == 0xFFFE) || (cp == 0xFFFF))  /* illegal values. */
       
   242         cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   243     else
       
   244     {
       
   245         /* There are seven "UTF-16 surrogates" that are illegal in UTF-8. */
       
   246         switch (cp)
       
   247         {
       
   248             case 0xD800:
       
   249             case 0xDB7F:
       
   250             case 0xDB80:
       
   251             case 0xDBFF:
       
   252             case 0xDC00:
       
   253             case 0xDF80:
       
   254             case 0xDFFF:
       
   255                 cp = UNICODE_BOGUS_CHAR_CODEPOINT;
       
   256         } /* switch */
       
   257     } /* else */
       
   258 
       
   259     /* Do the encoding... */
       
   260     if (cp < 0x80)
       
   261     {
       
   262         *(dst++) = (char) cp;
       
   263         len--;
       
   264     } /* if */
       
   265 
       
   266     else if (cp < 0x800)
       
   267     {
       
   268         if (len < 2)
       
   269             len = 0;
       
   270         else
       
   271         {
       
   272             *(dst++) = (char) ((cp >> 6) | 128 | 64);
       
   273             *(dst++) = (char) (cp & 0x3F) | 128;
       
   274             len -= 2;
       
   275         } /* else */
       
   276     } /* else if */
       
   277 
       
   278     else if (cp < 0x10000)
       
   279     {
       
   280         if (len < 3)
       
   281             len = 0;
       
   282         else
       
   283         {
       
   284             *(dst++) = (char) ((cp >> 12) | 128 | 64 | 32);
       
   285             *(dst++) = (char) ((cp >> 6) & 0x3F) | 128;
       
   286             *(dst++) = (char) (cp & 0x3F) | 128;
       
   287             len -= 3;
       
   288         } /* else */
       
   289     } /* else if */
       
   290 
       
   291     else
       
   292     {
       
   293         if (len < 4)
       
   294             len = 0;
       
   295         else
       
   296         {
       
   297             *(dst++) = (char) ((cp >> 18) | 128 | 64 | 32 | 16);
       
   298             *(dst++) = (char) ((cp >> 12) & 0x3F) | 128;
       
   299             *(dst++) = (char) ((cp >> 6) & 0x3F) | 128;
       
   300             *(dst++) = (char) (cp & 0x3F) | 128;
       
   301             len -= 4;
       
   302         } /* else if */
       
   303     } /* else */
       
   304 
       
   305     *_dst = dst;
       
   306     *_len = len;
       
   307 } /* utf8fromcodepoint */
       
   308 
       
   309 #define UTF8FROMTYPE(typ, src, dst, len) \
       
   310     len--;  \
       
   311     while (len) \
       
   312     { \
       
   313         const PHYSFS_uint32 cp = (PHYSFS_uint32) *(src++); \
       
   314         if (cp == 0) break; \
       
   315         utf8fromcodepoint(cp, &dst, &len); \
       
   316     } \
       
   317     *dst = '\0'; \
       
   318 
       
   319 void PHYSFS_utf8fromucs4(const PHYSFS_uint32 *src, char *dst, PHYSFS_uint64 len)
       
   320 {
       
   321     UTF8FROMTYPE(PHYSFS_uint32, src, dst, len);
       
   322 } /* PHYSFS_utf8fromucs4 */
       
   323 
       
   324 void PHYSFS_utf8fromucs2(const PHYSFS_uint16 *src, char *dst, PHYSFS_uint64 len)
       
   325 {
       
   326     UTF8FROMTYPE(PHYSFS_uint64, src, dst, len);
       
   327 } /* PHYSFS_utf8fromucs4 */
       
   328 
       
   329 /* latin1 maps to unicode codepoints directly, we just utf-8 encode it. */
       
   330 void PHYSFS_utf8fromlatin1(const char *src, char *dst, PHYSFS_uint64 len)
       
   331 {
       
   332     UTF8FROMTYPE(PHYSFS_uint8, src, dst, len);
       
   333 } /* PHYSFS_utf8fromlatin1 */
       
   334 
       
   335 #undef UTF8FROMTYPE
       
   336 
       
   337 /* end of physfs_unicode.c ... */
       
   338