mirror of https://github.com/postgres/postgres
> - corrects a bit the UTF-8 code from Tatsuo to allow Unicode 3.1 > characters (characters with values >= 0x10000, which are encoded on > four bytes). Also, update mb/expected/unicode.out. This is necessary since the patches affetc the result of queries using UTF-8. --------------------------------------------------------------- Hi, I should have sent the patch earlier, but got delayed by other stuff. Anyway, here is the patch: - most of the functionality is only activated when MULTIBYTE is defined, - check valid UTF-8 characters, client-side only yet, and only on output, you still can send invalid UTF-8 to the server (so, it's only partly compliant to Unicode 3.1, but that's better than nothing). - formats with the correct number of columns (that's why I made it in the first place after all), but only for UNICODE. However, the code allows to plug-in routines for other encodings, as Tatsuo did for the other multibyte functions. - corrects a bit the UTF-8 code from Tatsuo to allow Unicode 3.1 characters (characters with values >= 0x10000, which are encoded on four bytes). - doesn't depend on the locale capabilities of the glibc (useful for remote telnet). I would like somebody to check it closely, as it is my first patch to pgsql. Also, I created dummy .orig files, so that the two files I created are included, I hope that's the right way. Now, a lot of functionality is NOT included here, but I will keep that for 7.3 :) That includes all string checking on the server side (which will have to be a bit more optimised ;) ), and the input checking on the client side for UTF-8, though that should not be difficult. It's just to send the strings through mbvalidate() before sending them to the server. Strong checking on UTF-8 strings is mandatory to be compliant with Unicode 3.1+ . Do I have time to look for a patch to include iso-8859-15 for 7.2 ? The euro is coming 1. january 2002 (before 7.3 !) and over 280 millions people in Europe will need the euro sign and only iso-8859-15 and iso-8859-16 have it (and unfortunately, I don't think all Unices will switch to Unicode in the meantime).... err... yes, I know that this is not every single person in Europe that uses PostgreSql, so it's not exactly 280m, but it's just a matter of time ! ;) I'll come back (on pgsql-hackers) later to ask a few questions regarding the full unicode support (normalisation, collation, regexes,...) on the server side :) Here is the patch ! Patrice. -- Patrice HÉDÉ ------------------------------- patrice à islande org ----- -- Isn't it weird how scientists can imagine all the matter of the universe exploding out of a dot smaller than the head of a pin, but they can't come up with a more evocative name for it than "The Big Bang" ? -- What would _you_ call the creation of the universe ? -- "The HORRENDOUS SPACE KABLOOIE !" - Calvin and Hobbes ------------------------------------------ http://www.islande.org/ -----REL7_2_STABLE
Tatsuo Ishii
24 years ago
parent
d07bacd54a
commit
df4cba68cf
@ -0,0 +1,334 @@ |
||||
/*
|
||||
* psql - the PostgreSQL interactive terminal |
||||
* |
||||
* Copyright 2000 by PostgreSQL Global Development Group |
||||
* |
||||
* $Header: /cvsroot/pgsql/src/bin/psql/mbprint.c,v 1.1 2001/10/15 01:25:10 ishii Exp $ |
||||
*/ |
||||
|
||||
#include "postgres_fe.h" |
||||
#include "mbprint.h" |
||||
|
||||
#ifdef MULTIBYTE |
||||
|
||||
#include "mb/pg_wchar.h" |
||||
#include "settings.h" |
||||
|
||||
/*
|
||||
* This is an implementation of wcwidth() and wcswidth() as defined in |
||||
* "The Single UNIX Specification, Version 2, The Open Group, 1997" |
||||
* <http://www.UNIX-systems.org/online.html>
|
||||
* |
||||
* Markus Kuhn -- 2001-09-08 -- public domain |
||||
* |
||||
* customised for PostgreSQL |
||||
* |
||||
* original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
|
||||
*/ |
||||
|
||||
struct mbinterval { |
||||
unsigned short first; |
||||
unsigned short last; |
||||
}; |
||||
|
||||
/* auxiliary function for binary search in interval table */ |
||||
static int |
||||
mbbisearch(pg_wchar ucs, const struct mbinterval *table, int max) |
||||
{ |
||||
int min = 0; |
||||
int mid; |
||||
|
||||
if (ucs < table[0].first || ucs > table[max].last) |
||||
return 0; |
||||
while (max >= min) { |
||||
mid = (min + max) / 2; |
||||
if (ucs > table[mid].last) |
||||
min = mid + 1; |
||||
else if (ucs < table[mid].first) |
||||
max = mid - 1; |
||||
else |
||||
return 1; |
||||
} |
||||
|
||||
return 0; |
||||
} |
||||
|
||||
|
||||
/* The following functions define the column width of an ISO 10646
|
||||
* character as follows: |
||||
* |
||||
* - The null character (U+0000) has a column width of 0. |
||||
* |
||||
* - Other C0/C1 control characters and DEL will lead to a return |
||||
* value of -1. |
||||
* |
||||
* - Non-spacing and enclosing combining characters (general |
||||
* category code Mn or Me in the Unicode database) have a |
||||
* column width of 0. |
||||
* |
||||
* - Other format characters (general category code Cf in the Unicode |
||||
* database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. |
||||
* |
||||
* - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) |
||||
* have a column width of 0. |
||||
* |
||||
* - Spacing characters in the East Asian Wide (W) or East Asian |
||||
* FullWidth (F) category as defined in Unicode Technical |
||||
* Report #11 have a column width of 2. |
||||
* |
||||
* - All remaining characters (including all printable |
||||
* ISO 8859-1 and WGL4 characters, Unicode control characters, |
||||
* etc.) have a column width of 1. |
||||
* |
||||
* This implementation assumes that wchar_t characters are encoded |
||||
* in ISO 10646. |
||||
*/ |
||||
|
||||
static int |
||||
ucs_wcwidth(pg_wchar ucs) |
||||
{ |
||||
/* sorted list of non-overlapping intervals of non-spacing characters */ |
||||
static const struct mbinterval combining[] = { |
||||
{ 0x0300, 0x034E }, { 0x0360, 0x0362 }, { 0x0483, 0x0486 }, |
||||
{ 0x0488, 0x0489 }, { 0x0591, 0x05A1 }, { 0x05A3, 0x05B9 }, |
||||
{ 0x05BB, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, |
||||
{ 0x05C4, 0x05C4 }, { 0x064B, 0x0655 }, { 0x0670, 0x0670 }, |
||||
{ 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, |
||||
{ 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, |
||||
{ 0x07A6, 0x07B0 }, { 0x0901, 0x0902 }, { 0x093C, 0x093C }, |
||||
{ 0x0941, 0x0948 }, { 0x094D, 0x094D }, { 0x0951, 0x0954 }, |
||||
{ 0x0962, 0x0963 }, { 0x0981, 0x0981 }, { 0x09BC, 0x09BC }, |
||||
{ 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, { 0x09E2, 0x09E3 }, |
||||
{ 0x0A02, 0x0A02 }, { 0x0A3C, 0x0A3C }, { 0x0A41, 0x0A42 }, |
||||
{ 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, { 0x0A70, 0x0A71 }, |
||||
{ 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, { 0x0AC1, 0x0AC5 }, |
||||
{ 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, { 0x0B01, 0x0B01 }, |
||||
{ 0x0B3C, 0x0B3C }, { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, |
||||
{ 0x0B4D, 0x0B4D }, { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, |
||||
{ 0x0BC0, 0x0BC0 }, { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, |
||||
{ 0x0C46, 0x0C48 }, { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, |
||||
{ 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, |
||||
{ 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, { 0x0DCA, 0x0DCA }, |
||||
{ 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, { 0x0E31, 0x0E31 }, |
||||
{ 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, { 0x0EB1, 0x0EB1 }, |
||||
{ 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, { 0x0EC8, 0x0ECD }, |
||||
{ 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, { 0x0F37, 0x0F37 }, |
||||
{ 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, { 0x0F80, 0x0F84 }, |
||||
{ 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, { 0x0F99, 0x0FBC }, |
||||
{ 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, { 0x1032, 0x1032 }, |
||||
{ 0x1036, 0x1037 }, { 0x1039, 0x1039 }, { 0x1058, 0x1059 }, |
||||
{ 0x1160, 0x11FF }, { 0x17B7, 0x17BD }, { 0x17C6, 0x17C6 }, |
||||
{ 0x17C9, 0x17D3 }, { 0x180B, 0x180E }, { 0x18A9, 0x18A9 }, |
||||
{ 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x206A, 0x206F }, |
||||
{ 0x20D0, 0x20E3 }, { 0x302A, 0x302F }, { 0x3099, 0x309A }, |
||||
{ 0xFB1E, 0xFB1E }, { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, |
||||
{ 0xFFF9, 0xFFFB } |
||||
}; |
||||
|
||||
/* test for 8-bit control characters */ |
||||
if (ucs == 0) { |
||||
return 0; |
||||
} |
||||
|
||||
if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff) { |
||||
return -1; |
||||
} |
||||
|
||||
/* binary search in table of non-spacing characters */ |
||||
if (mbbisearch(ucs, combining, |
||||
sizeof(combining) / sizeof(struct mbinterval) - 1)) { |
||||
return 0; |
||||
} |
||||
|
||||
/* if we arrive here, ucs is not a combining or C0/C1 control character */ |
||||
|
||||
return 1 +
|
||||
(ucs >= 0x1100 && |
||||
(ucs <= 0x115f || /* Hangul Jamo init. consonants */ |
||||
(ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a && |
||||
ucs != 0x303f) || /* CJK ... Yi */ |
||||
(ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ |
||||
(ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ |
||||
(ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ |
||||
(ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */ |
||||
(ucs >= 0xffe0 && ucs <= 0xffe6) || |
||||
(ucs >= 0x20000 && ucs <= 0x2ffff))); |
||||
} |
||||
|
||||
pg_wchar |
||||
utf2ucs(const unsigned char *c) |
||||
{ |
||||
/* one char version of pg_utf2wchar_with_len.
|
||||
* no control here, c must point to a large enough string |
||||
*/ |
||||
if ((*c & 0x80) == 0) { |
||||
return (pg_wchar)c[0]; |
||||
} |
||||
else if ((*c & 0xe0) == 0xc0) { |
||||
return (pg_wchar)(((c[0] & 0x1f) << 6) | |
||||
(c[1] & 0x3f)); |
||||
} |
||||
else if ((*c & 0xf0) == 0xe0) { |
||||
return (pg_wchar)(((c[0] & 0x0f) << 12) | |
||||
((c[1] & 0x3f) << 6) | |
||||
(c[2] & 0x3f)); |
||||
} |
||||
else if ((*c & 0xf0) == 0xf0) { |
||||
return (pg_wchar)(((c[0] & 0x07) << 18) | |
||||
((c[1] & 0x3f) << 12) | |
||||
((c[2] & 0x3f) << 6) | |
||||
(c[3] & 0x3f)); |
||||
} |
||||
else { |
||||
/* that is an invalid code on purpose */ |
||||
return 0xffffffff; |
||||
} |
||||
} |
||||
|
||||
/* mb_utf_wcwidth : calculate column length for the utf8 string pwcs
|
||||
*/ |
||||
static int |
||||
mb_utf_wcswidth(unsigned char *pwcs, int len) |
||||
{ |
||||
int w, l = 0; |
||||
int width = 0; |
||||
|
||||
for (;*pwcs && len > 0; pwcs+=l) { |
||||
l = pg_utf_mblen(pwcs); |
||||
if ((len < l) || ((w = ucs_wcwidth(utf2ucs(pwcs))) < 0)) { |
||||
return width; |
||||
} |
||||
len -= l; |
||||
width += w; |
||||
} |
||||
return width; |
||||
} |
||||
|
||||
static int |
||||
utf_charcheck(const unsigned char *c) |
||||
{ |
||||
/* Unicode 3.1 compliant validation :
|
||||
* for each category, it checks the combination of each byte to make sur |
||||
* it maps to a valid range. It also returns -1 for the following UCS values: |
||||
* ucs > 0x10ffff |
||||
* ucs & 0xfffe = 0xfffe |
||||
* 0xfdd0 < ucs < 0xfdef |
||||
* ucs & 0xdb00 = 0xd800 (surrogates) |
||||
*/ |
||||
if ((*c & 0x80) == 0) { |
||||
return 1; |
||||
} |
||||
else if ((*c & 0xe0) == 0xc0) { |
||||
/* two-byte char */ |
||||
if(((c[1] & 0xc0) == 0x80) && ((c[0] & 0x1f) > 0x01)) { |
||||
return 2; |
||||
} |
||||
return -1; |
||||
} |
||||
else if ((*c & 0xf0) == 0xe0) { |
||||
/* three-byte char */ |
||||
if (((c[1] & 0xc0) == 0x80) && |
||||
(((c[0] & 0x0f) != 0x00) || ((c[1] & 0x20) == 0x20)) && |
||||
((c[2] & 0xc0) == 0x80)) { |
||||
int z = c[0] & 0x0f; |
||||
int yx = ((c[1] & 0x3f) << 6) | (c[0] & 0x3f); |
||||
int lx = yx & 0x7f; |
||||
|
||||
/* check 0xfffe/0xffff, 0xfdd0..0xfedf range, surrogates */ |
||||
if (((z == 0x0f) && |
||||
(((yx & 0xffe) == 0xffe) || |
||||
(((yx & 0xf80) == 0xd80) && (lx >= 0x30) && (lx <= 0x4f)))) || |
||||
((z == 0x0d) && ((yx & 0xb00) == 0x800))) { |
||||
return -1; |
||||
} |
||||
return 3; |
||||
} |
||||
return -1; |
||||
} |
||||
else if ((*c & 0xf8) == 0xf0) { |
||||
int u = ((c[0] & 0x07) << 2) | ((c[1] & 0x30) >> 4); |
||||
|
||||
/* four-byte char */ |
||||
if (((c[1] & 0xc0) == 0x80) && |
||||
(u > 0x00) && (u <= 0x10) && |
||||
((c[2] & 0xc0) == 0x80) && ((c[3] & 0xc0) == 0x80)) { |
||||
/* test for 0xzzzzfffe/0xzzzzfffff */ |
||||
if (((c[1] & 0x0f) == 0x0f) && ((c[2] & 0x3f) == 0x3f) && |
||||
((c[3] & 0x3e) == 0x3e)) { |
||||
return -1; |
||||
} |
||||
return 4; |
||||
} |
||||
return -1; |
||||
} |
||||
return -1; |
||||
} |
||||
|
||||
static unsigned char * |
||||
mb_utf_validate(unsigned char *pwcs) |
||||
{ |
||||
int l = 0; |
||||
unsigned char *p = pwcs; |
||||
unsigned char *p0 = pwcs; |
||||
|
||||
while( *pwcs ) { |
||||
if ((l = utf_charcheck(pwcs)) > 0) { |
||||
if (p != pwcs) { |
||||
int i; |
||||
for( i = 0; i < l; i++) { |
||||
*p++ = *pwcs++; |
||||
} |
||||
} |
||||
else { |
||||
pwcs += l; |
||||
p += l; |
||||
} |
||||
} |
||||
else { |
||||
/* we skip the char */ |
||||
pwcs++; |
||||
} |
||||
} |
||||
if (p != pwcs) { |
||||
*p = '\0'; |
||||
} |
||||
return p0; |
||||
} |
||||
|
||||
/*
|
||||
* public functions : wcswidth and mbvalidate |
||||
*/ |
||||
|
||||
int |
||||
pg_wcswidth(unsigned char *pwcs, int len) { |
||||
if (pset.encoding == PG_UTF8) { |
||||
return mb_utf_wcswidth(pwcs, len); |
||||
} |
||||
else { |
||||
/* obviously, other encodings may want to fix this, but I don't know them
|
||||
* myself, unfortunately. |
||||
*/ |
||||
return len; |
||||
} |
||||
} |
||||
|
||||
unsigned char * |
||||
mbvalidate(unsigned char *pwcs) { |
||||
if (pset.encoding == PG_UTF8) { |
||||
return mb_utf_validate(pwcs); |
||||
} |
||||
else { |
||||
/* other encodings needing validation should add their own routines here
|
||||
*/ |
||||
return pwcs; |
||||
} |
||||
} |
||||
#else /* !MULTIBYTE */ |
||||
|
||||
/* in single-byte environment, all cells take 1 column */ |
||||
int pg_wcswidth(unsigned char *pwcs, int len) { |
||||
return len; |
||||
} |
||||
#endif |
||||
|
||||
|
||||
Loading…
Reference in new issue