mirror of https://github.com/postgres/postgres
> - corrects a bit the UTF-8 code from Tatsuo to allow Unicode 3.1 > characters (characters with values >= 0x10000, which are encoded on > four bytes). Also, update mb/expected/unicode.out. This is necessary since the patches affetc the result of queries using UTF-8. --------------------------------------------------------------- Hi, I should have sent the patch earlier, but got delayed by other stuff. Anyway, here is the patch: - most of the functionality is only activated when MULTIBYTE is defined, - check valid UTF-8 characters, client-side only yet, and only on output, you still can send invalid UTF-8 to the server (so, it's only partly compliant to Unicode 3.1, but that's better than nothing). - formats with the correct number of columns (that's why I made it in the first place after all), but only for UNICODE. However, the code allows to plug-in routines for other encodings, as Tatsuo did for the other multibyte functions. - corrects a bit the UTF-8 code from Tatsuo to allow Unicode 3.1 characters (characters with values >= 0x10000, which are encoded on four bytes). - doesn't depend on the locale capabilities of the glibc (useful for remote telnet). I would like somebody to check it closely, as it is my first patch to pgsql. Also, I created dummy .orig files, so that the two files I created are included, I hope that's the right way. Now, a lot of functionality is NOT included here, but I will keep that for 7.3 :) That includes all string checking on the server side (which will have to be a bit more optimised ;) ), and the input checking on the client side for UTF-8, though that should not be difficult. It's just to send the strings through mbvalidate() before sending them to the server. Strong checking on UTF-8 strings is mandatory to be compliant with Unicode 3.1+ . Do I have time to look for a patch to include iso-8859-15 for 7.2 ? The euro is coming 1. january 2002 (before 7.3 !) and over 280 millions people in Europe will need the euro sign and only iso-8859-15 and iso-8859-16 have it (and unfortunately, I don't think all Unices will switch to Unicode in the meantime).... err... yes, I know that this is not every single person in Europe that uses PostgreSql, so it's not exactly 280m, but it's just a matter of time ! ;) I'll come back (on pgsql-hackers) later to ask a few questions regarding the full unicode support (normalisation, collation, regexes,...) on the server side :) Here is the patch ! Patrice. -- Patrice HÉDÉ ------------------------------- patrice à islande org ----- -- Isn't it weird how scientists can imagine all the matter of the universe exploding out of a dot smaller than the head of a pin, but they can't come up with a more evocative name for it than "The Big Bang" ? -- What would _you_ call the creation of the universe ? -- "The HORRENDOUS SPACE KABLOOIE !" - Calvin and Hobbes ------------------------------------------ http://www.islande.org/ -----REL7_2_STABLE
Tatsuo Ishii
24 years ago
parent
d07bacd54a
commit
df4cba68cf
@ -0,0 +1,334 @@ |
|||||||
|
/*
|
||||||
|
* psql - the PostgreSQL interactive terminal |
||||||
|
* |
||||||
|
* Copyright 2000 by PostgreSQL Global Development Group |
||||||
|
* |
||||||
|
* $Header: /cvsroot/pgsql/src/bin/psql/mbprint.c,v 1.1 2001/10/15 01:25:10 ishii Exp $ |
||||||
|
*/ |
||||||
|
|
||||||
|
#include "postgres_fe.h" |
||||||
|
#include "mbprint.h" |
||||||
|
|
||||||
|
#ifdef MULTIBYTE |
||||||
|
|
||||||
|
#include "mb/pg_wchar.h" |
||||||
|
#include "settings.h" |
||||||
|
|
||||||
|
/*
|
||||||
|
* This is an implementation of wcwidth() and wcswidth() as defined in |
||||||
|
* "The Single UNIX Specification, Version 2, The Open Group, 1997" |
||||||
|
* <http://www.UNIX-systems.org/online.html>
|
||||||
|
* |
||||||
|
* Markus Kuhn -- 2001-09-08 -- public domain |
||||||
|
* |
||||||
|
* customised for PostgreSQL |
||||||
|
* |
||||||
|
* original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
|
||||||
|
*/ |
||||||
|
|
||||||
|
struct mbinterval { |
||||||
|
unsigned short first; |
||||||
|
unsigned short last; |
||||||
|
}; |
||||||
|
|
||||||
|
/* auxiliary function for binary search in interval table */ |
||||||
|
static int |
||||||
|
mbbisearch(pg_wchar ucs, const struct mbinterval *table, int max) |
||||||
|
{ |
||||||
|
int min = 0; |
||||||
|
int mid; |
||||||
|
|
||||||
|
if (ucs < table[0].first || ucs > table[max].last) |
||||||
|
return 0; |
||||||
|
while (max >= min) { |
||||||
|
mid = (min + max) / 2; |
||||||
|
if (ucs > table[mid].last) |
||||||
|
min = mid + 1; |
||||||
|
else if (ucs < table[mid].first) |
||||||
|
max = mid - 1; |
||||||
|
else |
||||||
|
return 1; |
||||||
|
} |
||||||
|
|
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
/* The following functions define the column width of an ISO 10646
|
||||||
|
* character as follows: |
||||||
|
* |
||||||
|
* - The null character (U+0000) has a column width of 0. |
||||||
|
* |
||||||
|
* - Other C0/C1 control characters and DEL will lead to a return |
||||||
|
* value of -1. |
||||||
|
* |
||||||
|
* - Non-spacing and enclosing combining characters (general |
||||||
|
* category code Mn or Me in the Unicode database) have a |
||||||
|
* column width of 0. |
||||||
|
* |
||||||
|
* - Other format characters (general category code Cf in the Unicode |
||||||
|
* database) and ZERO WIDTH SPACE (U+200B) have a column width of 0. |
||||||
|
* |
||||||
|
* - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF) |
||||||
|
* have a column width of 0. |
||||||
|
* |
||||||
|
* - Spacing characters in the East Asian Wide (W) or East Asian |
||||||
|
* FullWidth (F) category as defined in Unicode Technical |
||||||
|
* Report #11 have a column width of 2. |
||||||
|
* |
||||||
|
* - All remaining characters (including all printable |
||||||
|
* ISO 8859-1 and WGL4 characters, Unicode control characters, |
||||||
|
* etc.) have a column width of 1. |
||||||
|
* |
||||||
|
* This implementation assumes that wchar_t characters are encoded |
||||||
|
* in ISO 10646. |
||||||
|
*/ |
||||||
|
|
||||||
|
static int |
||||||
|
ucs_wcwidth(pg_wchar ucs) |
||||||
|
{ |
||||||
|
/* sorted list of non-overlapping intervals of non-spacing characters */ |
||||||
|
static const struct mbinterval combining[] = { |
||||||
|
{ 0x0300, 0x034E }, { 0x0360, 0x0362 }, { 0x0483, 0x0486 }, |
||||||
|
{ 0x0488, 0x0489 }, { 0x0591, 0x05A1 }, { 0x05A3, 0x05B9 }, |
||||||
|
{ 0x05BB, 0x05BD }, { 0x05BF, 0x05BF }, { 0x05C1, 0x05C2 }, |
||||||
|
{ 0x05C4, 0x05C4 }, { 0x064B, 0x0655 }, { 0x0670, 0x0670 }, |
||||||
|
{ 0x06D6, 0x06E4 }, { 0x06E7, 0x06E8 }, { 0x06EA, 0x06ED }, |
||||||
|
{ 0x070F, 0x070F }, { 0x0711, 0x0711 }, { 0x0730, 0x074A }, |
||||||
|
{ 0x07A6, 0x07B0 }, { 0x0901, 0x0902 }, { 0x093C, 0x093C }, |
||||||
|
{ 0x0941, 0x0948 }, { 0x094D, 0x094D }, { 0x0951, 0x0954 }, |
||||||
|
{ 0x0962, 0x0963 }, { 0x0981, 0x0981 }, { 0x09BC, 0x09BC }, |
||||||
|
{ 0x09C1, 0x09C4 }, { 0x09CD, 0x09CD }, { 0x09E2, 0x09E3 }, |
||||||
|
{ 0x0A02, 0x0A02 }, { 0x0A3C, 0x0A3C }, { 0x0A41, 0x0A42 }, |
||||||
|
{ 0x0A47, 0x0A48 }, { 0x0A4B, 0x0A4D }, { 0x0A70, 0x0A71 }, |
||||||
|
{ 0x0A81, 0x0A82 }, { 0x0ABC, 0x0ABC }, { 0x0AC1, 0x0AC5 }, |
||||||
|
{ 0x0AC7, 0x0AC8 }, { 0x0ACD, 0x0ACD }, { 0x0B01, 0x0B01 }, |
||||||
|
{ 0x0B3C, 0x0B3C }, { 0x0B3F, 0x0B3F }, { 0x0B41, 0x0B43 }, |
||||||
|
{ 0x0B4D, 0x0B4D }, { 0x0B56, 0x0B56 }, { 0x0B82, 0x0B82 }, |
||||||
|
{ 0x0BC0, 0x0BC0 }, { 0x0BCD, 0x0BCD }, { 0x0C3E, 0x0C40 }, |
||||||
|
{ 0x0C46, 0x0C48 }, { 0x0C4A, 0x0C4D }, { 0x0C55, 0x0C56 }, |
||||||
|
{ 0x0CBF, 0x0CBF }, { 0x0CC6, 0x0CC6 }, { 0x0CCC, 0x0CCD }, |
||||||
|
{ 0x0D41, 0x0D43 }, { 0x0D4D, 0x0D4D }, { 0x0DCA, 0x0DCA }, |
||||||
|
{ 0x0DD2, 0x0DD4 }, { 0x0DD6, 0x0DD6 }, { 0x0E31, 0x0E31 }, |
||||||
|
{ 0x0E34, 0x0E3A }, { 0x0E47, 0x0E4E }, { 0x0EB1, 0x0EB1 }, |
||||||
|
{ 0x0EB4, 0x0EB9 }, { 0x0EBB, 0x0EBC }, { 0x0EC8, 0x0ECD }, |
||||||
|
{ 0x0F18, 0x0F19 }, { 0x0F35, 0x0F35 }, { 0x0F37, 0x0F37 }, |
||||||
|
{ 0x0F39, 0x0F39 }, { 0x0F71, 0x0F7E }, { 0x0F80, 0x0F84 }, |
||||||
|
{ 0x0F86, 0x0F87 }, { 0x0F90, 0x0F97 }, { 0x0F99, 0x0FBC }, |
||||||
|
{ 0x0FC6, 0x0FC6 }, { 0x102D, 0x1030 }, { 0x1032, 0x1032 }, |
||||||
|
{ 0x1036, 0x1037 }, { 0x1039, 0x1039 }, { 0x1058, 0x1059 }, |
||||||
|
{ 0x1160, 0x11FF }, { 0x17B7, 0x17BD }, { 0x17C6, 0x17C6 }, |
||||||
|
{ 0x17C9, 0x17D3 }, { 0x180B, 0x180E }, { 0x18A9, 0x18A9 }, |
||||||
|
{ 0x200B, 0x200F }, { 0x202A, 0x202E }, { 0x206A, 0x206F }, |
||||||
|
{ 0x20D0, 0x20E3 }, { 0x302A, 0x302F }, { 0x3099, 0x309A }, |
||||||
|
{ 0xFB1E, 0xFB1E }, { 0xFE20, 0xFE23 }, { 0xFEFF, 0xFEFF }, |
||||||
|
{ 0xFFF9, 0xFFFB } |
||||||
|
}; |
||||||
|
|
||||||
|
/* test for 8-bit control characters */ |
||||||
|
if (ucs == 0) { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
if (ucs < 32 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
|
||||||
|
/* binary search in table of non-spacing characters */ |
||||||
|
if (mbbisearch(ucs, combining, |
||||||
|
sizeof(combining) / sizeof(struct mbinterval) - 1)) { |
||||||
|
return 0; |
||||||
|
} |
||||||
|
|
||||||
|
/* if we arrive here, ucs is not a combining or C0/C1 control character */ |
||||||
|
|
||||||
|
return 1 +
|
||||||
|
(ucs >= 0x1100 && |
||||||
|
(ucs <= 0x115f || /* Hangul Jamo init. consonants */ |
||||||
|
(ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a && |
||||||
|
ucs != 0x303f) || /* CJK ... Yi */ |
||||||
|
(ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */ |
||||||
|
(ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility Ideographs */ |
||||||
|
(ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */ |
||||||
|
(ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */ |
||||||
|
(ucs >= 0xffe0 && ucs <= 0xffe6) || |
||||||
|
(ucs >= 0x20000 && ucs <= 0x2ffff))); |
||||||
|
} |
||||||
|
|
||||||
|
pg_wchar |
||||||
|
utf2ucs(const unsigned char *c) |
||||||
|
{ |
||||||
|
/* one char version of pg_utf2wchar_with_len.
|
||||||
|
* no control here, c must point to a large enough string |
||||||
|
*/ |
||||||
|
if ((*c & 0x80) == 0) { |
||||||
|
return (pg_wchar)c[0]; |
||||||
|
} |
||||||
|
else if ((*c & 0xe0) == 0xc0) { |
||||||
|
return (pg_wchar)(((c[0] & 0x1f) << 6) | |
||||||
|
(c[1] & 0x3f)); |
||||||
|
} |
||||||
|
else if ((*c & 0xf0) == 0xe0) { |
||||||
|
return (pg_wchar)(((c[0] & 0x0f) << 12) | |
||||||
|
((c[1] & 0x3f) << 6) | |
||||||
|
(c[2] & 0x3f)); |
||||||
|
} |
||||||
|
else if ((*c & 0xf0) == 0xf0) { |
||||||
|
return (pg_wchar)(((c[0] & 0x07) << 18) | |
||||||
|
((c[1] & 0x3f) << 12) | |
||||||
|
((c[2] & 0x3f) << 6) | |
||||||
|
(c[3] & 0x3f)); |
||||||
|
} |
||||||
|
else { |
||||||
|
/* that is an invalid code on purpose */ |
||||||
|
return 0xffffffff; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
/* mb_utf_wcwidth : calculate column length for the utf8 string pwcs
|
||||||
|
*/ |
||||||
|
static int |
||||||
|
mb_utf_wcswidth(unsigned char *pwcs, int len) |
||||||
|
{ |
||||||
|
int w, l = 0; |
||||||
|
int width = 0; |
||||||
|
|
||||||
|
for (;*pwcs && len > 0; pwcs+=l) { |
||||||
|
l = pg_utf_mblen(pwcs); |
||||||
|
if ((len < l) || ((w = ucs_wcwidth(utf2ucs(pwcs))) < 0)) { |
||||||
|
return width; |
||||||
|
} |
||||||
|
len -= l; |
||||||
|
width += w; |
||||||
|
} |
||||||
|
return width; |
||||||
|
} |
||||||
|
|
||||||
|
static int |
||||||
|
utf_charcheck(const unsigned char *c) |
||||||
|
{ |
||||||
|
/* Unicode 3.1 compliant validation :
|
||||||
|
* for each category, it checks the combination of each byte to make sur |
||||||
|
* it maps to a valid range. It also returns -1 for the following UCS values: |
||||||
|
* ucs > 0x10ffff |
||||||
|
* ucs & 0xfffe = 0xfffe |
||||||
|
* 0xfdd0 < ucs < 0xfdef |
||||||
|
* ucs & 0xdb00 = 0xd800 (surrogates) |
||||||
|
*/ |
||||||
|
if ((*c & 0x80) == 0) { |
||||||
|
return 1; |
||||||
|
} |
||||||
|
else if ((*c & 0xe0) == 0xc0) { |
||||||
|
/* two-byte char */ |
||||||
|
if(((c[1] & 0xc0) == 0x80) && ((c[0] & 0x1f) > 0x01)) { |
||||||
|
return 2; |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
else if ((*c & 0xf0) == 0xe0) { |
||||||
|
/* three-byte char */ |
||||||
|
if (((c[1] & 0xc0) == 0x80) && |
||||||
|
(((c[0] & 0x0f) != 0x00) || ((c[1] & 0x20) == 0x20)) && |
||||||
|
((c[2] & 0xc0) == 0x80)) { |
||||||
|
int z = c[0] & 0x0f; |
||||||
|
int yx = ((c[1] & 0x3f) << 6) | (c[0] & 0x3f); |
||||||
|
int lx = yx & 0x7f; |
||||||
|
|
||||||
|
/* check 0xfffe/0xffff, 0xfdd0..0xfedf range, surrogates */ |
||||||
|
if (((z == 0x0f) && |
||||||
|
(((yx & 0xffe) == 0xffe) || |
||||||
|
(((yx & 0xf80) == 0xd80) && (lx >= 0x30) && (lx <= 0x4f)))) || |
||||||
|
((z == 0x0d) && ((yx & 0xb00) == 0x800))) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
return 3; |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
else if ((*c & 0xf8) == 0xf0) { |
||||||
|
int u = ((c[0] & 0x07) << 2) | ((c[1] & 0x30) >> 4); |
||||||
|
|
||||||
|
/* four-byte char */ |
||||||
|
if (((c[1] & 0xc0) == 0x80) && |
||||||
|
(u > 0x00) && (u <= 0x10) && |
||||||
|
((c[2] & 0xc0) == 0x80) && ((c[3] & 0xc0) == 0x80)) { |
||||||
|
/* test for 0xzzzzfffe/0xzzzzfffff */ |
||||||
|
if (((c[1] & 0x0f) == 0x0f) && ((c[2] & 0x3f) == 0x3f) && |
||||||
|
((c[3] & 0x3e) == 0x3e)) { |
||||||
|
return -1; |
||||||
|
} |
||||||
|
return 4; |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
return -1; |
||||||
|
} |
||||||
|
|
||||||
|
static unsigned char * |
||||||
|
mb_utf_validate(unsigned char *pwcs) |
||||||
|
{ |
||||||
|
int l = 0; |
||||||
|
unsigned char *p = pwcs; |
||||||
|
unsigned char *p0 = pwcs; |
||||||
|
|
||||||
|
while( *pwcs ) { |
||||||
|
if ((l = utf_charcheck(pwcs)) > 0) { |
||||||
|
if (p != pwcs) { |
||||||
|
int i; |
||||||
|
for( i = 0; i < l; i++) { |
||||||
|
*p++ = *pwcs++; |
||||||
|
} |
||||||
|
} |
||||||
|
else { |
||||||
|
pwcs += l; |
||||||
|
p += l; |
||||||
|
} |
||||||
|
} |
||||||
|
else { |
||||||
|
/* we skip the char */ |
||||||
|
pwcs++; |
||||||
|
} |
||||||
|
} |
||||||
|
if (p != pwcs) { |
||||||
|
*p = '\0'; |
||||||
|
} |
||||||
|
return p0; |
||||||
|
} |
||||||
|
|
||||||
|
/*
|
||||||
|
* public functions : wcswidth and mbvalidate |
||||||
|
*/ |
||||||
|
|
||||||
|
int |
||||||
|
pg_wcswidth(unsigned char *pwcs, int len) { |
||||||
|
if (pset.encoding == PG_UTF8) { |
||||||
|
return mb_utf_wcswidth(pwcs, len); |
||||||
|
} |
||||||
|
else { |
||||||
|
/* obviously, other encodings may want to fix this, but I don't know them
|
||||||
|
* myself, unfortunately. |
||||||
|
*/ |
||||||
|
return len; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
unsigned char * |
||||||
|
mbvalidate(unsigned char *pwcs) { |
||||||
|
if (pset.encoding == PG_UTF8) { |
||||||
|
return mb_utf_validate(pwcs); |
||||||
|
} |
||||||
|
else { |
||||||
|
/* other encodings needing validation should add their own routines here
|
||||||
|
*/ |
||||||
|
return pwcs; |
||||||
|
} |
||||||
|
} |
||||||
|
#else /* !MULTIBYTE */ |
||||||
|
|
||||||
|
/* in single-byte environment, all cells take 1 column */ |
||||||
|
int pg_wcswidth(unsigned char *pwcs, int len) { |
||||||
|
return len; |
||||||
|
} |
||||||
|
#endif |
||||||
|
|
||||||
|
|
||||||
Loading…
Reference in new issue