open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve documentation about MULE encoding.

Browse Source 
This commit improves the comments in pg_wchar.h and creates #define symbols
for some formerly hard-coded values.  No substantive code changes.

Tatsuo Ishii and Tom Lane
pull/3/head
Tom Lane 13 years ago
parent 47a2adc83c
commit 09022de1f5
3 changed files with 110 additions and 58 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      src/backend/utils/mb/conversion_procs/euc_tw_and_big5/euc_tw_and_big5.c
  2. 6
      src/backend/utils/mb/wchar.c
  3. 145
      src/include/mb/pg_wchar.h

17
src/backend/utils/mb/conversion_procs/euc_tw_and_big5/euc_tw_and_big5.c
Unescape View File

@ -168,7 +168,8 @@ euc_tw2mic(const unsigned char *euc, unsigned char *p, int len)
*p++ = LC_CNS11643_2;
else
{
*p++ = 0x9d; /* LCPRV2 */
/* other planes are MULE private charsets */
*p++ = LCPRV2_B;
*p++ = c1 - 0xa3 + LC_CNS11643_3;
}
*p++ = euc[2];
@ -235,9 +236,9 @@ mic2euc_tw(const unsigned char *mic, unsigned char *p, int len)
*p++ = mic[1];
*p++ = mic[2];
}
else if (c1 == 0x9d &&
else if (c1 == LCPRV2_B &&
mic[1] >= LC_CNS11643_3 && mic[1] <= LC_CNS11643_7)
{ /* LCPRV2? */
{
*p++ = SS2;
*p++ = mic[1] - LC_CNS11643_3 + 0xa3;
*p++ = mic[2];
@ -286,10 +287,9 @@ big52mic(const unsigned char *big5, unsigned char *p, int len)
cnsBuf = BIG5toCNS(big5buf, &lc);
if (lc != 0)
{
/* Planes 3 and 4 are MULE private charsets */
if (lc == LC_CNS11643_3 || lc == LC_CNS11643_4)
{
*p++ = 0x9d; /* LCPRV2 */
}
*p++ = LCPRV2_B;
*p++ = lc; /* Plane No. */
*p++ = (cnsBuf >> 8) & 0x00ff;
*p++ = cnsBuf & 0x00ff;
@ -332,10 +332,9 @@ mic2big5(const unsigned char *mic, unsigned char *p, int len)
if (l < 0)
report_invalid_encoding(PG_MULE_INTERNAL,
(const char *) mic, len);
/* 0x9d means LCPRV2 */
if (c1 == LC_CNS11643_1 || c1 == LC_CNS11643_2 || c1 == 0x9d)
if (c1 == LC_CNS11643_1 || c1 == LC_CNS11643_2 || c1 == LCPRV2_B)
{
if (c1 == 0x9d)
if (c1 == LCPRV2_B)
{
c1 = mic[1]; /* get plane no. */
cnsBuf = (mic[2] << 8) | mic[3];

6
src/backend/utils/mb/wchar.c
Unescape View File

@ -742,6 +742,12 @@ pg_mule_dsplen(const unsigned char *s)
{
int len;
/*
* Note: it's not really appropriate to assume that all multibyte charsets
* are double-wide on screen. But this seems an okay approximation for
* the MULE charsets we currently support.
*/
if (IS_LC1(*s))
len = 1;
else if (IS_LCPRV1(*s))

145
src/include/mb/pg_wchar.h
Unescape View File

@ -36,36 +36,60 @@ typedef unsigned int pg_wchar;
#define ISSJISHEAD(c) (((c) >= 0x81 && (c) <= 0x9f) || ((c) >= 0xe0 && (c) <= 0xfc))
#define ISSJISTAIL(c) (((c) >= 0x40 && (c) <= 0x7e) || ((c) >= 0x80 && (c) <= 0xfc))
/*
* Leading byte types or leading prefix byte for MULE internal code.
* See http://www.xemacs.org for more details. (there is a doc titled
* "XEmacs Internals Manual", "MULE Character Sets and Encodings"
* section.)
*/
/*
* Is a leading byte for "official" single byte encodings?
*/
#define IS_LC1(c) ((unsigned char)(c) >= 0x81 && (unsigned char)(c) <= 0x8d)
/*
* Is a prefix byte for "private" single byte encodings?
*/
#define IS_LCPRV1(c) ((unsigned char)(c) == 0x9a || (unsigned char)(c) == 0x9b)
/*
* Is a leading byte for "official" multibyte encodings?
*/
#define IS_LC2(c) ((unsigned char)(c) >= 0x90 && (unsigned char)(c) <= 0x99)
/*
* Is a prefix byte for "private" multibyte encodings?
*/
#define IS_LCPRV2(c) ((unsigned char)(c) == 0x9c || (unsigned char)(c) == 0x9d)
/*----------------------------------------------------
* leading characters
* MULE Internal Encoding (MIC)
*
* This encoding follows the design used within XEmacs; it is meant to
* subsume many externally-defined character sets. Each character includes
* identification of the character set it belongs to, so the encoding is
* general but somewhat bulky.
*
* Currently PostgreSQL supports 5 types of MULE character sets:
*
* 1) 1-byte ASCII characters. Each byte is below 0x80.
*
* 2) "Official" single byte charsets such as ISO-8859-1 (Latin1).
* Each MULE character consists of 2 bytes: LC1 + C1, where LC1 is
* an identifier for the charset (in the range 0x81 to 0x8d) and C1
* is the character code (in the range 0xa0 to 0xff).
*
* 3) "Private" single byte charsets such as SISHENG. Each MULE
* character consists of 3 bytes: LCPRV1 + LC12 + C1, where LCPRV1
* is a private-charset flag, LC12 is an identifier for the charset,
* and C1 is the character code (in the range 0xa0 to 0xff).
* LCPRV1 is either 0x9a (if LC12 is in the range 0xa0 to 0xdf)
* or 0x9b (if LC12 is in the range 0xe0 to 0xef).
*
* 4) "Official" multibyte charsets such as JIS X0208. Each MULE
* character consists of 3 bytes: LC2 + C1 + C2, where LC2 is
* an identifier for the charset (in the range 0x90 to 0x99) and C1
* and C2 form the character code (each in the range 0xa0 to 0xff).
*
* 5) "Private" multibyte charsets such as CNS 11643-1992 Plane 3.
* Each MULE character consists of 4 bytes: LCPRV2 + LC22 + C1 + C2,
* where LCPRV2 is a private-charset flag, LC22 is an identifier for
* the charset, and C1 and C2 form the character code (each in the range
* 0xa0 to 0xff). LCPRV2 is either 0x9c (if LC22 is in the range 0xf0
* to 0xf4) or 0x9d (if LC22 is in the range 0xf5 to 0xfe).
*
* "Official" encodings are those that have been assigned code numbers by
* the XEmacs project; "private" encodings have Postgres-specific charset
* identifiers.
*
* See the "XEmacs Internals Manual", available at http://www.xemacs.org,
* for more details. Note that for historical reasons, Postgres'
* private-charset flag values do not match what XEmacs says they should be,
* so this isn't really exactly MULE (not that private charsets would be
* interoperable anyway).
*----------------------------------------------------
*/
/*
* Official single byte encodings (0x81-0x8e)
* Charset identifiers (also called "leading bytes" in the MULE documentation)
*/
/*
* Charset IDs for official single byte encodings (0x81-0x8e)
*/
#define LC_ISO8859_1 0x81 /* ISO8859 Latin 1 */
#define LC_ISO8859_2 0x82 /* ISO8859 Latin 2 */
@ -79,21 +103,19 @@ typedef unsigned int pg_wchar;
#define LC_JISX0201R 0x8a /* Japanese 1 byte Roman */
/* Note that 0x8b seems to be unused as of Emacs 20.7.
* However, there might be a chance that 0x8b could be used
* in later version of Emacs.
* in later versions of Emacs.
*/
#define LC_KOI8_R 0x8b /* Cyrillic KOI8-R */
#define LC_KOI8_U 0x8b /* Cyrillic KOI8-U */
#define LC_ISO8859_5 0x8c /* ISO8859 Cyrillic */
#define LC_ISO8859_9 0x8d /* ISO8859 Latin 5 (not supported yet) */
/* #define FREE 0x8e free (unused) */
/* #define CONTROL_1 0x8f control characters (unused) */
/*
* Unused
*/
#define CONTROL_1 0x8f /* control characters (unused) */
/* Is a leading byte for "official" single byte encodings? */
#define IS_LC1(c) ((unsigned char)(c) >= 0x81 && (unsigned char)(c) <= 0x8d)
/*
* Official multibyte byte encodings (0x90-0x99)
* Charset IDs for official multibyte encodings (0x90-0x99)
* 0x9a-0x9d are free. 0x9e and 0x9f are reserved.
*/
#define LC_JISX0208_1978 0x90 /* Japanese Kanji, old JIS (not supported) */
@ -108,45 +130,70 @@ typedef unsigned int pg_wchar;
#define LC_BIG5_1 0x98 /* Plane 1 Chinese traditional (not supported) */
#define LC_BIG5_2 0x99 /* Plane 1 Chinese traditional (not supported) */
/* Is a leading byte for "official" multibyte encodings? */
#define IS_LC2(c) ((unsigned char)(c) >= 0x90 && (unsigned char)(c) <= 0x99)
/*
* Private single byte encodings (0xa0-0xef)
* Postgres-specific prefix bytes for "private" single byte encodings
* (According to the MULE docs, we should be using 0x9e for this)
*/
#define LC_SISHENG 0xa0/* Chinese SiSheng characters for
* PinYin/ZhuYin (not supported) */
#define LC_IPA 0xa1/* IPA (International Phonetic Association)
* (not supported) */
#define LC_VISCII_LOWER 0xa2/* Vietnamese VISCII1.1 lower-case (not
* supported) */
#define LC_VISCII_UPPER 0xa3/* Vietnamese VISCII1.1 upper-case (not
* supported) */
#define LCPRV1_A 0x9a
#define LCPRV1_B 0x9b
#define IS_LCPRV1(c) ((unsigned char)(c) == LCPRV1_A || (unsigned char)(c) == LCPRV1_B)
/*
* Postgres-specific prefix bytes for "private" multibyte encodings
* (According to the MULE docs, we should be using 0x9f for this)
*/
#define LCPRV2_A 0x9c
#define LCPRV2_B 0x9d
#define IS_LCPRV2(c) ((unsigned char)(c) == LCPRV2_A || (unsigned char)(c) == LCPRV2_B)
/*
* Charset IDs for private single byte encodings (0xa0-0xef)
*/
#define LC_SISHENG 0xa0 /* Chinese SiSheng characters for
* PinYin/ZhuYin (not supported) */
#define LC_IPA 0xa1 /* IPA (International Phonetic Association)
* (not supported) */
#define LC_VISCII_LOWER 0xa2 /* Vietnamese VISCII1.1 lower-case (not
* supported) */
#define LC_VISCII_UPPER 0xa3 /* Vietnamese VISCII1.1 upper-case (not
* supported) */
#define LC_ARABIC_DIGIT 0xa4 /* Arabic digit (not supported) */
#define LC_ARABIC_1_COLUMN 0xa5 /* Arabic 1-column (not supported) */
#define LC_ASCII_RIGHT_TO_LEFT 0xa6 /* ASCII (left half of ISO8859-1) with
* right-to-left direction (not
* supported) */
#define LC_LAO 0xa7/* Lao characters (ISO10646 0E80..0EDF) (not
* supported) */
#define LC_LAO 0xa7 /* Lao characters (ISO10646 0E80..0EDF)
* (not supported) */
#define LC_ARABIC_2_COLUMN 0xa8 /* Arabic 1-column (not supported) */
/*
* Private multibyte encodings (0xf0-0xff)
* Charset IDs for private multibyte encodings (0xf0-0xff)
*/
#define LC_INDIAN_1_COLUMN 0xf0/* Indian charset for 1-column width glypps
* (not supported) */
#define LC_TIBETAN_1_COLUMN 0xf1 /* Tibetan 1 column glyph (not supported) */
#define LC_INDIAN_1_COLUMN 0xf0 /* Indian charset for 1-column width glyphs
* (not supported) */
#define LC_TIBETAN_1_COLUMN 0xf1 /* Tibetan 1-column width glyphs
* (not supported) */
#define LC_ETHIOPIC 0xf5 /* Ethiopic characters (not supported) */
#define LC_CNS11643_3 0xf6 /* CNS 11643-1992 Plane 3 */
#define LC_CNS11643_4 0xf7 /* CNS 11643-1992 Plane 4 */
#define LC_CNS11643_5 0xf8 /* CNS 11643-1992 Plane 5 */
#define LC_CNS11643_6 0xf9 /* CNS 11643-1992 Plane 6 */
#define LC_CNS11643_7 0xfa /* CNS 11643-1992 Plane 7 */
#define LC_INDIAN_2_COLUMN 0xfb/* Indian charset for 2-column width glypps
* (not supported) */
#define LC_INDIAN_2_COLUMN 0xfb /* Indian charset for 2-column width glyphs
* (not supported) */
#define LC_TIBETAN 0xfc /* Tibetan (not supported) */
/* #define FREE 0xfd free (unused) */
/* #define FREE 0xfe free (unused) */
/* #define FREE 0xff free (unused) */
/*----------------------------------------------------
* end of MULE stuff
*----------------------------------------------------
*/
/*
* PostgreSQL encoding identifiers
*

Write Preview
Loading…
Cancel
Save