open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Eliminate query length limitation imposed by pg_client_to_server

Browse Source 
and pg_server_to_client.  Eliminate copy.c's restriction on the length
of a single attribute.
REL7_0_PATCHES
Tom Lane 27 years ago
parent b65ab31910
commit 4644fc8071
4 changed files with 137 additions and 162 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 174
      src/backend/commands/copy.c
  2. 41
      src/backend/libpq/pqformat.c
  3. 76
      src/backend/utils/mb/mbutils.c
  4. 8
      src/include/config.h.in

174
src/backend/commands/copy.c
Unescape View File

@ -6,7 +6,7 @@
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.86 1999/07/22 02:40:06 tgl Exp $
* $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.87 1999/09/11 22:28:11 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -26,6 +26,7 @@
#include "commands/copy.h"
#include "commands/trigger.h"
#include "executor/executor.h"
#include "lib/stringinfo.h"
#include "libpq/libpq.h"
#include "miscadmin.h"
#include "utils/acl.h"
@ -51,14 +52,10 @@ static void GetIndexRelations(Oid main_relation_oid,
int *n_indices,
Relation **index_rels);
#ifdef COPY_PATCH
static void CopyReadNewline(FILE *fp, int *newline);
static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline);
#else
static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim);
#endif
static void CopyAttributeOut(FILE *fp, char *string, char *delim, int is_array);
static void CopyAttributeOut(FILE *fp, char *string, char *delim);
static int CountTuples(Relation relation);
/*
@ -431,7 +428,7 @@ CopyTo(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
{
string = (char *) (*fmgr_faddr(&out_functions[i]))
(value, elements[i], typmod[i]);
CopyAttributeOut(fp, string, delim, attr[i]->attnelems);
CopyAttributeOut(fp, string, delim);
pfree(string);
}
else
@ -691,18 +688,12 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
{
if (!binary)
{
#ifdef COPY_PATCH
int newline = 0;
#endif
lineno++;
if (oids)
{
#ifdef COPY_PATCH
string = CopyReadAttribute(fp, &isnull, delim, &newline);
#else
string = CopyReadAttribute(fp, &isnull, delim);
#endif
if (string == NULL)
done = 1;
else
@ -710,19 +701,18 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
loaded_oid = oidin(string);
if (loaded_oid < BootstrapObjectIdData)
elog(ERROR, "COPY TEXT: Invalid Oid. line: %d", lineno);
pfree(string);
}
}
for (i = 0; i < attr_count && !done; i++)
{
#ifdef COPY_PATCH
string = CopyReadAttribute(fp, &isnull, delim, &newline);
#else
string = CopyReadAttribute(fp, &isnull, delim);
#endif
if (isnull)
{
values[i] = PointerGetDatum(NULL);
nulls[i] = 'n';
if (string)
pfree(string);
}
else if (string == NULL)
done = 1;
@ -739,12 +729,11 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
if (!PointerIsValid(values[i]) &&
!(rel->rd_att->attrs[i]->attbyval))
elog(ERROR, "copy from line %d: Bad file format", lineno);
pfree(string);
}
}
#ifdef COPY_PATCH
if (!done)
CopyReadNewline(fp, &newline);
#endif
}
else
{ /* binary */
@ -812,11 +801,6 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
if (done)
continue;
/*
* Does it have any sence ? - vadim 12/14/96
*
* tupDesc = CreateTupleDesc(attr_count, attr);
*/
tuple = heap_formtuple(tupDesc, values, nulls);
if (oids)
tuple->t_data->t_oid = loaded_oid;
@ -1086,30 +1070,18 @@ GetIndexRelations(Oid main_relation_oid,
}
}
#define EXT_ATTLEN (5 * BLCKSZ)
/*
returns 1 is c is in s
returns 1 if c is in s
*/
static bool
inString(char c, char *s)
{
int i;
if (s)
{
i = 0;
while (s[i] != '\0')
{
if (s[i] == c)
return 1;
i++;
}
}
if (s && c)
return strchr(s, c) != NULL;
return 0;
}
#ifdef COPY_PATCH
/*
* Reads input from fp until an end of line is seen.
*/
@ -1125,64 +1097,57 @@ CopyReadNewline(FILE *fp, int *newline)
*newline = 0;
}
#endif
/*
* Reads input from fp until eof is seen. If we are reading from standard
* input, AND we see a dot on a line by itself (a dot followed immediately
* by a newline), we exit as if we saw eof. This is so that copy pipelines
* can be used as standard input.
* Read the value of a single attribute.
*
* Result is either a palloc'd string, or NULL (if EOF or a null attribute).
* *isnull is set true if a null attribute, else false.
*
* delim is the string of acceptable delimiter characters(s).
* *newline remembers whether we've seen a newline ending this tuple.
*/
static char *
#ifdef COPY_PATCH
CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline)
#else
CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
#endif
{
static char attribute[EXT_ATTLEN];
StringInfoData attribute_buf;
char c;
int done = 0;
int i = 0;
#ifdef MULTIBYTE
int mblen;
int encoding;
unsigned char s[2];
char *cvt;
int j;
#endif
#ifdef MULTIBYTE
encoding = pg_get_client_encoding();
s[1] = 0;
#endif
#ifdef COPY_PATCH
/* if last delimiter was a newline return a NULL attribute */
if (*newline)
{
*isnull = (bool) true;
return NULL;
}
#endif
*isnull = (bool) false; /* set default */
initStringInfo(&attribute_buf);
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
while (!done)
for (;;)
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
else if (c == '\\')
goto endOfFile;
if (c == '\\')
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
switch (c)
{
case '0':
@ -1212,14 +1177,14 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
else
{
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
CopyDonePeek(fp, c, 0); /* Return to stream! */
}
}
else
{
if (CopyGetEof(fp))
return NULL;
goto endOfFile;
CopyDonePeek(fp, c, 0); /* Return to stream! */
}
c = val & 0377;
@ -1244,66 +1209,70 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
c = '\v';
break;
case 'N':
attribute[0] = '\0'; /* just to be safe */
*isnull = (bool) true;
break;
case '.':
c = CopyGetChar(fp);
if (c != '\n')
elog(ERROR, "CopyReadAttribute - end of record marker corrupted. line: %d", lineno);
return NULL;
goto endOfFile;
break;
}
}
else if (inString(c, delim) || c == '\n')
else if (c == '\n' || inString(c, delim))
{
#ifdef COPY_PATCH
if (c == '\n')
*newline = 1;
#endif
done = 1;
break;
}
if (!done)
attribute[i++] = c;
appendStringInfoChar(&attribute_buf, c);
#ifdef MULTIBYTE
/* get additional bytes of the char, if any */
s[0] = c;
mblen = pg_encoding_mblen(encoding, s);
mblen--;
for (j = 0; j < mblen; j++)
for (j = 1; j < mblen; j++)
{
c = CopyGetChar(fp);
if (CopyGetEof(fp))
return NULL;
attribute[i++] = c;
goto endOfFile;
appendStringInfoChar(&attribute_buf, c);
}
#endif
if (i == EXT_ATTLEN - 1)
elog(ERROR, "CopyReadAttribute - attribute length too long. line: %d", lineno);
}
attribute[i] = '\0';
#ifdef MULTIBYTE
return (pg_client_to_server((unsigned char *) attribute, strlen(attribute)));
#else
return &attribute[0];
cvt = (char *) pg_client_to_server((unsigned char *) attribute_buf.data,
attribute_buf.len);
if (cvt != attribute_buf.data)
{
pfree(attribute_buf.data);
return cvt;
}
#endif
return attribute_buf.data;
endOfFile:
pfree(attribute_buf.data);
return NULL;
}
static void
CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
CopyAttributeOut(FILE *fp, char *server_string, char *delim)
{
char *string;
char c;
#ifdef MULTIBYTE
int mblen;
char *string_start;
int encoding;
int mblen;
int i;
#endif
#ifdef MULTIBYTE
string = pg_server_to_client(server_string, strlen(server_string));
encoding = pg_get_client_encoding();
string = (char *) pg_server_to_client((unsigned char *) server_string,
strlen(server_string));
string_start = string;
#else
string = server_string;
#endif
@ -1315,33 +1284,20 @@ CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
for (; (c = *string) != '\0'; string++)
#endif
{
if (c == delim[0] || c == '\n' ||
(c == '\\' && !is_array))
if (c == delim[0] || c == '\n' || c == '\\')
CopySendChar('\\', fp);
else if (c == '\\' && is_array)
{
if (*(string + 1) == '\\')
{
/* translate \\ to \\\\ */
CopySendChar('\\', fp);
CopySendChar('\\', fp);
CopySendChar('\\', fp);
string++;
}
else if (*(string + 1) == '"')
{
/* translate \" to \\\" */
CopySendChar('\\', fp);
CopySendChar('\\', fp);
}
}
#ifdef MULTIBYTE
for (i = 0; i < mblen; i++)
CopySendChar(*(string + i), fp);
#else
CopySendChar(*string, fp);
CopySendChar(c, fp);
#endif
}
#ifdef MULTIBYTE
if (string_start != server_string)
pfree(string_start); /* pfree pg_server_to_client result */
#endif
}
/*

41
src/backend/libpq/pqformat.c
Unescape View File

@ -15,7 +15,7 @@
*
* Copyright (c) 1994, Regents of the University of California
*
* $Id: pqformat.c,v 1.8 1999/08/31 04:26:37 tgl Exp $
* $Id: pqformat.c,v 1.9 1999/09/11 22:28:05 tgl Exp $
*
*-------------------------------------------------------------------------
*/
@ -125,13 +125,16 @@ void
pq_sendcountedtext(StringInfo buf, const char *str, int slen)
{
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
slen = strlen(p);
pq_sendint(buf, slen + 4, 4);
appendBinaryStringInfo(buf, p, slen);
pfree(p);
return;
}
#endif
pq_sendint(buf, slen + 4, 4);
@ -149,15 +152,16 @@ void
pq_sendstring(StringInfo buf, const char *str)
{
int slen = strlen(str);
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
slen = strlen(p);
appendBinaryStringInfo(buf, p, slen + 1);
pfree(p);
return;
}
#endif
appendBinaryStringInfo(buf, str, slen + 1);
@ -229,15 +233,15 @@ int
pq_puttextmessage(char msgtype, const char *str)
{
int slen = strlen(str);
#ifdef MULTIBYTE
const char *p;
char *p;
p = (const char *) pg_server_to_client((unsigned char *) str, slen);
p = (char *) pg_server_to_client((unsigned char *) str, slen);
if (p != str) /* actual conversion has been done? */
{
str = p;
slen = strlen(str);
int result = pq_putmessage(msgtype, p, strlen(p) + 1);
pfree(p);
return result;
}
#endif
return pq_putmessage(msgtype, str, slen + 1);
@ -299,12 +303,12 @@ pq_getint(int *result, int b)
int
pq_getstr(StringInfo s)
{
int c;
int result;
#ifdef MULTIBYTE
char *p;
#endif
c = pq_getstring(s);
result = pq_getstring(s);
#ifdef MULTIBYTE
p = (char *) pg_client_to_server((unsigned char *) s->data, s->len);
@ -314,8 +318,9 @@ pq_getstr(StringInfo s)
s->len = 0;
s->data[0] = '\0';
appendBinaryStringInfo(s, p, strlen(p));
pfree(p);
}
#endif
return c;
return result;
}

76
src/backend/utils/mb/mbutils.c
Unescape View File

@ -3,7 +3,7 @@
* client encoding and server internal encoding.
* (currently mule internal code (mic) is used)
* Tatsuo Ishii
* $Id: mbutils.c,v 1.8 1999/07/17 20:18:10 momjian Exp $ */
* $Id: mbutils.c,v 1.9 1999/09/11 22:28:00 tgl Exp $ */
#include "postgres.h"
@ -93,59 +93,81 @@ pg_get_client_encoding()
}
/*
* convert client encoding to server encoding. if server_encoding ==
* client_encoding or no conversion function exists,
* returns s. So be careful.
* convert client encoding to server encoding.
*
* CASE 1: if no conversion is required, then the given pointer s is returned.
*
* CASE 2: if conversion is required, a palloc'd string is returned.
*
* Callers must check whether return value differs from passed value
* to determine whether to pfree the result or not!
*
* Note: we assume that conversion cannot cause more than a 4-to-1 growth
* in the length of the string --- is this enough?
*/
unsigned char *
pg_client_to_server(unsigned char *s, int len)
{
static unsigned char b1[MAX_PARSE_BUFFER * 4]; /* is this enough? */
static unsigned char b2[MAX_PARSE_BUFFER * 4]; /* is this enough? */
unsigned char *p = s;
unsigned char *result = s;
unsigned char *buf;
if (client_encoding == GetDatabaseEncoding())
return (p);
return result;
if (client_to_mic)
{
(*client_to_mic) (s, b1, len);
len = strlen(b1);
p = b1;
buf = (unsigned char *) palloc(len * 4 + 1);
(*client_to_mic) (result, buf, len);
result = buf;
len = strlen(result);
}
if (server_from_mic)
{
(*server_from_mic) (p, b2, len);
p = b2;
buf = (unsigned char *) palloc(len * 4 + 1);
(*server_from_mic) (result, buf, len);
if (result != s)
pfree(result); /* release first buffer */
result = buf;
}
return (p);
return result;
}
/*
* convert server encoding to client encoding. if server_encoding ==
* client_encoding or no conversion function exists,
* returns s. So be careful.
* convert server encoding to client encoding.
*
* CASE 1: if no conversion is required, then the given pointer s is returned.
*
* CASE 2: if conversion is required, a palloc'd string is returned.
*
* Callers must check whether return value differs from passed value
* to determine whether to pfree the result or not!
*
* Note: we assume that conversion cannot cause more than a 4-to-1 growth
* in the length of the string --- is this enough?
*/
unsigned char *
pg_server_to_client(unsigned char *s, int len)
{
static unsigned char b1[MAX_PARSE_BUFFER * 4]; /* is this enough? */
static unsigned char b2[MAX_PARSE_BUFFER * 4]; /* is this enough? */
unsigned char *p = s;
unsigned char *result = s;
unsigned char *buf;
if (client_encoding == GetDatabaseEncoding())
return (p);
return result;
if (server_to_mic)
{
(*server_to_mic) (s, b1, len);
len = strlen(b1);
p = b1;
buf = (unsigned char *) palloc(len * 4 + 1);
(*server_to_mic) (result, buf, len);
result = buf;
len = strlen(result);
}
if (client_from_mic)
{
(*client_from_mic) (p, b2, len);
p = b2;
buf = (unsigned char *) palloc(len * 4 + 1);
(*client_from_mic) (result, buf, len);
if (result != s)
pfree(result); /* release first buffer */
result = buf;
}
return (p);
return result;
}
/* convert a multi-byte string to a wchar */

8
src/include/config.h.in
Unescape View File

@ -123,14 +123,6 @@
*/
/* #define TCL_ARRAYS */
/*
* The following flag allows copying tables from files with number of columns
* different than the number of attributes setting missing attributes to NULL
* and ignoring extra columns. This also avoids the shift of the attributes
* of the rest of the file if one line has a wrong column count.
*/
#define COPY_PATCH
/*
* User locks are handled totally on the application side as long term
* cooperative locks which extend beyond the normal transaction boundaries.

Write Preview
Loading…
Cancel
Save