open-dsi
/
postgres
mirror of https://github.com/postgres/postgres
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle carriage returns and line feeds in COPY CSV mode.

Browse Source 
Andrew Dunstan
REL8_1_STABLE
Bruce Momjian 21 years ago
parent 06a61d66f9
commit 5fdd9418ee
1 changed files with 120 additions and 76 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 196
      src/backend/commands/copy.c

196
src/backend/commands/copy.c
Unescape View File

@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.236 2004/12/31 21:59:41 pgsql Exp $
* $PostgreSQL: pgsql/src/backend/commands/copy.c,v 1.237 2005/03/12 05:41:34 momjian Exp $
*
*-------------------------------------------------------------------------
*/
@ -98,7 +98,6 @@ static bool fe_eof; /* true if detected end of copy data */
static EolType eol_type; /* EOL type of input */
static int client_encoding; /* remote side's character encoding */
static int server_encoding; /* local encoding */
static bool embedded_line_warning;
/* these are just for error messages, see copy_in_error_callback */
static bool copy_binary; /* is it a binary copy? */
@ -139,7 +138,7 @@ static void CopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
static void CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
char *delim, char *null_print, bool csv_mode, char *quote, char *escape,
List *force_notnull_atts);
static bool CopyReadLine(void);
static bool CopyReadLine(char * quote, char * escape);
static char *CopyReadAttribute(const char *delim, const char *null_print,
CopyReadResult *result, bool *isnull);
static char *CopyReadAttributeCSV(const char *delim, const char *null_print,
@ -1191,7 +1190,6 @@ CopyTo(Relation rel, List *attnumlist, bool binary, bool oids,
attr = tupDesc->attrs;
num_phys_attrs = tupDesc->natts;
attr_count = list_length(attnumlist);
embedded_line_warning = false;
/*
* Get info about the columns we need to process.
@ -1718,7 +1716,8 @@ CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
ListCell *cur;
/* Actually read the line into memory here */
done = CopyReadLine();
done = csv_mode ?
CopyReadLine(quote, escape) : CopyReadLine(NULL, NULL);
/*
* EOF at start of line means we're done. If we see EOF after
@ -2006,7 +2005,7 @@ CopyFrom(Relation rel, List *attnumlist, bool binary, bool oids,
* by newline.
*/
static bool
CopyReadLine(void)
CopyReadLine(char * quote, char * escape)
{
bool result;
bool change_encoding = (client_encoding != server_encoding);
@ -2015,6 +2014,19 @@ CopyReadLine(void)
int j;
unsigned char s[2];
char *cvt;
bool in_quote = false, last_was_esc = false, csv_mode = false;
char quotec = '\0', escapec = '\0';
if (quote)
{
csv_mode = true;
quotec = quote[0];
escapec = escape[0];
/* ignore special escape processing if it's the same as quotec */
if (quotec == escapec)
escapec = '\0';
}
s[1] = 0;
@ -2031,11 +2043,20 @@ CopyReadLine(void)
/*
* In this loop we only care for detecting newlines (\r and/or \n) and
* the end-of-copy marker (\.). For backwards compatibility we allow
* the end-of-copy marker (\.).
*
* In Text mode, for backwards compatibility we allow
* backslashes to escape newline characters. Backslashes other than
* the end marker get put into the line_buf, since CopyReadAttribute
* does its own escape processing. These four characters, and only
* these four, are assumed the same in frontend and backend encodings.
* does its own escape processing.
*
* In CSV mode, CR and NL inside q quoted field are just part of the
* data value and are put in line_buf. We keep just enough state
* to know if we are currently in a quoted field or not.
*
* These four characters, and only these four, are assumed the same in
* frontend and backend encodings.
*
* We do not assume that second and later bytes of a frontend
* multibyte character couldn't look like ASCII characters.
*/
@ -2047,13 +2068,49 @@ CopyReadLine(void)
result = true;
break;
}
if (c == '\r')
if (csv_mode)
{
/*
* Dealing with quotes and escapes here is mildly tricky. If the
* quote char is also the escape char, there's no problem - we
* just use the char as a toggle. If they are different, we need
* to ensure that we only take account of an escape inside a quoted
* field and immediately preceding a quote char, and not the
* second in a escape-escape sequence.
*/
if (in_quote && c == escapec)
last_was_esc = ! last_was_esc;
if (c == quotec && ! last_was_esc)
in_quote = ! in_quote;
if (c != escapec)
last_was_esc = false;
/*
* updating the line count for embedded CR and/or LF chars is
* necessarily a little fragile - this test is probably about
* the best we can do.
*/
if (in_quote && c == (eol_type == EOL_CR ? '\r' : '\n'))
copy_lineno++;
}
if (!in_quote && c == '\r')
{
if (eol_type == EOL_NL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal carriage return found in data"),
errhint("Use \"\\r\" to represent carriage return.")));
{
if (! csv_mode)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal carriage return found in data"),
errhint("Use \"\\r\" to represent carriage return.")));
else
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unquoted carriage return found in CSV data"),
errhint("Use quoted CSV field to represent carriage return.")));
}
/* Check for \r\n on first line, _and_ handle \r\n. */
if (eol_type == EOL_UNKNOWN || eol_type == EOL_CRNL)
{
@ -2068,10 +2125,19 @@ CopyReadLine(void)
{
/* found \r, but no \n */
if (eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal carriage return found in data"),
errhint("Use \"\\r\" to represent carriage return.")));
{
if (!csv_mode)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal carriage return found in data"),
errhint("Use \"\\r\" to represent carriage return.")));
else
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unquoted carriage return found in data"),
errhint("Use quoted CSV field to represent carriage return.")));
}
/*
* if we got here, it is the first line and we didn't
@ -2083,26 +2149,47 @@ CopyReadLine(void)
}
break;
}
if (c == '\n')
if (!in_quote && c == '\n')
{
if (eol_type == EOL_CR || eol_type == EOL_CRNL)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal newline found in data"),
errhint("Use \"\\n\" to represent newline.")));
{
if (!csv_mode)
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("literal newline found in data"),
errhint("Use \"\\n\" to represent newline.")));
else
ereport(ERROR,
(errcode(ERRCODE_BAD_COPY_FILE_FORMAT),
errmsg("unquoted newline found in data"),
errhint("Use quoted CSV field to represent newline.")));
}
eol_type = EOL_NL;
break;
}
if (c == '\\')
if ((line_buf.len == 0 || !csv_mode) && c == '\\')
{
c = CopyGetChar();
if (c == EOF)
int c2;
if (csv_mode)
c2 = CopyPeekChar();
else
c2 = c = CopyGetChar();
if (c2 == EOF)
{
result = true;
if (csv_mode)
CopyDonePeek(c2, true);
break;
}
if (c == '.')
if (c2 == '.')
{
if (csv_mode)
CopyDonePeek(c2, true); /* allow keep calling GetChar() */
if (eol_type == EOL_CRNL)
{
c = CopyGetChar();
@ -2140,8 +2227,12 @@ CopyReadLine(void)
result = true; /* report EOF */
break;
}
/* not EOF mark, so emit \ and following char literally */
appendStringInfoCharMacro(&line_buf, '\\');
if (csv_mode)
CopyDonePeek(c2, false); /* not a dot, so put it back */
else
/* not EOF mark, so emit \ and following char literally */
appendStringInfoCharMacro(&line_buf, '\\');
}
appendStringInfoCharMacro(&line_buf, c);
@ -2369,34 +2460,6 @@ CopyReadAttributeCSV(const char *delim, const char *null_print, char *quote,
for (;;)
{
/* handle multiline quoted fields */
if (in_quote && line_buf.cursor >= line_buf.len)
{
bool done;
switch (eol_type)
{
case EOL_NL:
appendStringInfoString(&attribute_buf, "\n");
break;
case EOL_CR:
appendStringInfoString(&attribute_buf, "\r");
break;
case EOL_CRNL:
appendStringInfoString(&attribute_buf, "\r\n");
break;
case EOL_UNKNOWN:
/* shouldn't happen - just keep going */
break;
}
copy_lineno++;
done = CopyReadLine();
if (done && line_buf.len == 0)
break;
start_cursor = line_buf.cursor;
}
end_cursor = line_buf.cursor;
if (line_buf.cursor >= line_buf.len)
break;
@ -2629,25 +2692,6 @@ CopyAttributeOutCSV(char *server_string, char *delim, char *quote,
!use_quote && (c = *test_string) != '\0';
test_string += mblen)
{
/*
* We don't know here what the surrounding line end characters
* might be. It might not even be under postgres' control. So
* we simple warn on ANY embedded line ending character.
*
* This warning will disappear when we make line parsing field-aware,
* so that we can reliably read in embedded line ending characters
* regardless of the file's line-end context.
*
*/
if (!embedded_line_warning && (c == '\n' || c == '\r') )
{
embedded_line_warning = true;
elog(WARNING,
"CSV fields with embedded linefeed or carriage return "
"characters might not be able to be reimported");
}
if (c == delimc || c == quotec || c == '\n' || c == '\r')
use_quote = true;
if (!same_encoding)

Write Preview
Loading…
Cancel
Save