mirror of https://github.com/postgres/postgres
Previously, ScanKeywordLookup was passed an array of string pointers. This had some performance deficiencies: the strings themselves might be scattered all over the place depending on the compiler (and some quick checking shows that at least with gcc-on-Linux, they indeed weren't reliably close together). That led to very cache-unfriendly behavior as the binary search touched strings in many different pages. Also, depending on the platform, the string pointers might need to be adjusted at program start, so that they couldn't be simple constant data. And the ScanKeyword struct had been designed with an eye to 32-bit machines originally; on 64-bit it requires 16 bytes per keyword, making it even more cache-unfriendly. Redesign so that the keyword strings themselves are allocated consecutively (as part of one big char-string constant), thereby eliminating the touch-lots-of-unrelated-pages syndrome. And get rid of the ScanKeyword array in favor of three separate arrays: uint16 offsets into the keyword array, uint16 token codes, and uint8 keyword categories. That reduces the overhead per keyword to 5 bytes instead of 16 (even less in programs that only need one of the token codes and categories); moreover, the binary search only touches the offsets array, further reducing its cache footprint. This also lets us put the token codes somewhere else than the keyword strings are, which avoids some unpleasant build dependencies. While we're at it, wrap the data used by ScanKeywordLookup into a struct that can be treated as an opaque type by most callers. That doesn't change things much right now, but it will make it less painful to switch to a hash-based lookup method, as is being discussed in the mailing list thread. Most of the change here is associated with adding a generator script that can build the new data structure from the same list-of-PG_KEYWORD header representation we used before. The PG_KEYWORD lists that plpgsql and ecpg used to embed in their scanner .c files have to be moved into headers, and the Makefiles have to be taught to invoke the generator script. This work is also necessary if we're to consider hash-based lookup, since the generator script is what would be responsible for constructing a hash table. Aside from saving a few kilobytes in each program that includes the keyword table, this seems to speed up raw parsing (flex+bison) by a few percent. So it's worth doing even as it stands, though we think we can gain even more with a follow-on patch to switch to hash-based lookup. John Naylor, with further hacking by me Discussion: https://postgr.es/m/CAJVSVGXdFVU2sgym89XPL=Lv1zOS5=EHHQ8XWNzFL=mTXkKMLw@mail.gmail.compull/36/head
Tom Lane
7 years ago
parent
c5c7fa261f
commit
afb0d0712f
@ -0,0 +1 @@ |
||||
/kwlist_d.h |
||||
@ -0,0 +1,94 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* kwlookup.c |
||||
* Key word lookup for PostgreSQL |
||||
* |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* |
||||
* IDENTIFICATION |
||||
* src/common/kwlookup.c |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#include "c.h" |
||||
|
||||
#include "common/kwlookup.h" |
||||
|
||||
|
||||
/*
|
||||
* ScanKeywordLookup - see if a given word is a keyword |
||||
* |
||||
* The list of keywords to be matched against is passed as a ScanKeywordList. |
||||
* |
||||
* Returns the keyword number (0..N-1) of the keyword, or -1 if no match. |
||||
* Callers typically use the keyword number to index into information |
||||
* arrays, but that is no concern of this code. |
||||
* |
||||
* The match is done case-insensitively. Note that we deliberately use a |
||||
* dumbed-down case conversion that will only translate 'A'-'Z' into 'a'-'z', |
||||
* even if we are in a locale where tolower() would produce more or different |
||||
* translations. This is to conform to the SQL99 spec, which says that |
||||
* keywords are to be matched in this way even though non-keyword identifiers |
||||
* receive a different case-normalization mapping. |
||||
*/ |
||||
int |
||||
ScanKeywordLookup(const char *text, |
||||
const ScanKeywordList *keywords) |
||||
{ |
||||
int len, |
||||
i; |
||||
char word[NAMEDATALEN]; |
||||
const char *kw_string; |
||||
const uint16 *kw_offsets; |
||||
const uint16 *low; |
||||
const uint16 *high; |
||||
|
||||
len = strlen(text); |
||||
|
||||
if (len > keywords->max_kw_len) |
||||
return -1; /* too long to be any keyword */ |
||||
|
||||
/* We assume all keywords are shorter than NAMEDATALEN. */ |
||||
Assert(len < NAMEDATALEN); |
||||
|
||||
/*
|
||||
* Apply an ASCII-only downcasing. We must not use tolower() since it may |
||||
* produce the wrong translation in some locales (eg, Turkish). |
||||
*/ |
||||
for (i = 0; i < len; i++) |
||||
{ |
||||
char ch = text[i]; |
||||
|
||||
if (ch >= 'A' && ch <= 'Z') |
||||
ch += 'a' - 'A'; |
||||
word[i] = ch; |
||||
} |
||||
word[len] = '\0'; |
||||
|
||||
/*
|
||||
* Now do a binary search using plain strcmp() comparison. |
||||
*/ |
||||
kw_string = keywords->kw_string; |
||||
kw_offsets = keywords->kw_offsets; |
||||
low = kw_offsets; |
||||
high = kw_offsets + (keywords->num_keywords - 1); |
||||
while (low <= high) |
||||
{ |
||||
const uint16 *middle; |
||||
int difference; |
||||
|
||||
middle = low + (high - low) / 2; |
||||
difference = strcmp(kw_string + *middle, word); |
||||
if (difference == 0) |
||||
return middle - kw_offsets; |
||||
else if (difference < 0) |
||||
low = middle + 1; |
||||
else |
||||
high = middle - 1; |
||||
} |
||||
|
||||
return -1; |
||||
} |
||||
@ -0,0 +1,40 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* kwlookup.h |
||||
* Key word lookup for PostgreSQL |
||||
* |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* src/include/common/kwlookup.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
#ifndef KWLOOKUP_H |
||||
#define KWLOOKUP_H |
||||
|
||||
/*
|
||||
* This struct contains the data needed by ScanKeywordLookup to perform a |
||||
* search within a set of keywords. The contents are typically generated by |
||||
* src/tools/gen_keywordlist.pl from a header containing PG_KEYWORD macros. |
||||
*/ |
||||
typedef struct ScanKeywordList |
||||
{ |
||||
const char *kw_string; /* all keywords in order, separated by \0 */ |
||||
const uint16 *kw_offsets; /* offsets to the start of each keyword */ |
||||
int num_keywords; /* number of keywords */ |
||||
int max_kw_len; /* length of longest keyword */ |
||||
} ScanKeywordList; |
||||
|
||||
|
||||
extern int ScanKeywordLookup(const char *text, const ScanKeywordList *keywords); |
||||
|
||||
/* Code that wants to retrieve the text of the N'th keyword should use this. */ |
||||
static inline const char * |
||||
GetScanKeyword(int n, const ScanKeywordList *keywords) |
||||
{ |
||||
return keywords->kw_string + keywords->kw_offsets[n]; |
||||
} |
||||
|
||||
#endif /* KWLOOKUP_H */ |
||||
@ -0,0 +1,53 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* c_kwlist.h |
||||
* |
||||
* The keyword lists are kept in their own source files for use by |
||||
* automatic tools. The exact representation of a keyword is determined |
||||
* by the PG_KEYWORD macro, which is not defined in this file; it can |
||||
* be defined by the caller for special purposes. |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* src/interfaces/ecpg/preproc/c_kwlist.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/* There is deliberately not an #ifndef C_KWLIST_H here. */ |
||||
|
||||
/*
|
||||
* List of (keyword-name, keyword-token-value) pairs. |
||||
* |
||||
* !!WARNING!!: This list must be sorted by ASCII name, because binary |
||||
* search is used to locate entries. |
||||
*/ |
||||
|
||||
/* name, value */ |
||||
PG_KEYWORD("VARCHAR", VARCHAR) |
||||
PG_KEYWORD("auto", S_AUTO) |
||||
PG_KEYWORD("bool", SQL_BOOL) |
||||
PG_KEYWORD("char", CHAR_P) |
||||
PG_KEYWORD("const", S_CONST) |
||||
PG_KEYWORD("enum", ENUM_P) |
||||
PG_KEYWORD("extern", S_EXTERN) |
||||
PG_KEYWORD("float", FLOAT_P) |
||||
PG_KEYWORD("hour", HOUR_P) |
||||
PG_KEYWORD("int", INT_P) |
||||
PG_KEYWORD("long", SQL_LONG) |
||||
PG_KEYWORD("minute", MINUTE_P) |
||||
PG_KEYWORD("month", MONTH_P) |
||||
PG_KEYWORD("register", S_REGISTER) |
||||
PG_KEYWORD("second", SECOND_P) |
||||
PG_KEYWORD("short", SQL_SHORT) |
||||
PG_KEYWORD("signed", SQL_SIGNED) |
||||
PG_KEYWORD("static", S_STATIC) |
||||
PG_KEYWORD("struct", SQL_STRUCT) |
||||
PG_KEYWORD("to", TO) |
||||
PG_KEYWORD("typedef", S_TYPEDEF) |
||||
PG_KEYWORD("union", UNION) |
||||
PG_KEYWORD("unsigned", SQL_UNSIGNED) |
||||
PG_KEYWORD("varchar", VARCHAR) |
||||
PG_KEYWORD("volatile", S_VOLATILE) |
||||
PG_KEYWORD("year", YEAR_P) |
||||
@ -0,0 +1,68 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* ecpg_kwlist.h |
||||
* |
||||
* The keyword lists are kept in their own source files for use by |
||||
* automatic tools. The exact representation of a keyword is determined |
||||
* by the PG_KEYWORD macro, which is not defined in this file; it can |
||||
* be defined by the caller for special purposes. |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* src/interfaces/ecpg/preproc/ecpg_kwlist.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/* There is deliberately not an #ifndef ECPG_KWLIST_H here. */ |
||||
|
||||
/*
|
||||
* List of (keyword-name, keyword-token-value) pairs. |
||||
* |
||||
* !!WARNING!!: This list must be sorted by ASCII name, because binary |
||||
* search is used to locate entries. |
||||
*/ |
||||
|
||||
/* name, value */ |
||||
PG_KEYWORD("allocate", SQL_ALLOCATE) |
||||
PG_KEYWORD("autocommit", SQL_AUTOCOMMIT) |
||||
PG_KEYWORD("bool", SQL_BOOL) |
||||
PG_KEYWORD("break", SQL_BREAK) |
||||
PG_KEYWORD("cardinality", SQL_CARDINALITY) |
||||
PG_KEYWORD("connect", SQL_CONNECT) |
||||
PG_KEYWORD("count", SQL_COUNT) |
||||
PG_KEYWORD("datetime_interval_code", SQL_DATETIME_INTERVAL_CODE) |
||||
PG_KEYWORD("datetime_interval_precision", SQL_DATETIME_INTERVAL_PRECISION) |
||||
PG_KEYWORD("describe", SQL_DESCRIBE) |
||||
PG_KEYWORD("descriptor", SQL_DESCRIPTOR) |
||||
PG_KEYWORD("disconnect", SQL_DISCONNECT) |
||||
PG_KEYWORD("found", SQL_FOUND) |
||||
PG_KEYWORD("free", SQL_FREE) |
||||
PG_KEYWORD("get", SQL_GET) |
||||
PG_KEYWORD("go", SQL_GO) |
||||
PG_KEYWORD("goto", SQL_GOTO) |
||||
PG_KEYWORD("identified", SQL_IDENTIFIED) |
||||
PG_KEYWORD("indicator", SQL_INDICATOR) |
||||
PG_KEYWORD("key_member", SQL_KEY_MEMBER) |
||||
PG_KEYWORD("length", SQL_LENGTH) |
||||
PG_KEYWORD("long", SQL_LONG) |
||||
PG_KEYWORD("nullable", SQL_NULLABLE) |
||||
PG_KEYWORD("octet_length", SQL_OCTET_LENGTH) |
||||
PG_KEYWORD("open", SQL_OPEN) |
||||
PG_KEYWORD("output", SQL_OUTPUT) |
||||
PG_KEYWORD("reference", SQL_REFERENCE) |
||||
PG_KEYWORD("returned_length", SQL_RETURNED_LENGTH) |
||||
PG_KEYWORD("returned_octet_length", SQL_RETURNED_OCTET_LENGTH) |
||||
PG_KEYWORD("scale", SQL_SCALE) |
||||
PG_KEYWORD("section", SQL_SECTION) |
||||
PG_KEYWORD("short", SQL_SHORT) |
||||
PG_KEYWORD("signed", SQL_SIGNED) |
||||
PG_KEYWORD("sqlerror", SQL_SQLERROR) |
||||
PG_KEYWORD("sqlprint", SQL_SQLPRINT) |
||||
PG_KEYWORD("sqlwarning", SQL_SQLWARNING) |
||||
PG_KEYWORD("stop", SQL_STOP) |
||||
PG_KEYWORD("struct", SQL_STRUCT) |
||||
PG_KEYWORD("unsigned", SQL_UNSIGNED) |
||||
PG_KEYWORD("var", SQL_VAR) |
||||
PG_KEYWORD("whenever", SQL_WHENEVER) |
||||
@ -0,0 +1,53 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* pl_reserved_kwlist.h |
||||
* |
||||
* The keyword lists are kept in their own source files for use by |
||||
* automatic tools. The exact representation of a keyword is determined |
||||
* by the PG_KEYWORD macro, which is not defined in this file; it can |
||||
* be defined by the caller for special purposes. |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* src/pl/plpgsql/src/pl_reserved_kwlist.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/* There is deliberately not an #ifndef PL_RESERVED_KWLIST_H here. */ |
||||
|
||||
/*
|
||||
* List of (keyword-name, keyword-token-value) pairs. |
||||
* |
||||
* Be careful not to put the same word in both lists. |
||||
* |
||||
* !!WARNING!!: This list must be sorted by ASCII name, because binary |
||||
* search is used to locate entries. |
||||
*/ |
||||
|
||||
/* name, value */ |
||||
PG_KEYWORD("all", K_ALL) |
||||
PG_KEYWORD("begin", K_BEGIN) |
||||
PG_KEYWORD("by", K_BY) |
||||
PG_KEYWORD("case", K_CASE) |
||||
PG_KEYWORD("declare", K_DECLARE) |
||||
PG_KEYWORD("else", K_ELSE) |
||||
PG_KEYWORD("end", K_END) |
||||
PG_KEYWORD("execute", K_EXECUTE) |
||||
PG_KEYWORD("for", K_FOR) |
||||
PG_KEYWORD("foreach", K_FOREACH) |
||||
PG_KEYWORD("from", K_FROM) |
||||
PG_KEYWORD("if", K_IF) |
||||
PG_KEYWORD("in", K_IN) |
||||
PG_KEYWORD("into", K_INTO) |
||||
PG_KEYWORD("loop", K_LOOP) |
||||
PG_KEYWORD("not", K_NOT) |
||||
PG_KEYWORD("null", K_NULL) |
||||
PG_KEYWORD("or", K_OR) |
||||
PG_KEYWORD("strict", K_STRICT) |
||||
PG_KEYWORD("then", K_THEN) |
||||
PG_KEYWORD("to", K_TO) |
||||
PG_KEYWORD("using", K_USING) |
||||
PG_KEYWORD("when", K_WHEN) |
||||
PG_KEYWORD("while", K_WHILE) |
||||
@ -0,0 +1,111 @@ |
||||
/*-------------------------------------------------------------------------
|
||||
* |
||||
* pl_unreserved_kwlist.h |
||||
* |
||||
* The keyword lists are kept in their own source files for use by |
||||
* automatic tools. The exact representation of a keyword is determined |
||||
* by the PG_KEYWORD macro, which is not defined in this file; it can |
||||
* be defined by the caller for special purposes. |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* src/pl/plpgsql/src/pl_unreserved_kwlist.h |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
/* There is deliberately not an #ifndef PL_UNRESERVED_KWLIST_H here. */ |
||||
|
||||
/*
|
||||
* List of (keyword-name, keyword-token-value) pairs. |
||||
* |
||||
* Be careful not to put the same word in both lists. Also be sure that |
||||
* pl_gram.y's unreserved_keyword production agrees with this list. |
||||
* |
||||
* !!WARNING!!: This list must be sorted by ASCII name, because binary |
||||
* search is used to locate entries. |
||||
*/ |
||||
|
||||
/* name, value */ |
||||
PG_KEYWORD("absolute", K_ABSOLUTE) |
||||
PG_KEYWORD("alias", K_ALIAS) |
||||
PG_KEYWORD("array", K_ARRAY) |
||||
PG_KEYWORD("assert", K_ASSERT) |
||||
PG_KEYWORD("backward", K_BACKWARD) |
||||
PG_KEYWORD("call", K_CALL) |
||||
PG_KEYWORD("close", K_CLOSE) |
||||
PG_KEYWORD("collate", K_COLLATE) |
||||
PG_KEYWORD("column", K_COLUMN) |
||||
PG_KEYWORD("column_name", K_COLUMN_NAME) |
||||
PG_KEYWORD("commit", K_COMMIT) |
||||
PG_KEYWORD("constant", K_CONSTANT) |
||||
PG_KEYWORD("constraint", K_CONSTRAINT) |
||||
PG_KEYWORD("constraint_name", K_CONSTRAINT_NAME) |
||||
PG_KEYWORD("continue", K_CONTINUE) |
||||
PG_KEYWORD("current", K_CURRENT) |
||||
PG_KEYWORD("cursor", K_CURSOR) |
||||
PG_KEYWORD("datatype", K_DATATYPE) |
||||
PG_KEYWORD("debug", K_DEBUG) |
||||
PG_KEYWORD("default", K_DEFAULT) |
||||
PG_KEYWORD("detail", K_DETAIL) |
||||
PG_KEYWORD("diagnostics", K_DIAGNOSTICS) |
||||
PG_KEYWORD("do", K_DO) |
||||
PG_KEYWORD("dump", K_DUMP) |
||||
PG_KEYWORD("elseif", K_ELSIF) |
||||
PG_KEYWORD("elsif", K_ELSIF) |
||||
PG_KEYWORD("errcode", K_ERRCODE) |
||||
PG_KEYWORD("error", K_ERROR) |
||||
PG_KEYWORD("exception", K_EXCEPTION) |
||||
PG_KEYWORD("exit", K_EXIT) |
||||
PG_KEYWORD("fetch", K_FETCH) |
||||
PG_KEYWORD("first", K_FIRST) |
||||
PG_KEYWORD("forward", K_FORWARD) |
||||
PG_KEYWORD("get", K_GET) |
||||
PG_KEYWORD("hint", K_HINT) |
||||
PG_KEYWORD("import", K_IMPORT) |
||||
PG_KEYWORD("info", K_INFO) |
||||
PG_KEYWORD("insert", K_INSERT) |
||||
PG_KEYWORD("is", K_IS) |
||||
PG_KEYWORD("last", K_LAST) |
||||
PG_KEYWORD("log", K_LOG) |
||||
PG_KEYWORD("message", K_MESSAGE) |
||||
PG_KEYWORD("message_text", K_MESSAGE_TEXT) |
||||
PG_KEYWORD("move", K_MOVE) |
||||
PG_KEYWORD("next", K_NEXT) |
||||
PG_KEYWORD("no", K_NO) |
||||
PG_KEYWORD("notice", K_NOTICE) |
||||
PG_KEYWORD("open", K_OPEN) |
||||
PG_KEYWORD("option", K_OPTION) |
||||
PG_KEYWORD("perform", K_PERFORM) |
||||
PG_KEYWORD("pg_context", K_PG_CONTEXT) |
||||
PG_KEYWORD("pg_datatype_name", K_PG_DATATYPE_NAME) |
||||
PG_KEYWORD("pg_exception_context", K_PG_EXCEPTION_CONTEXT) |
||||
PG_KEYWORD("pg_exception_detail", K_PG_EXCEPTION_DETAIL) |
||||
PG_KEYWORD("pg_exception_hint", K_PG_EXCEPTION_HINT) |
||||
PG_KEYWORD("print_strict_params", K_PRINT_STRICT_PARAMS) |
||||
PG_KEYWORD("prior", K_PRIOR) |
||||
PG_KEYWORD("query", K_QUERY) |
||||
PG_KEYWORD("raise", K_RAISE) |
||||
PG_KEYWORD("relative", K_RELATIVE) |
||||
PG_KEYWORD("reset", K_RESET) |
||||
PG_KEYWORD("return", K_RETURN) |
||||
PG_KEYWORD("returned_sqlstate", K_RETURNED_SQLSTATE) |
||||
PG_KEYWORD("reverse", K_REVERSE) |
||||
PG_KEYWORD("rollback", K_ROLLBACK) |
||||
PG_KEYWORD("row_count", K_ROW_COUNT) |
||||
PG_KEYWORD("rowtype", K_ROWTYPE) |
||||
PG_KEYWORD("schema", K_SCHEMA) |
||||
PG_KEYWORD("schema_name", K_SCHEMA_NAME) |
||||
PG_KEYWORD("scroll", K_SCROLL) |
||||
PG_KEYWORD("set", K_SET) |
||||
PG_KEYWORD("slice", K_SLICE) |
||||
PG_KEYWORD("sqlstate", K_SQLSTATE) |
||||
PG_KEYWORD("stacked", K_STACKED) |
||||
PG_KEYWORD("table", K_TABLE) |
||||
PG_KEYWORD("table_name", K_TABLE_NAME) |
||||
PG_KEYWORD("type", K_TYPE) |
||||
PG_KEYWORD("use_column", K_USE_COLUMN) |
||||
PG_KEYWORD("use_variable", K_USE_VARIABLE) |
||||
PG_KEYWORD("variable_conflict", K_VARIABLE_CONFLICT) |
||||
PG_KEYWORD("warning", K_WARNING) |
||||
@ -0,0 +1,156 @@ |
||||
#---------------------------------------------------------------------- |
||||
# |
||||
# gen_keywordlist.pl |
||||
# Perl script that transforms a list of keywords into a ScanKeywordList |
||||
# data structure that can be passed to ScanKeywordLookup(). |
||||
# |
||||
# The input is a C header file containing a series of macro calls |
||||
# PG_KEYWORD("keyword", ...) |
||||
# Lines not starting with PG_KEYWORD are ignored. The keywords are |
||||
# implicitly numbered 0..N-1 in order of appearance in the header file. |
||||
# Currently, the keywords are required to appear in ASCII order. |
||||
# |
||||
# The output is a C header file that defines a "const ScanKeywordList" |
||||
# variable named according to the -v switch ("ScanKeywords" by default). |
||||
# The variable is marked "static" unless the -e switch is given. |
||||
# |
||||
# |
||||
# Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
# Portions Copyright (c) 1994, Regents of the University of California |
||||
# |
||||
# src/tools/gen_keywordlist.pl |
||||
# |
||||
#---------------------------------------------------------------------- |
||||
|
||||
use strict; |
||||
use warnings; |
||||
use Getopt::Long; |
||||
|
||||
my $output_path = ''; |
||||
my $extern = 0; |
||||
my $varname = 'ScanKeywords'; |
||||
|
||||
GetOptions( |
||||
'output:s' => \$output_path, |
||||
'extern' => \$extern, |
||||
'varname:s' => \$varname) || usage(); |
||||
|
||||
my $kw_input_file = shift @ARGV || die "No input file.\n"; |
||||
|
||||
# Make sure output_path ends in a slash if needed. |
||||
if ($output_path ne '' && substr($output_path, -1) ne '/') |
||||
{ |
||||
$output_path .= '/'; |
||||
} |
||||
|
||||
$kw_input_file =~ /(\w+)\.h$/ || die "Input file must be named something.h.\n"; |
||||
my $base_filename = $1 . '_d'; |
||||
my $kw_def_file = $output_path . $base_filename . '.h'; |
||||
|
||||
open(my $kif, '<', $kw_input_file) || die "$kw_input_file: $!\n"; |
||||
open(my $kwdef, '>', $kw_def_file) || die "$kw_def_file: $!\n"; |
||||
|
||||
# Opening boilerplate for keyword definition header. |
||||
printf $kwdef <<EOM, $base_filename, uc $base_filename, uc $base_filename; |
||||
/*------------------------------------------------------------------------- |
||||
* |
||||
* %s.h |
||||
* List of keywords represented as a ScanKeywordList. |
||||
* |
||||
* Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group |
||||
* Portions Copyright (c) 1994, Regents of the University of California |
||||
* |
||||
* NOTES |
||||
* ****************************** |
||||
* *** DO NOT EDIT THIS FILE! *** |
||||
* ****************************** |
||||
* |
||||
* It has been GENERATED by src/tools/gen_keywordlist.pl |
||||
* |
||||
*------------------------------------------------------------------------- |
||||
*/ |
||||
|
||||
#ifndef %s_H |
||||
#define %s_H |
||||
|
||||
#include "common/kwlookup.h" |
||||
|
||||
EOM |
||||
|
||||
# Parse input file for keyword names. |
||||
my @keywords; |
||||
while (<$kif>) |
||||
{ |
||||
if (/^PG_KEYWORD\("(\w+)"/) |
||||
{ |
||||
push @keywords, $1; |
||||
} |
||||
} |
||||
|
||||
# Error out if the keyword names are not in ASCII order. |
||||
for my $i (0..$#keywords - 1) |
||||
{ |
||||
die qq|The keyword "$keywords[$i + 1]" is out of order in $kw_input_file\n| |
||||
if ($keywords[$i] cmp $keywords[$i + 1]) >= 0; |
||||
} |
||||
|
||||
# Emit the string containing all the keywords. |
||||
|
||||
printf $kwdef qq|static const char %s_kw_string[] =\n\t"|, $varname; |
||||
print $kwdef join qq|\\0"\n\t"|, @keywords; |
||||
print $kwdef qq|";\n\n|; |
||||
|
||||
# Emit an array of numerical offsets which will be used to index into the |
||||
# keyword string. Also determine max keyword length. |
||||
|
||||
printf $kwdef "static const uint16 %s_kw_offsets[] = {\n", $varname; |
||||
|
||||
my $offset = 0; |
||||
my $max_len = 0; |
||||
foreach my $name (@keywords) |
||||
{ |
||||
my $this_length = length($name); |
||||
|
||||
print $kwdef "\t$offset,\n"; |
||||
|
||||
# Calculate the cumulative offset of the next keyword, |
||||
# taking into account the null terminator. |
||||
$offset += $this_length + 1; |
||||
|
||||
# Update max keyword length. |
||||
$max_len = $this_length if $max_len < $this_length; |
||||
} |
||||
|
||||
print $kwdef "};\n\n"; |
||||
|
||||
# Emit a macro defining the number of keywords. |
||||
# (In some places it's useful to have access to that as a constant.) |
||||
|
||||
printf $kwdef "#define %s_NUM_KEYWORDS %d\n\n", uc $varname, scalar @keywords; |
||||
|
||||
# Emit the struct that wraps all this lookup info into one variable. |
||||
|
||||
print $kwdef "static " if !$extern; |
||||
printf $kwdef "const ScanKeywordList %s = {\n", $varname; |
||||
printf $kwdef qq|\t%s_kw_string,\n|, $varname; |
||||
printf $kwdef qq|\t%s_kw_offsets,\n|, $varname; |
||||
printf $kwdef qq|\t%s_NUM_KEYWORDS,\n|, uc $varname; |
||||
printf $kwdef qq|\t%d\n|, $max_len; |
||||
print $kwdef "};\n\n"; |
||||
|
||||
printf $kwdef "#endif\t\t\t\t\t\t\t/* %s_H */\n", uc $base_filename; |
||||
|
||||
|
||||
sub usage |
||||
{ |
||||
die <<EOM; |
||||
Usage: gen_keywordlist.pl [--output/-o <path>] [--varname/-v <varname>] [--extern/-e] input_file |
||||
--output Output directory (default '.') |
||||
--varname Name for ScanKeywordList variable (default 'ScanKeywords') |
||||
--extern Allow the ScanKeywordList variable to be globally visible |
||||
|
||||
gen_keywordlist.pl transforms a list of keywords into a ScanKeywordList. |
||||
The output filename is derived from the input file by inserting _d, |
||||
for example kwlist_d.h is produced from kwlist.h. |
||||
EOM |
||||
} |
||||
Loading…
Reference in new issue