Move psql's psqlscan.l into src/fe_utils.

This completes (at least for now) the project of getting rid of ad-hoc
linkages among the src/bin/ subdirectories.  Everything they share is now
in src/fe_utils/ and is included from a static library at link time.

A side benefit is that we can restore the FLEX_NO_BACKUP check for
psqlscanslash.l.  We might need to think of another way to do that check
if we ever need to build two lexers with that property in the same source
directory, but there's no foreseeable reason to need that.

src/backend/parser/scan.l

@@ -6,7 +6,7 @@
  *
  * NOTE NOTE NOTE:
  *
- * The rules in this file must be kept in sync with psql's psqlscan.l!
+ * The rules in this file must be kept in sync with src/fe_utils/psqlscan.l!
  *
  * The rules are designed so that the scanner never has to backtrack,
  * in the sense that there is always a rule that can match the input

src/bin/pgbench/Makefile

@@ -7,10 +7,10 @@ subdir = src/bin/pgbench
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = pgbench.o exprparse.o psqlscan.o $(WIN32RES)
+OBJS = pgbench.o exprparse.o $(WIN32RES)
 
-override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) \
-	-I$(top_srcdir)/src/bin/psql $(CPPFLAGS)
+override CPPFLAGS := -I. -I$(srcdir) -I$(libpq_srcdir) $(CPPFLAGS)
+LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils
 
 ifneq ($(PORTNAME), win32)
 override CFLAGS += $(PTHREAD_CFLAGS)
@@ -19,21 +19,12 @@ endif
 
 all: pgbench
 
-pgbench: $(OBJS) | submake-libpq submake-libpgport
+pgbench: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils
 	$(CC) $(CFLAGS) $^ $(libpq_pgport) $(PTHREAD_LIBS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
 
 # exprscan is compiled as part of exprparse
 exprparse.o: exprscan.c
 
-# we import psqlscan.o as-is from psql
-submake-psqlscan:
-	$(MAKE) -C $(top_builddir)/src/bin/psql psqlscan.o
-
-psqlscan.o: | submake-psqlscan
-	rm -f $@ && $(LN_S) $(top_builddir)/src/bin/psql/psqlscan.o .
-
-.PHONY: submake-psqlscan
-
 distprep: exprparse.c exprscan.c
 
 install: all installdirs

src/bin/pgbench/exprscan.l

@@ -23,7 +23,7 @@
  *-------------------------------------------------------------------------
  */
 
-#include "psqlscan_int.h"
+#include "fe_utils/psqlscan_int.h"
 
 /* context information for reporting errors in expressions */
 static const char *expr_source = NULL;

src/bin/pgbench/pgbench.h

@@ -11,7 +11,7 @@
 #ifndef PGBENCH_H
 #define PGBENCH_H
 
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
 /*
  * This file is included outside exprscan.l, in places where we can't see

src/bin/psql/.gitignore

@@ -1,4 +1,3 @@
-/psqlscan.c
 /psqlscanslash.c
 /sql_help.h
 /sql_help.c

src/bin/psql/Makefile

@@ -24,7 +24,7 @@ LDFLAGS += -L$(top_builddir)/src/fe_utils -lpgfeutils
 OBJS=	command.o common.o help.o input.o stringutils.o mainloop.o copy.o \
 	startup.o prompt.o variables.o large_obj.o describe.o \
 	tab-complete.o \
-	sql_help.o psqlscan.o psqlscanslash.o \
+	sql_help.o psqlscanslash.o \
 	$(WIN32RES)
 
 
@@ -39,20 +39,15 @@ sql_help.c: sql_help.h ;
 sql_help.h: create_help.pl $(wildcard $(REFDOCDIR)/*.sgml)
 	$(PERL) $< $(REFDOCDIR) $*
 
-psqlscan.c: FLEXFLAGS = -Cfe -p -p
-psqlscan.c: FLEX_NO_BACKUP=yes
-
 psqlscanslash.c: FLEXFLAGS = -Cfe -p -p
-# Ideally we'd check this, but parallel make causes problems:
-# psqlscanslash.c: FLEX_NO_BACKUP=yes
+psqlscanslash.c: FLEX_NO_BACKUP=yes
 
-# Latest flex causes warnings in these files.
+# Latest flex causes warnings in this file.
 ifeq ($(GCC),yes)
-psqlscan.o: CFLAGS += -Wno-error
 psqlscanslash.o: CFLAGS += -Wno-error
 endif
 
-distprep: sql_help.h psqlscan.c psqlscanslash.c
+distprep: sql_help.h psqlscanslash.c
 
 install: all installdirs
 	$(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)'
@@ -70,4 +65,4 @@ clean distclean:
 # files removed here are supposed to be in the distribution tarball,
 # so do not clean them in the clean/distclean rules
 maintainer-clean: distclean
-	rm -f sql_help.h sql_help.c psqlscan.c psqlscanslash.c
+	rm -f sql_help.h sql_help.c psqlscanslash.c

src/bin/psql/command.h

@@ -9,7 +9,7 @@
 #define COMMAND_H
 
 #include "fe_utils/print.h"
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
 
 typedef enum _backslashResult

src/bin/psql/mainloop.c

@@ -11,6 +11,7 @@
 #include "command.h"
 #include "common.h"
 #include "input.h"
+#include "prompt.h"
 #include "settings.h"
 
 #include "mb/pg_wchar.h"

src/bin/psql/mainloop.h

@@ -8,7 +8,7 @@
 #ifndef MAINLOOP_H
 #define MAINLOOP_H
 
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
 extern const PsqlScanCallbacks psqlscan_callbacks;
 

src/bin/psql/nls.mk

@@ -2,10 +2,10 @@
 CATALOG_NAME     = psql
 AVAIL_LANGUAGES  = cs de es fr it ja pl pt_BR ru zh_CN zh_TW
 GETTEXT_FILES    = command.c common.c copy.c help.c input.c large_obj.c \
-                   mainloop.c psqlscan.c psqlscanslash.c startup.c \
+                   mainloop.c psqlscanslash.c startup.c \
                    describe.c sql_help.h sql_help.c \
                    tab-complete.c variables.c \
-                   ../../fe_utils/print.c \
+                   ../../fe_utils/print.c ../../fe_utils/psqlscan.c \
                    ../../common/exec.c ../../common/fe_memutils.c ../../common/username.c \
                    ../../common/wait_error.c
 GETTEXT_TRIGGERS = N_ psql_error simple_prompt

src/bin/psql/prompt.h

@@ -8,17 +8,8 @@
 #ifndef PROMPT_H
 #define PROMPT_H
 
-typedef enum _promptStatus
-{
-	PROMPT_READY,
-	PROMPT_CONTINUE,
-	PROMPT_COMMENT,
-	PROMPT_SINGLEQUOTE,
-	PROMPT_DOUBLEQUOTE,
-	PROMPT_DOLLARQUOTE,
-	PROMPT_PAREN,
-	PROMPT_COPY
-} promptStatus_t;
+/* enum promptStatus_t is now defined by psqlscan.h */
+#include "fe_utils/psqlscan.h"
 
 char	   *get_prompt(promptStatus_t status);
 

src/bin/psql/psqlscanslash.h

@@ -8,7 +8,7 @@
 #ifndef PSQLSCANSLASH_H
 #define PSQLSCANSLASH_H
 
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
 
 /* Different ways for scan_slash_option to handle parameter words */

src/bin/psql/psqlscanslash.l

@@ -6,7 +6,7 @@
  *
  * XXX Avoid creating backtracking cases --- see the backend lexer for info.
  *
- * See psqlscan_int.h for additional commentary.
+ * See fe_utils/psqlscan_int.h for additional commentary.
  *
  * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
@@ -24,7 +24,7 @@
 }
 
 %{
-#include "psqlscan_int.h"
+#include "fe_utils/psqlscan_int.h"
 
 /*
  * We must have a typedef YYSTYPE for yylex's first argument, but this lexer

src/fe_utils/.gitignore

@@ -0,0 +1 @@
+/psqlscan.c

src/fe_utils/Makefile

@@ -1,11 +1,13 @@
 #-------------------------------------------------------------------------
 #
-# Makefile
-#    Makefile for src/fe_utils
+# Makefile for src/fe_utils
 #
 # This makefile generates a static library, libpgfeutils.a,
 # for use by client applications
 #
+# Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
 # IDENTIFICATION
 #    src/fe_utils/Makefile
 #
@@ -17,7 +19,7 @@ include $(top_builddir)/src/Makefile.global
 
 override CPPFLAGS := -DFRONTEND -I$(libpq_srcdir) $(CPPFLAGS)
 
-OBJS = mbprint.o print.o simple_list.o string_utils.o
+OBJS = mbprint.o print.o psqlscan.o simple_list.o string_utils.o
 
 all: libpgfeutils.a
 
@@ -25,6 +27,16 @@ libpgfeutils.a: $(OBJS)
 	rm -f $@
 	$(AR) $(AROPT) $@ $^
 
+psqlscan.c: FLEXFLAGS = -Cfe -p -p
+psqlscan.c: FLEX_NO_BACKUP=yes
+
+# Latest flex causes warnings in this file.
+ifeq ($(GCC),yes)
+psqlscan.o: CFLAGS += -Wno-error
+endif
+
+distprep: psqlscan.c
+
 # libpgfeutils could be useful to contrib, so install it
 install: all installdirs
 	$(INSTALL_STLIB) libpgfeutils.a '$(DESTDIR)$(libdir)/libpgfeutils.a'
@@ -35,5 +47,10 @@ installdirs:
 uninstall:
 	rm -f '$(DESTDIR)$(libdir)/libpgfeutils.a'
 
-clean distclean maintainer-clean:
-	rm -f libpgfeutils.a $(OBJS)
+clean distclean:
+	rm -f libpgfeutils.a $(OBJS) lex.backup
+
+# psqlscan.c is supposed to be in the distribution tarball,
+# so do not clean it in the clean/distclean rules
+maintainer-clean: distclean
+	rm -f psqlscan.c

src/fe_utils/psqlscan.l

@@ -2,14 +2,20 @@
 /*-------------------------------------------------------------------------
  *
  * psqlscan.l
- *	  lexical scanner for psql (and other frontend programs)
+ *	  lexical scanner for SQL commands
  *
- * This code is mainly needed to determine where the end of a SQL statement
- * is: we are looking for semicolons that are not within quotes, comments,
- * or parentheses.  The most reliable way to handle this is to borrow the
- * backend's flex lexer rules, lock, stock, and barrel.  The rules below
- * are (except for a few) the same as the backend's, but their actions are
- * just ECHO whereas the backend's actions generally do other things.
+ * This lexer used to be part of psql, and that heritage is reflected in
+ * the file name as well as function and typedef names, though it can now
+ * be used by other frontend programs as well.  It's also possible to extend
+ * this lexer with a compatible add-on lexer to handle program-specific
+ * backslash commands.
+ *
+ * This code is mainly concerned with determining where the end of a SQL
+ * statement is: we are looking for semicolons that are not within quotes,
+ * comments, or parentheses.  The most reliable way to handle this is to
+ * borrow the backend's flex lexer rules, lock, stock, and barrel.  The rules
+ * below are (except for a few) the same as the backend's, but their actions
+ * are just ECHO whereas the backend's actions generally do other things.
  *
  * XXX The rules in this file must be kept in sync with the backend lexer!!!
  *
@@ -21,19 +27,19 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  src/bin/psql/psqlscan.l
+ *	  src/fe_utils/psqlscan.l
  *
  *-------------------------------------------------------------------------
  */
 #include "postgres_fe.h"
 
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
 #include "libpq-fe.h"
 }
 
 %{
-#include "psqlscan_int.h"
+#include "fe_utils/psqlscan_int.h"
 
 /*
  * We must have a typedef YYSTYPE for yylex's first argument, but this lexer
@@ -54,8 +60,6 @@ typedef int YYSTYPE;
 #define LEXRES_BACKSLASH	2	/* backslash command start */
 
 
-static bool var_is_current_source(PsqlScanState state, const char *varname);
-
 #define ECHO psqlscan_emit(cur_state, yytext, yyleng)
 
 /*
@@ -703,7 +707,7 @@ other			.
 					if (value)
 					{
 						/* It is a variable, check for recursion */
-						if (var_is_current_source(cur_state, varname))
+						if (psqlscan_var_is_current_source(cur_state, varname))
 						{
 							/* Recursive expansion --- don't go there */
 							cur_state->callbacks->write_error("skipping recursive expansion of variable \"%s\"\n",
@@ -1264,8 +1268,8 @@ psqlscan_select_top_buffer(PsqlScanState state)
  * Check if specified variable name is the source for any string
  * currently being scanned
  */
-static bool
-var_is_current_source(PsqlScanState state, const char *varname)
+bool
+psqlscan_var_is_current_source(PsqlScanState state, const char *varname)
 {
 	StackElem  *stackelem;
 

src/include/fe_utils/psqlscan.h

@@ -1,17 +1,27 @@
-/*
- * psql - the PostgreSQL interactive terminal
+/*-------------------------------------------------------------------------
  *
- * Copyright (c) 2000-2016, PostgreSQL Global Development Group
+ * psqlscan.h
+ *	  lexical scanner for SQL commands
  *
- * src/bin/psql/psqlscan.h
+ * This lexer used to be part of psql, and that heritage is reflected in
+ * the file name as well as function and typedef names, though it can now
+ * be used by other frontend programs as well.  It's also possible to extend
+ * this lexer with a compatible add-on lexer to handle program-specific
+ * backslash commands.
+ *
+ *
+ * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/include/fe_utils/psqlscan.h
+ *
+ *-------------------------------------------------------------------------
  */
 #ifndef PSQLSCAN_H
 #define PSQLSCAN_H
 
 #include "pqexpbuffer.h"
 
-#include "prompt.h"
-
 
 /* Abstract type for lexer's internal state */
 typedef struct PsqlScanStateData *PsqlScanState;
@@ -25,6 +35,19 @@ typedef enum
 	PSCAN_EOL					/* end of line, SQL possibly complete */
 } PsqlScanResult;
 
+/* Prompt type returned by psql_scan() */
+typedef enum _promptStatus
+{
+	PROMPT_READY,
+	PROMPT_CONTINUE,
+	PROMPT_COMMENT,
+	PROMPT_SINGLEQUOTE,
+	PROMPT_DOUBLEQUOTE,
+	PROMPT_DOLLARQUOTE,
+	PROMPT_PAREN,
+	PROMPT_COPY
+} promptStatus_t;
+
 /* Callback functions to be used by the lexer */
 typedef struct PsqlScanCallbacks
 {

src/include/fe_utils/psqlscan_int.h

@@ -1,4 +1,5 @@
-/*
+/*-------------------------------------------------------------------------
+ *
  * psqlscan_int.h
  *	  lexical scanner internal declarations
  *
@@ -30,22 +31,34 @@
  * if we were using lexers with separate static state we would soon end up
  * with dangling buffer pointers in one or the other.  Also note that this
  * is unlikely to work very nicely if the lexers aren't all built with the
- * same flex version.
+ * same flex version, or if they don't use the same flex options.
+ *
+ *
+ * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
  *
- * Copyright (c) 2000-2016, PostgreSQL Global Development Group
+ * src/include/fe_utils/psqlscan_int.h
  *
- * src/bin/psql/psqlscan_int.h
+ *-------------------------------------------------------------------------
  */
 #ifndef PSQLSCAN_INT_H
 #define PSQLSCAN_INT_H
 
-#include "psqlscan.h"
+#include "fe_utils/psqlscan.h"
 
-/* This is just to allow this file to be compilable standalone */
+/*
+ * These are just to allow this file to be compilable standalone for header
+ * validity checking; in actual use, this file should always be included
+ * from the body of a flex file, where these symbols are already defined.
+ */
 #ifndef YY_TYPEDEF_YY_BUFFER_STATE
 #define YY_TYPEDEF_YY_BUFFER_STATE
 typedef struct yy_buffer_state *YY_BUFFER_STATE;
 #endif
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void *yyscan_t;
+#endif
 
 /*
  * We use a stack of flex buffers to handle substitution of psql variables.
@@ -116,6 +129,8 @@ extern void psqlscan_push_new_buffer(PsqlScanState state,
 						 const char *newstr, const char *varname);
 extern void psqlscan_pop_buffer_stack(PsqlScanState state);
 extern void psqlscan_select_top_buffer(PsqlScanState state);
+extern bool psqlscan_var_is_current_source(PsqlScanState state,
+							   const char *varname);
 extern YY_BUFFER_STATE psqlscan_prepare_buffer(PsqlScanState state,
 						const char *txt, int len,
 						char **txtcopy);

src/tools/msvc/Mkvcbuild.pm

@@ -63,16 +63,14 @@ my $frontend_extralibs = {
 	'psql'       => ['ws2_32.lib'] };
 my $frontend_extraincludes = {
 	'initdb' => ['src/timezone'],
-	'psql'   => [ 'src/backend' ],
-	'pgbench' => [ 'src/bin/psql' ] };
+	'psql'   => [ 'src/backend' ] };
 my $frontend_extrasource = {
-	'psql' => ['src/bin/psql/psqlscan.l', 'src/bin/psql/psqlscanslash.l'],
+	'psql' => [ 'src/bin/psql/psqlscanslash.l' ],
 	'pgbench' =>
-	  [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y',
-	    'src/bin/psql/psqlscan.l' ] };
+	  [ 'src/bin/pgbench/exprscan.l', 'src/bin/pgbench/exprparse.y' ] };
 my @frontend_excludes = (
 	'pgevent',     'pg_basebackup', 'pg_rewind', 'pg_dump',
-	'pg_xlogdump', 'scripts',       'pgbench');
+	'pg_xlogdump', 'scripts');
 
 sub mkvcbuild
 {
@@ -120,7 +118,7 @@ sub mkvcbuild
 	our @pgcommonbkndfiles = @pgcommonallfiles;
 
 	our @pgfeutilsfiles = qw(
-	  mbprint.c print.c simple_list.c string_utils.c);
+	  mbprint.c print.c psqlscan.l psqlscan.c simple_list.c string_utils.c);
 
 	$libpgport = $solution->AddProject('libpgport', 'lib', 'misc');
 	$libpgport->AddDefine('FRONTEND');
@@ -659,11 +657,6 @@ sub mkvcbuild
 	}
 	$pg_xlogdump->AddFile('src/backend/access/transam/xlogreader.c');
 
-	# fix up pgbench once it's been set up
-	# we're borrowing psqlscan.c from psql, so grab it from the correct place
-	my $pgbench = AddSimpleFrontend('pgbench');
-	$pgbench->ReplaceFile('src/bin/pgbench/psqlscan.c', 'src/bin/psql/psqlscan.c');
-
 	$solution->Save();
 	return $solution->{vcver};
 }

src/tools/msvc/clean.bat

@@ -75,7 +75,7 @@ if exist src\pl\plperl\spi.c del /q src\pl\plperl\spi.c
 if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.c del /q src\pl\plpgsql\src\pl_gram.c
 if %DIST%==1 if exist src\pl\plpgsql\src\pl_gram.h del /q src\pl\plpgsql\src\pl_gram.h
 
-if %DIST%==1 if exist src\bin\psql\psqlscan.c del /q src\bin\psql\psqlscan.c
+if %DIST%==1 if exist src\fe_utils\psqlscan.c del /q src\fe_utils\psqlscan.c
 if %DIST%==1 if exist src\bin\psql\psqlscanslash.c del /q src\bin\psql\psqlscanslash.c
 
 if %DIST%==1 if exist contrib\cube\cubescan.c del /q contrib\cube\cubescan.c