mirror of https://github.com/grafana/grafana
Remove support for Google Spanner database. (#105846)
* Remove support for Google Spanner database.pull/105930/head
Peter Štibraný
7 months ago
committed by
GitHub
parent
9769871a88
commit
c4d3eb1cd0
@ -1,355 +0,0 @@ |
|||||||
//go:build enterprise || pro
|
|
||||||
|
|
||||||
package migrator |
|
||||||
|
|
||||||
import ( |
|
||||||
"context" |
|
||||||
_ "embed" |
|
||||||
"encoding/json" |
|
||||||
"errors" |
|
||||||
"fmt" |
|
||||||
"strings" |
|
||||||
"time" |
|
||||||
|
|
||||||
"cloud.google.com/go/spanner" |
|
||||||
database "cloud.google.com/go/spanner/admin/database/apiv1" |
|
||||||
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb" |
|
||||||
"github.com/googleapis/gax-go/v2" |
|
||||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
|
||||||
"github.com/grafana/grafana/pkg/util/xorm" |
|
||||||
"google.golang.org/grpc/codes" |
|
||||||
|
|
||||||
"github.com/grafana/dskit/concurrency" |
|
||||||
utilspanner "github.com/grafana/grafana/pkg/util/spanner" |
|
||||||
"github.com/grafana/grafana/pkg/util/xorm/core" |
|
||||||
) |
|
||||||
|
|
||||||
type SpannerDialect struct { |
|
||||||
BaseDialect |
|
||||||
d core.Dialect |
|
||||||
} |
|
||||||
|
|
||||||
func init() { |
|
||||||
supportedDialects[Spanner] = NewSpannerDialect |
|
||||||
} |
|
||||||
|
|
||||||
func NewSpannerDialect() Dialect { |
|
||||||
d := SpannerDialect{d: core.QueryDialect(Spanner)} |
|
||||||
d.dialect = &d |
|
||||||
d.driverName = Spanner |
|
||||||
return &d |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) AutoIncrStr() string { return s.d.AutoIncrStr() } |
|
||||||
func (s *SpannerDialect) Quote(name string) string { return s.d.Quote(name) } |
|
||||||
func (s *SpannerDialect) SupportEngine() bool { return s.d.SupportEngine() } |
|
||||||
|
|
||||||
func (s *SpannerDialect) LikeOperator(column string, wildcardBefore bool, pattern string, wildcardAfter bool) (string, string) { |
|
||||||
param := strings.ToLower(pattern) |
|
||||||
if wildcardBefore { |
|
||||||
param = "%" + param |
|
||||||
} |
|
||||||
if wildcardAfter { |
|
||||||
param = param + "%" |
|
||||||
} |
|
||||||
return fmt.Sprintf("LOWER(%s) LIKE ?", column), param |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) IndexCheckSQL(tableName, indexName string) (string, []any) { |
|
||||||
return s.d.IndexCheckSql(tableName, indexName) |
|
||||||
} |
|
||||||
func (s *SpannerDialect) SQLType(col *Column) string { |
|
||||||
c := core.NewColumn(col.Name, "", core.SQLType{Name: col.Type}, col.Length, col.Length2, col.Nullable) |
|
||||||
return s.d.SqlType(c) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) BatchSize() int { return 1000 } |
|
||||||
|
|
||||||
func (s *SpannerDialect) BooleanValue(b bool) any { |
|
||||||
return b |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) BooleanStr(b bool) string { |
|
||||||
if b { |
|
||||||
return "true" |
|
||||||
} |
|
||||||
return "false" |
|
||||||
} |
|
||||||
func (s *SpannerDialect) ErrorMessage(err error) string { |
|
||||||
return spanner.ErrDesc(spanner.ToSpannerError(err)) |
|
||||||
} |
|
||||||
func (s *SpannerDialect) IsDeadlock(err error) bool { |
|
||||||
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.Aborted |
|
||||||
} |
|
||||||
func (s *SpannerDialect) IsUniqueConstraintViolation(err error) bool { |
|
||||||
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.AlreadyExists |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) CreateTableSQL(table *Table) string { |
|
||||||
t := core.NewEmptyTable() |
|
||||||
t.Name = table.Name |
|
||||||
t.PrimaryKeys = table.PrimaryKeys |
|
||||||
for _, c := range table.Columns { |
|
||||||
col := core.NewColumn(c.Name, c.Name, core.SQLType{Name: c.Type}, c.Length, c.Length2, c.Nullable) |
|
||||||
col.IsAutoIncrement = c.IsAutoIncrement |
|
||||||
col.Default = c.Default |
|
||||||
t.AddColumn(col) |
|
||||||
} |
|
||||||
if len(t.PrimaryKeys) == 0 { |
|
||||||
for _, ix := range table.Indices { |
|
||||||
if ix.Name == "PRIMARY_KEY" { |
|
||||||
t.PrimaryKeys = append(t.PrimaryKeys, ix.Cols...) |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
return s.d.CreateTableSql(t, t.Name, "", "") |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) CreateIndexSQL(tableName string, index *Index) string { |
|
||||||
idx := core.NewIndex(index.Name, index.Type) |
|
||||||
idx.Cols = index.Cols |
|
||||||
return s.d.CreateIndexSql(tableName, idx) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) UpsertMultipleSQL(tableName string, keyCols, updateCols []string, count int) (string, error) { |
|
||||||
return "", errors.New("not supported") |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) DropIndexSQL(tableName string, index *Index) string { |
|
||||||
return fmt.Sprintf("DROP INDEX %v", s.Quote(index.XName(tableName))) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) DropTable(tableName string) string { |
|
||||||
return fmt.Sprintf("DROP TABLE %s", s.Quote(tableName)) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) ColStringNoPk(col *Column) string { |
|
||||||
sql := s.dialect.Quote(col.Name) + " " |
|
||||||
|
|
||||||
sql += s.dialect.SQLType(col) + " " |
|
||||||
|
|
||||||
if s.dialect.ShowCreateNull() && !col.Nullable { |
|
||||||
sql += "NOT NULL " |
|
||||||
} |
|
||||||
|
|
||||||
if col.Default != "" { |
|
||||||
// Default value must be in parentheses.
|
|
||||||
sql += "DEFAULT (" + s.dialect.Default(col) + ") " |
|
||||||
} |
|
||||||
|
|
||||||
return sql |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) TruncateDBTables(engine *xorm.Engine) error { |
|
||||||
// Get tables names only, no columns or indexes.
|
|
||||||
tables, err := engine.Dialect().GetTables() |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
sess := engine.NewSession() |
|
||||||
defer sess.Close() |
|
||||||
|
|
||||||
var statements []string |
|
||||||
|
|
||||||
for _, table := range tables { |
|
||||||
switch table.Name { |
|
||||||
case "": |
|
||||||
continue |
|
||||||
case "autoincrement_sequences": |
|
||||||
// Don't delete sequence number for migration_log.id column.
|
|
||||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE name <> 'migration_log:id'", s.Quote(table.Name))) |
|
||||||
case "migration_log": |
|
||||||
continue |
|
||||||
case "dashboard_acl": |
|
||||||
// keep default dashboard permissions
|
|
||||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE dashboard_id != -1 AND org_id != -1;", s.Quote(table.Name))) |
|
||||||
default: |
|
||||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE TRUE;", s.Quote(table.Name))) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// Run statements concurrently.
|
|
||||||
return concurrency.ForEachJob(context.Background(), len(statements), 10, func(ctx context.Context, idx int) error { |
|
||||||
_, err := sess.Exec(statements[idx]) |
|
||||||
return err |
|
||||||
}) |
|
||||||
} |
|
||||||
|
|
||||||
// CleanDB drops all existing tables and their indexes.
|
|
||||||
func (s *SpannerDialect) CleanDB(engine *xorm.Engine) error { |
|
||||||
tables, err := engine.DBMetas() |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
// Collect all DROP statements.
|
|
||||||
changeStreams, err := s.findChangeStreams(engine) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
statements := make([]string, 0, len(tables)+len(changeStreams)) |
|
||||||
for _, cs := range changeStreams { |
|
||||||
statements = append(statements, fmt.Sprintf("DROP CHANGE STREAM `%s`", cs)) |
|
||||||
} |
|
||||||
|
|
||||||
for _, table := range tables { |
|
||||||
// Indexes must be dropped first, otherwise dropping tables fails.
|
|
||||||
for _, index := range table.Indexes { |
|
||||||
if !index.IsRegular { |
|
||||||
// Don't drop primary key.
|
|
||||||
continue |
|
||||||
} |
|
||||||
sql := fmt.Sprintf("DROP INDEX %s", s.Quote(index.XName(table.Name))) |
|
||||||
statements = append(statements, sql) |
|
||||||
} |
|
||||||
|
|
||||||
sql := fmt.Sprintf("DROP TABLE %s", s.Quote(table.Name)) |
|
||||||
statements = append(statements, sql) |
|
||||||
} |
|
||||||
|
|
||||||
if len(statements) == 0 { |
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
return s.executeDDLStatements(context.Background(), engine, statements) |
|
||||||
} |
|
||||||
|
|
||||||
//go:embed snapshot/spanner-ddl.json
|
|
||||||
var snapshotDDL string |
|
||||||
|
|
||||||
//go:embed snapshot/spanner-log.json
|
|
||||||
var snapshotMigrations string |
|
||||||
|
|
||||||
func (s *SpannerDialect) CreateDatabaseFromSnapshot(ctx context.Context, engine *xorm.Engine, tableName string) error { |
|
||||||
var statements, migrationIDs []string |
|
||||||
err := json.Unmarshal([]byte(snapshotDDL), &statements) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
err = json.Unmarshal([]byte(snapshotMigrations), &migrationIDs) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
err = s.executeDDLStatements(ctx, engine, statements) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
return s.recordMigrationsToMigrationLog(engine, migrationIDs, tableName) |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) recordMigrationsToMigrationLog(engine *xorm.Engine, migrationIDs []string, tableName string) error { |
|
||||||
now := time.Now() |
|
||||||
makeRecord := func(id string) MigrationLog { |
|
||||||
return MigrationLog{ |
|
||||||
MigrationID: id, |
|
||||||
SQL: "", |
|
||||||
Success: true, |
|
||||||
Timestamp: now, |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
sess := engine.NewSession() |
|
||||||
defer sess.Close() |
|
||||||
|
|
||||||
// Insert records in batches to avoid many roundtrips to database.
|
|
||||||
// Inserting all records at once fails due to "Number of parameters in query exceeds the maximum
|
|
||||||
// allowed limit of 950." error, so we use smaller batches.
|
|
||||||
const batchSize = 100 |
|
||||||
|
|
||||||
if err := sess.Begin(); err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
records := make([]MigrationLog, 0, len(migrationIDs)) |
|
||||||
for _, mid := range migrationIDs { |
|
||||||
records = append(records, makeRecord(mid)) |
|
||||||
|
|
||||||
if len(records) >= batchSize { |
|
||||||
if _, err := sess.Table(tableName).InsertMulti(records); err != nil { |
|
||||||
err2 := sess.Rollback() |
|
||||||
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2) |
|
||||||
} |
|
||||||
records = records[:0] |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
// Insert remaining records.
|
|
||||||
if len(records) > 0 { |
|
||||||
if _, err := sess.Table(tableName).InsertMulti(records); err != nil { |
|
||||||
err2 := sess.Rollback() |
|
||||||
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
if err := sess.Commit(); err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
// Spanner can be very slow at executing single DDL statements (it can take up to a minute), but when
|
|
||||||
// many DDL statements are batched together, Spanner is *much* faster (total time to execute all statements
|
|
||||||
// is often in tens of seconds). We can't execute batch of DDL statements using sql wrapper, we use "database admin client"
|
|
||||||
// from Spanner library instead.
|
|
||||||
func (s *SpannerDialect) executeDDLStatements(ctx context.Context, engine *xorm.Engine, statements []string) error { |
|
||||||
// Datasource name contains string used for sql.Open.
|
|
||||||
dsn := engine.Dialect().DataSourceName() |
|
||||||
cfg, err := spannerdriver.ExtractConnectorConfig(dsn) |
|
||||||
if err != nil { |
|
||||||
return err |
|
||||||
} |
|
||||||
|
|
||||||
opts := utilspanner.ConnectorConfigToClientOptions(cfg) |
|
||||||
|
|
||||||
databaseAdminClient, err := database.NewDatabaseAdminClient(ctx, opts...) |
|
||||||
if err != nil { |
|
||||||
return fmt.Errorf("failed to create database admin client: %v", err) |
|
||||||
} |
|
||||||
//nolint:errcheck // If the databaseAdminClient.Close fails, we simply don't care.
|
|
||||||
defer databaseAdminClient.Close() |
|
||||||
|
|
||||||
databaseName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", cfg.Project, cfg.Instance, cfg.Database) |
|
||||||
|
|
||||||
op, err := databaseAdminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ |
|
||||||
Database: databaseName, |
|
||||||
Statements: statements, |
|
||||||
}, gax.WithTimeout(0)) /* disable default timeout */ |
|
||||||
if err != nil { |
|
||||||
return fmt.Errorf("failed to start database DDL update: %v", err) |
|
||||||
} |
|
||||||
|
|
||||||
err = op.Wait(ctx, gax.WithTimeout(0)) /* disable default timeout */ |
|
||||||
if err != nil { |
|
||||||
return fmt.Errorf("failed to apply database DDL update: %v", err) |
|
||||||
} |
|
||||||
return nil |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) UnionDistinct() string { |
|
||||||
return "UNION DISTINCT" |
|
||||||
} |
|
||||||
|
|
||||||
func (s *SpannerDialect) findChangeStreams(engine *xorm.Engine) ([]string, error) { |
|
||||||
var result []string |
|
||||||
query := `SELECT c.CHANGE_STREAM_NAME |
|
||||||
FROM INFORMATION_SCHEMA.CHANGE_STREAMS AS C |
|
||||||
WHERE C.CHANGE_STREAM_CATALOG='' |
|
||||||
AND C.CHANGE_STREAM_SCHEMA=''` |
|
||||||
rows, err := engine.DB().Query(query) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
//nolint:errcheck // If the rows.Close fails, we simply don't care.
|
|
||||||
defer rows.Close() |
|
||||||
for rows.Next() { |
|
||||||
var name string |
|
||||||
if err := rows.Scan(&name); err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
result = append(result, name) |
|
||||||
} |
|
||||||
return result, nil |
|
||||||
} |
|
||||||
@ -1,32 +0,0 @@ |
|||||||
package sqltemplate |
|
||||||
|
|
||||||
// Spanner is an implementation of Dialect for the Google Spanner database.
|
|
||||||
var Spanner = spanner{} |
|
||||||
|
|
||||||
var _ Dialect = Spanner |
|
||||||
|
|
||||||
type spanner struct{} |
|
||||||
|
|
||||||
func (s spanner) DialectName() string { |
|
||||||
return "spanner" |
|
||||||
} |
|
||||||
|
|
||||||
func (s spanner) Ident(a string) (string, error) { |
|
||||||
return backtickIdent{}.Ident(a) |
|
||||||
} |
|
||||||
|
|
||||||
func (s spanner) ArgPlaceholder(argNum int) string { |
|
||||||
return argFmtSQL92.ArgPlaceholder(argNum) |
|
||||||
} |
|
||||||
|
|
||||||
func (s spanner) SelectFor(a ...string) (string, error) { |
|
||||||
return rowLockingClauseSpanner.SelectFor(a...) |
|
||||||
} |
|
||||||
|
|
||||||
func (spanner) CurrentEpoch() string { |
|
||||||
return "UNIX_MICROS(CURRENT_TIMESTAMP())" |
|
||||||
} |
|
||||||
|
|
||||||
var rowLockingClauseSpanner = rowLockingClauseMap{ |
|
||||||
SelectForUpdate: SelectForUpdate, |
|
||||||
} |
|
||||||
@ -1,40 +0,0 @@ |
|||||||
// Package spanner should only be used from tests, or from enterprise code (eg. protected by build tags).
|
|
||||||
package spanner |
|
||||||
|
|
||||||
import ( |
|
||||||
"strconv" |
|
||||||
|
|
||||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
|
||||||
"google.golang.org/api/option" |
|
||||||
"google.golang.org/grpc" |
|
||||||
"google.golang.org/grpc/credentials/insecure" |
|
||||||
) |
|
||||||
|
|
||||||
func UsePlainText(connectorConfig spannerdriver.ConnectorConfig) bool { |
|
||||||
if strval, ok := connectorConfig.Params["useplaintext"]; ok { |
|
||||||
if val, err := strconv.ParseBool(strval); err == nil { |
|
||||||
return val |
|
||||||
} |
|
||||||
} |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
// ConnectorConfigToClientOptions is adapted from https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L341-L477, from version 1.11.1.
|
|
||||||
func ConnectorConfigToClientOptions(connectorConfig spannerdriver.ConnectorConfig) []option.ClientOption { |
|
||||||
var opts []option.ClientOption |
|
||||||
if connectorConfig.Host != "" { |
|
||||||
opts = append(opts, option.WithEndpoint(connectorConfig.Host)) |
|
||||||
} |
|
||||||
if strval, ok := connectorConfig.Params["credentials"]; ok { |
|
||||||
opts = append(opts, option.WithCredentialsFile(strval)) |
|
||||||
} |
|
||||||
if strval, ok := connectorConfig.Params["credentialsjson"]; ok { |
|
||||||
opts = append(opts, option.WithCredentialsJSON([]byte(strval))) |
|
||||||
} |
|
||||||
if UsePlainText(connectorConfig) { |
|
||||||
opts = append(opts, |
|
||||||
option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), |
|
||||||
option.WithoutAuthentication()) |
|
||||||
} |
|
||||||
return opts |
|
||||||
} |
|
||||||
@ -1,399 +0,0 @@ |
|||||||
//go:build enterprise || pro
|
|
||||||
|
|
||||||
package xorm |
|
||||||
|
|
||||||
import ( |
|
||||||
"database/sql" |
|
||||||
"fmt" |
|
||||||
"strconv" |
|
||||||
"strings" |
|
||||||
|
|
||||||
spannerclient "cloud.google.com/go/spanner" |
|
||||||
_ "github.com/googleapis/go-sql-spanner" |
|
||||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
|
||||||
"github.com/grafana/grafana/pkg/util/xorm/core" |
|
||||||
"google.golang.org/grpc/codes" |
|
||||||
) |
|
||||||
|
|
||||||
func init() { |
|
||||||
core.RegisterDriver("spanner", &spannerDriver{}) |
|
||||||
core.RegisterDialect("spanner", func() core.Dialect { return &spanner{} }) |
|
||||||
} |
|
||||||
|
|
||||||
// https://cloud.google.com/spanner/docs/reference/standard-sql/lexical#reserved_keywords
|
|
||||||
var spannerReservedKeywords = map[string]struct{}{ |
|
||||||
"ALL": {}, |
|
||||||
"AND": {}, |
|
||||||
"ANY": {}, |
|
||||||
"ARRAY": {}, |
|
||||||
"AS": {}, |
|
||||||
"ASC": {}, |
|
||||||
"ASSERT_ROWS_MODIFIED": {}, |
|
||||||
"AT": {}, |
|
||||||
"BETWEEN": {}, |
|
||||||
"BY": {}, |
|
||||||
"CASE": {}, |
|
||||||
"CAST": {}, |
|
||||||
"COLLATE": {}, |
|
||||||
"CONTAINS": {}, |
|
||||||
"CREATE": {}, |
|
||||||
"CROSS": {}, |
|
||||||
"CUBE": {}, |
|
||||||
"CURRENT": {}, |
|
||||||
"DEFAULT": {}, |
|
||||||
"DEFINE": {}, |
|
||||||
"DESC": {}, |
|
||||||
"DISTINCT": {}, |
|
||||||
"ELSE": {}, |
|
||||||
"END": {}, |
|
||||||
"ENUM": {}, |
|
||||||
"ESCAPE": {}, |
|
||||||
"EXCEPT": {}, |
|
||||||
"EXCLUDE": {}, |
|
||||||
"EXISTS": {}, |
|
||||||
"EXTRACT": {}, |
|
||||||
"FALSE": {}, |
|
||||||
"FETCH": {}, |
|
||||||
"FOLLOWING": {}, |
|
||||||
"FOR": {}, |
|
||||||
"FROM": {}, |
|
||||||
"FULL": {}, |
|
||||||
"GROUP": {}, |
|
||||||
"GROUPING": {}, |
|
||||||
"GROUPS": {}, |
|
||||||
"HASH": {}, |
|
||||||
"HAVING": {}, |
|
||||||
"IF": {}, |
|
||||||
"IGNORE": {}, |
|
||||||
"IN": {}, |
|
||||||
"INNER": {}, |
|
||||||
"INTERSECT": {}, |
|
||||||
"INTERVAL": {}, |
|
||||||
"INTO": {}, |
|
||||||
"IS": {}, |
|
||||||
"JOIN": {}, |
|
||||||
"LATERAL": {}, |
|
||||||
"LEFT": {}, |
|
||||||
"LIKE": {}, |
|
||||||
"LIMIT": {}, |
|
||||||
"LOOKUP": {}, |
|
||||||
"MERGE": {}, |
|
||||||
"NATURAL": {}, |
|
||||||
"NEW": {}, |
|
||||||
"NO": {}, |
|
||||||
"NOT": {}, |
|
||||||
"NULL": {}, |
|
||||||
"NULLS": {}, |
|
||||||
"OF": {}, |
|
||||||
"ON": {}, |
|
||||||
"OR": {}, |
|
||||||
"ORDER": {}, |
|
||||||
"OUTER": {}, |
|
||||||
"OVER": {}, |
|
||||||
"PARTITION": {}, |
|
||||||
"PRECEDING": {}, |
|
||||||
"PROTO": {}, |
|
||||||
"RANGE": {}, |
|
||||||
"RECURSIVE": {}, |
|
||||||
"RESPECT": {}, |
|
||||||
"RIGHT": {}, |
|
||||||
"ROLLUP": {}, |
|
||||||
"ROWS": {}, |
|
||||||
"SELECT": {}, |
|
||||||
"SET": {}, |
|
||||||
"SOME": {}, |
|
||||||
"STRUCT": {}, |
|
||||||
"TABLESAMPLE": {}, |
|
||||||
"THEN": {}, |
|
||||||
"TO": {}, |
|
||||||
"TREAT": {}, |
|
||||||
"TRUE": {}, |
|
||||||
"UNBOUNDED": {}, |
|
||||||
"UNION": {}, |
|
||||||
"UNNEST": {}, |
|
||||||
"USING": {}, |
|
||||||
"WHEN": {}, |
|
||||||
"WHERE": {}, |
|
||||||
"WINDOW": {}, |
|
||||||
"WITH": {}, |
|
||||||
"WITHIN": {}, |
|
||||||
} |
|
||||||
|
|
||||||
type spannerDriver struct{} |
|
||||||
|
|
||||||
func (d *spannerDriver) Parse(_driverName, datasourceName string) (*core.Uri, error) { |
|
||||||
return &core.Uri{DbType: "spanner", DbName: datasourceName}, nil |
|
||||||
} |
|
||||||
|
|
||||||
type spanner struct { |
|
||||||
core.Base |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) Init(db *core.DB, uri *core.Uri, driverName string, datasourceName string) error { |
|
||||||
return s.Base.Init(db, s, uri, driverName, datasourceName) |
|
||||||
} |
|
||||||
func (s *spanner) Filters() []core.Filter { return []core.Filter{&core.IdFilter{}} } |
|
||||||
func (s *spanner) IsReserved(name string) bool { |
|
||||||
_, exists := spannerReservedKeywords[name] |
|
||||||
return exists |
|
||||||
} |
|
||||||
func (s *spanner) AndStr() string { return "AND" } |
|
||||||
func (s *spanner) OrStr() string { return "OR" } |
|
||||||
func (s *spanner) EqStr() string { return "=" } |
|
||||||
func (s *spanner) RollBackStr() string { return "ROLL BACK" } |
|
||||||
func (s *spanner) AutoIncrStr() string { |
|
||||||
// Spanner does not support auto-increment, but supports unique generated IDs (not sequential!).
|
|
||||||
return "GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)" |
|
||||||
} |
|
||||||
func (s *spanner) SupportInsertMany() bool { return false } // Needs manual transaction batching
|
|
||||||
func (s *spanner) SupportEngine() bool { return false } // No support for engine selection
|
|
||||||
func (s *spanner) SupportCharset() bool { return false } // ...or charsets
|
|
||||||
func (s *spanner) SupportDropIfExists() bool { return false } // Drop should be handled differently
|
|
||||||
func (s *spanner) IndexOnTable() bool { return false } |
|
||||||
func (s *spanner) ShowCreateNull() bool { return false } |
|
||||||
func (s *spanner) Quote(name string) string { return "`" + name + "`" } |
|
||||||
func (s *spanner) SqlType(col *core.Column) string { |
|
||||||
switch col.SQLType.Name { |
|
||||||
case core.Int, core.SmallInt, core.BigInt: |
|
||||||
return "INT64" |
|
||||||
case core.Varchar, core.Text, core.MediumText, core.LongText, core.Char, core.NVarchar, core.NChar, core.NText: |
|
||||||
l := col.Length |
|
||||||
if l == 0 { |
|
||||||
l = col.SQLType.DefaultLength |
|
||||||
} |
|
||||||
if l > 0 { |
|
||||||
return fmt.Sprintf("STRING(%d)", l) |
|
||||||
} |
|
||||||
return "STRING(MAX)" |
|
||||||
case core.Jsonb: |
|
||||||
return "STRING(MAX)" |
|
||||||
case core.Bool, core.TinyInt: |
|
||||||
return "BOOL" |
|
||||||
case core.Float, core.Double: |
|
||||||
return "FLOAT64" |
|
||||||
case core.Bytea, core.Blob, core.MediumBlob, core.LongBlob: |
|
||||||
l := col.Length |
|
||||||
if l == 0 { |
|
||||||
l = col.SQLType.DefaultLength |
|
||||||
} |
|
||||||
if l > 0 { |
|
||||||
return fmt.Sprintf("BYTES(%d)", l) |
|
||||||
} |
|
||||||
return "BYTES(MAX)" |
|
||||||
case core.DateTime, core.TimeStamp: |
|
||||||
return "TIMESTAMP" |
|
||||||
default: |
|
||||||
panic("unknown column type: " + col.SQLType.Name) |
|
||||||
//default:
|
|
||||||
// return "STRING(MAX)" // XXX: more types to add
|
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) GetColumns(tableName string) ([]string, map[string]*core.Column, error) { |
|
||||||
query := `SELECT COLUMN_NAME, SPANNER_TYPE, IS_NULLABLE, IS_IDENTITY, IDENTITY_GENERATION, IDENTITY_KIND, COLUMN_DEFAULT |
|
||||||
FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA="" ORDER BY ORDINAL_POSITION` |
|
||||||
rows, err := s.DB().Query(query, tableName) |
|
||||||
if err != nil { |
|
||||||
return nil, nil, err |
|
||||||
} |
|
||||||
defer rows.Close() |
|
||||||
|
|
||||||
columns := make(map[string]*core.Column) |
|
||||||
var colNames []string |
|
||||||
|
|
||||||
var name, sqlType, isNullable string |
|
||||||
var isIdentity, identityGeneration, identityKind, columnDefault sql.NullString |
|
||||||
for rows.Next() { |
|
||||||
if err := rows.Scan(&name, &sqlType, &isNullable, &isIdentity, &identityGeneration, &identityKind, &columnDefault); err != nil { |
|
||||||
return nil, nil, err |
|
||||||
} |
|
||||||
|
|
||||||
var length int |
|
||||||
switch { |
|
||||||
case sqlType == "INT64": |
|
||||||
sqlType = core.Int |
|
||||||
case sqlType == "FLOAT32" || sqlType == "FLOAT64": |
|
||||||
sqlType = core.Float |
|
||||||
case sqlType == "BOOL": |
|
||||||
sqlType = core.Bool |
|
||||||
case sqlType == "BYTES(MAX)": |
|
||||||
sqlType = core.Blob |
|
||||||
case sqlType == "STRING(MAX)": |
|
||||||
sqlType = core.NVarchar |
|
||||||
case sqlType == "TIMESTAMP": |
|
||||||
sqlType = core.DateTime |
|
||||||
case strings.HasPrefix(sqlType, "BYTES("): |
|
||||||
// 6 == len(`BYTES(`), we also remove ")" from the end.
|
|
||||||
if l, err := strconv.Atoi(sqlType[6 : len(sqlType)-1]); err == nil { |
|
||||||
length = l |
|
||||||
} |
|
||||||
sqlType = core.Blob |
|
||||||
case strings.HasPrefix(sqlType, "STRING("): |
|
||||||
// 7 == len(`STRING(`), we also remove ")" from the end.
|
|
||||||
if l, err := strconv.Atoi(sqlType[7 : len(sqlType)-1]); err == nil { |
|
||||||
length = l |
|
||||||
} |
|
||||||
sqlType = core.Varchar |
|
||||||
default: |
|
||||||
panic("unknown column type: " + sqlType) |
|
||||||
} |
|
||||||
|
|
||||||
autoincrement := isIdentity.Valid && isIdentity.String == "YES" && |
|
||||||
identityGeneration.Valid && identityGeneration.String == "BY DEFAULT" && |
|
||||||
identityKind.Valid && identityKind.String == "BIT_REVERSED_POSITIVE_SEQUENCE" |
|
||||||
|
|
||||||
defValue := "" |
|
||||||
defEmpty := true |
|
||||||
if columnDefault.Valid { |
|
||||||
defValue = columnDefault.String |
|
||||||
defEmpty = false |
|
||||||
} |
|
||||||
|
|
||||||
col := &core.Column{ |
|
||||||
Name: name, |
|
||||||
SQLType: core.SQLType{Name: sqlType}, |
|
||||||
Length: length, |
|
||||||
Nullable: isNullable == "YES", |
|
||||||
IsAutoIncrement: autoincrement, |
|
||||||
Indexes: map[string]int{}, |
|
||||||
Default: defValue, |
|
||||||
DefaultIsEmpty: defEmpty, |
|
||||||
} |
|
||||||
columns[name] = col |
|
||||||
colNames = append(colNames, name) |
|
||||||
} |
|
||||||
|
|
||||||
return colNames, columns, rows.Err() |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) CreateTableSql(table *core.Table, tableName, _, charset string) string { |
|
||||||
sql := "CREATE TABLE " + s.Quote(tableName) + " (" |
|
||||||
|
|
||||||
for i, col := range table.Columns() { |
|
||||||
if i > 0 { |
|
||||||
sql += ", " |
|
||||||
} |
|
||||||
|
|
||||||
sql += s.Quote(col.Name) + " " + s.SqlType(col) |
|
||||||
if !col.Nullable { |
|
||||||
sql += " NOT NULL" |
|
||||||
} |
|
||||||
if col.Default != "" { |
|
||||||
sql += " DEFAULT (" + col.Default + ")" |
|
||||||
} |
|
||||||
if col.IsAutoIncrement { |
|
||||||
sql += " GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)" |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
sql += ") PRIMARY KEY (" + strings.Join(table.PrimaryKeys, ",") + ")" |
|
||||||
return sql |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) CreateIndexSql(tableName string, index *core.Index) string { |
|
||||||
sql := "CREATE " |
|
||||||
if index.Type == core.UniqueType { |
|
||||||
sql += "UNIQUE NULL_FILTERED " |
|
||||||
} |
|
||||||
sql += "INDEX " + s.Quote(index.XName(tableName)) + " ON " + s.Quote(tableName) + " (" + strings.Join(index.Cols, ", ") + ")" |
|
||||||
return sql |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) IndexCheckSql(tableName, indexName string) (string, []any) { |
|
||||||
return `SELECT index_name FROM information_schema.indexes |
|
||||||
WHERE table_name = ? AND table_schema = "" AND index_name = ?`, |
|
||||||
[]any{tableName, indexName} |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) TableCheckSql(tableName string) (string, []any) { |
|
||||||
return `SELECT table_name FROM information_schema.tables |
|
||||||
WHERE table_name = ? AND table_schema = ""`, |
|
||||||
[]any{tableName} |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) GetTables() ([]*core.Table, error) { |
|
||||||
res, err := s.DB().Query(`SELECT table_name FROM information_schema.tables WHERE table_schema = ""`) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
defer res.Close() |
|
||||||
|
|
||||||
tables := []*core.Table{} |
|
||||||
for res.Next() { |
|
||||||
var name string |
|
||||||
if err := res.Scan(&name); err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
t := core.NewEmptyTable() |
|
||||||
t.Name = name |
|
||||||
tables = append(tables, t) |
|
||||||
} |
|
||||||
return tables, res.Err() |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) GetIndexes(tableName string) (map[string]*core.Index, error) { |
|
||||||
res, err := s.DB().Query(`SELECT ix.INDEX_NAME, ix.INDEX_TYPE, ix.IS_UNIQUE, c.COLUMN_NAME |
|
||||||
FROM INFORMATION_SCHEMA.INDEXES ix |
|
||||||
JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON (ix.TABLE_NAME=c.TABLE_NAME AND ix.INDEX_NAME=c.INDEX_NAME) |
|
||||||
WHERE ix.TABLE_SCHEMA = "" AND ix.TABLE_NAME=? |
|
||||||
ORDER BY ix.INDEX_NAME, c.ORDINAL_POSITION`, tableName) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
defer res.Close() |
|
||||||
|
|
||||||
indexes := map[string]*core.Index{} |
|
||||||
var ixName, ixType, colName string |
|
||||||
var isUnique bool |
|
||||||
for res.Next() { |
|
||||||
err := res.Scan(&ixName, &ixType, &isUnique, &colName) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
|
|
||||||
isRegular := false |
|
||||||
if strings.HasPrefix(ixName, "IDX_"+tableName) || strings.HasPrefix(ixName, "UQE_"+tableName) { |
|
||||||
ixName = ixName[5+len(tableName):] |
|
||||||
isRegular = true |
|
||||||
} |
|
||||||
|
|
||||||
var index *core.Index |
|
||||||
var ok bool |
|
||||||
if index, ok = indexes[ixName]; !ok { |
|
||||||
t := core.IndexType // ixType == "INDEX" && !isUnique
|
|
||||||
if ixType == "PRIMARY KEY" || isUnique { |
|
||||||
t = core.UniqueType |
|
||||||
} |
|
||||||
|
|
||||||
index = &core.Index{} |
|
||||||
index.IsRegular = isRegular |
|
||||||
index.Type = t |
|
||||||
index.Name = ixName |
|
||||||
indexes[ixName] = index |
|
||||||
} |
|
||||||
index.AddColumn(colName) |
|
||||||
} |
|
||||||
return indexes, res.Err() |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) CreateSequenceGenerator(db *sql.DB) (SequenceGenerator, error) { |
|
||||||
dsn := s.DataSourceName() |
|
||||||
connectorConfig, err := spannerdriver.ExtractConnectorConfig(dsn) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
|
|
||||||
if connectorConfig.Params[strings.ToLower("inMemSequenceGenerator")] == "true" { |
|
||||||
// Switch to using in-memory sequence number generator.
|
|
||||||
// Using database-based sequence generator doesn't work with emulator, as emulator
|
|
||||||
// only supports single transaction. If there is already another transaction started
|
|
||||||
// generating new ID via database-based sequence generator would always fail.
|
|
||||||
return newInMemSequenceGenerator(), nil |
|
||||||
} |
|
||||||
|
|
||||||
return newSequenceGenerator(db), nil |
|
||||||
} |
|
||||||
|
|
||||||
func (s *spanner) RetryOnError(err error) bool { |
|
||||||
return err != nil && spannerclient.ErrCode(spannerclient.ToSpannerError(err)) == codes.Aborted |
|
||||||
} |
|
||||||
@ -1,29 +0,0 @@ |
|||||||
//go:build enterprise || pro
|
|
||||||
|
|
||||||
package xorm |
|
||||||
|
|
||||||
import ( |
|
||||||
"fmt" |
|
||||||
"testing" |
|
||||||
|
|
||||||
"cloud.google.com/go/spanner/spannertest" |
|
||||||
_ "github.com/mattn/go-sqlite3" |
|
||||||
"github.com/stretchr/testify/require" |
|
||||||
) |
|
||||||
|
|
||||||
func TestBasicOperationsWithSpanner(t *testing.T) { |
|
||||||
span, err := spannertest.NewServer("localhost:0") |
|
||||||
require.NoError(t, err) |
|
||||||
defer span.Close() |
|
||||||
|
|
||||||
eng, err := NewEngine("spanner", fmt.Sprintf("%s/projects/test/instances/test/databases/test;usePlainText=true", span.Addr)) |
|
||||||
require.NoError(t, err) |
|
||||||
require.NotNil(t, eng) |
|
||||||
require.Equal(t, "spanner", eng.DriverName()) |
|
||||||
|
|
||||||
_, err = eng.Exec("CREATE TABLE test_struct (id int64, comment string(max), json string(max)) primary key (id)") |
|
||||||
require.NoError(t, err) |
|
||||||
|
|
||||||
// Currently broken because simple INSERT into spannertest doesn't work: https://github.com/googleapis/go-sql-spanner/issues/392
|
|
||||||
// testBasicOperations(t, eng)
|
|
||||||
} |
|
||||||
Loading…
Reference in new issue