mirror of https://github.com/grafana/grafana
Remove support for Google Spanner database. (#105846)
* Remove support for Google Spanner database.pull/105930/head
Peter Štibraný
2 months ago
committed by
GitHub
parent
9769871a88
commit
c4d3eb1cd0
@ -1,355 +0,0 @@ |
||||
//go:build enterprise || pro
|
||||
|
||||
package migrator |
||||
|
||||
import ( |
||||
"context" |
||||
_ "embed" |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"strings" |
||||
"time" |
||||
|
||||
"cloud.google.com/go/spanner" |
||||
database "cloud.google.com/go/spanner/admin/database/apiv1" |
||||
"cloud.google.com/go/spanner/admin/database/apiv1/databasepb" |
||||
"github.com/googleapis/gax-go/v2" |
||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
||||
"github.com/grafana/grafana/pkg/util/xorm" |
||||
"google.golang.org/grpc/codes" |
||||
|
||||
"github.com/grafana/dskit/concurrency" |
||||
utilspanner "github.com/grafana/grafana/pkg/util/spanner" |
||||
"github.com/grafana/grafana/pkg/util/xorm/core" |
||||
) |
||||
|
||||
type SpannerDialect struct { |
||||
BaseDialect |
||||
d core.Dialect |
||||
} |
||||
|
||||
func init() { |
||||
supportedDialects[Spanner] = NewSpannerDialect |
||||
} |
||||
|
||||
func NewSpannerDialect() Dialect { |
||||
d := SpannerDialect{d: core.QueryDialect(Spanner)} |
||||
d.dialect = &d |
||||
d.driverName = Spanner |
||||
return &d |
||||
} |
||||
|
||||
func (s *SpannerDialect) AutoIncrStr() string { return s.d.AutoIncrStr() } |
||||
func (s *SpannerDialect) Quote(name string) string { return s.d.Quote(name) } |
||||
func (s *SpannerDialect) SupportEngine() bool { return s.d.SupportEngine() } |
||||
|
||||
func (s *SpannerDialect) LikeOperator(column string, wildcardBefore bool, pattern string, wildcardAfter bool) (string, string) { |
||||
param := strings.ToLower(pattern) |
||||
if wildcardBefore { |
||||
param = "%" + param |
||||
} |
||||
if wildcardAfter { |
||||
param = param + "%" |
||||
} |
||||
return fmt.Sprintf("LOWER(%s) LIKE ?", column), param |
||||
} |
||||
|
||||
func (s *SpannerDialect) IndexCheckSQL(tableName, indexName string) (string, []any) { |
||||
return s.d.IndexCheckSql(tableName, indexName) |
||||
} |
||||
func (s *SpannerDialect) SQLType(col *Column) string { |
||||
c := core.NewColumn(col.Name, "", core.SQLType{Name: col.Type}, col.Length, col.Length2, col.Nullable) |
||||
return s.d.SqlType(c) |
||||
} |
||||
|
||||
func (s *SpannerDialect) BatchSize() int { return 1000 } |
||||
|
||||
func (s *SpannerDialect) BooleanValue(b bool) any { |
||||
return b |
||||
} |
||||
|
||||
func (s *SpannerDialect) BooleanStr(b bool) string { |
||||
if b { |
||||
return "true" |
||||
} |
||||
return "false" |
||||
} |
||||
func (s *SpannerDialect) ErrorMessage(err error) string { |
||||
return spanner.ErrDesc(spanner.ToSpannerError(err)) |
||||
} |
||||
func (s *SpannerDialect) IsDeadlock(err error) bool { |
||||
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.Aborted |
||||
} |
||||
func (s *SpannerDialect) IsUniqueConstraintViolation(err error) bool { |
||||
return spanner.ErrCode(spanner.ToSpannerError(err)) == codes.AlreadyExists |
||||
} |
||||
|
||||
func (s *SpannerDialect) CreateTableSQL(table *Table) string { |
||||
t := core.NewEmptyTable() |
||||
t.Name = table.Name |
||||
t.PrimaryKeys = table.PrimaryKeys |
||||
for _, c := range table.Columns { |
||||
col := core.NewColumn(c.Name, c.Name, core.SQLType{Name: c.Type}, c.Length, c.Length2, c.Nullable) |
||||
col.IsAutoIncrement = c.IsAutoIncrement |
||||
col.Default = c.Default |
||||
t.AddColumn(col) |
||||
} |
||||
if len(t.PrimaryKeys) == 0 { |
||||
for _, ix := range table.Indices { |
||||
if ix.Name == "PRIMARY_KEY" { |
||||
t.PrimaryKeys = append(t.PrimaryKeys, ix.Cols...) |
||||
} |
||||
} |
||||
} |
||||
return s.d.CreateTableSql(t, t.Name, "", "") |
||||
} |
||||
|
||||
func (s *SpannerDialect) CreateIndexSQL(tableName string, index *Index) string { |
||||
idx := core.NewIndex(index.Name, index.Type) |
||||
idx.Cols = index.Cols |
||||
return s.d.CreateIndexSql(tableName, idx) |
||||
} |
||||
|
||||
func (s *SpannerDialect) UpsertMultipleSQL(tableName string, keyCols, updateCols []string, count int) (string, error) { |
||||
return "", errors.New("not supported") |
||||
} |
||||
|
||||
func (s *SpannerDialect) DropIndexSQL(tableName string, index *Index) string { |
||||
return fmt.Sprintf("DROP INDEX %v", s.Quote(index.XName(tableName))) |
||||
} |
||||
|
||||
func (s *SpannerDialect) DropTable(tableName string) string { |
||||
return fmt.Sprintf("DROP TABLE %s", s.Quote(tableName)) |
||||
} |
||||
|
||||
func (s *SpannerDialect) ColStringNoPk(col *Column) string { |
||||
sql := s.dialect.Quote(col.Name) + " " |
||||
|
||||
sql += s.dialect.SQLType(col) + " " |
||||
|
||||
if s.dialect.ShowCreateNull() && !col.Nullable { |
||||
sql += "NOT NULL " |
||||
} |
||||
|
||||
if col.Default != "" { |
||||
// Default value must be in parentheses.
|
||||
sql += "DEFAULT (" + s.dialect.Default(col) + ") " |
||||
} |
||||
|
||||
return sql |
||||
} |
||||
|
||||
func (s *SpannerDialect) TruncateDBTables(engine *xorm.Engine) error { |
||||
// Get tables names only, no columns or indexes.
|
||||
tables, err := engine.Dialect().GetTables() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
sess := engine.NewSession() |
||||
defer sess.Close() |
||||
|
||||
var statements []string |
||||
|
||||
for _, table := range tables { |
||||
switch table.Name { |
||||
case "": |
||||
continue |
||||
case "autoincrement_sequences": |
||||
// Don't delete sequence number for migration_log.id column.
|
||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE name <> 'migration_log:id'", s.Quote(table.Name))) |
||||
case "migration_log": |
||||
continue |
||||
case "dashboard_acl": |
||||
// keep default dashboard permissions
|
||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE dashboard_id != -1 AND org_id != -1;", s.Quote(table.Name))) |
||||
default: |
||||
statements = append(statements, fmt.Sprintf("DELETE FROM %v WHERE TRUE;", s.Quote(table.Name))) |
||||
} |
||||
} |
||||
|
||||
// Run statements concurrently.
|
||||
return concurrency.ForEachJob(context.Background(), len(statements), 10, func(ctx context.Context, idx int) error { |
||||
_, err := sess.Exec(statements[idx]) |
||||
return err |
||||
}) |
||||
} |
||||
|
||||
// CleanDB drops all existing tables and their indexes.
|
||||
func (s *SpannerDialect) CleanDB(engine *xorm.Engine) error { |
||||
tables, err := engine.DBMetas() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Collect all DROP statements.
|
||||
changeStreams, err := s.findChangeStreams(engine) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
statements := make([]string, 0, len(tables)+len(changeStreams)) |
||||
for _, cs := range changeStreams { |
||||
statements = append(statements, fmt.Sprintf("DROP CHANGE STREAM `%s`", cs)) |
||||
} |
||||
|
||||
for _, table := range tables { |
||||
// Indexes must be dropped first, otherwise dropping tables fails.
|
||||
for _, index := range table.Indexes { |
||||
if !index.IsRegular { |
||||
// Don't drop primary key.
|
||||
continue |
||||
} |
||||
sql := fmt.Sprintf("DROP INDEX %s", s.Quote(index.XName(table.Name))) |
||||
statements = append(statements, sql) |
||||
} |
||||
|
||||
sql := fmt.Sprintf("DROP TABLE %s", s.Quote(table.Name)) |
||||
statements = append(statements, sql) |
||||
} |
||||
|
||||
if len(statements) == 0 { |
||||
return nil |
||||
} |
||||
|
||||
return s.executeDDLStatements(context.Background(), engine, statements) |
||||
} |
||||
|
||||
//go:embed snapshot/spanner-ddl.json
|
||||
var snapshotDDL string |
||||
|
||||
//go:embed snapshot/spanner-log.json
|
||||
var snapshotMigrations string |
||||
|
||||
func (s *SpannerDialect) CreateDatabaseFromSnapshot(ctx context.Context, engine *xorm.Engine, tableName string) error { |
||||
var statements, migrationIDs []string |
||||
err := json.Unmarshal([]byte(snapshotDDL), &statements) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
err = json.Unmarshal([]byte(snapshotMigrations), &migrationIDs) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = s.executeDDLStatements(ctx, engine, statements) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
return s.recordMigrationsToMigrationLog(engine, migrationIDs, tableName) |
||||
} |
||||
|
||||
func (s *SpannerDialect) recordMigrationsToMigrationLog(engine *xorm.Engine, migrationIDs []string, tableName string) error { |
||||
now := time.Now() |
||||
makeRecord := func(id string) MigrationLog { |
||||
return MigrationLog{ |
||||
MigrationID: id, |
||||
SQL: "", |
||||
Success: true, |
||||
Timestamp: now, |
||||
} |
||||
} |
||||
|
||||
sess := engine.NewSession() |
||||
defer sess.Close() |
||||
|
||||
// Insert records in batches to avoid many roundtrips to database.
|
||||
// Inserting all records at once fails due to "Number of parameters in query exceeds the maximum
|
||||
// allowed limit of 950." error, so we use smaller batches.
|
||||
const batchSize = 100 |
||||
|
||||
if err := sess.Begin(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
records := make([]MigrationLog, 0, len(migrationIDs)) |
||||
for _, mid := range migrationIDs { |
||||
records = append(records, makeRecord(mid)) |
||||
|
||||
if len(records) >= batchSize { |
||||
if _, err := sess.Table(tableName).InsertMulti(records); err != nil { |
||||
err2 := sess.Rollback() |
||||
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2) |
||||
} |
||||
records = records[:0] |
||||
} |
||||
} |
||||
|
||||
// Insert remaining records.
|
||||
if len(records) > 0 { |
||||
if _, err := sess.Table(tableName).InsertMulti(records); err != nil { |
||||
err2 := sess.Rollback() |
||||
return errors.Join(fmt.Errorf("failed to insert migration logs: %w", err), err2) |
||||
} |
||||
} |
||||
|
||||
if err := sess.Commit(); err != nil { |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// Spanner can be very slow at executing single DDL statements (it can take up to a minute), but when
|
||||
// many DDL statements are batched together, Spanner is *much* faster (total time to execute all statements
|
||||
// is often in tens of seconds). We can't execute batch of DDL statements using sql wrapper, we use "database admin client"
|
||||
// from Spanner library instead.
|
||||
func (s *SpannerDialect) executeDDLStatements(ctx context.Context, engine *xorm.Engine, statements []string) error { |
||||
// Datasource name contains string used for sql.Open.
|
||||
dsn := engine.Dialect().DataSourceName() |
||||
cfg, err := spannerdriver.ExtractConnectorConfig(dsn) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
opts := utilspanner.ConnectorConfigToClientOptions(cfg) |
||||
|
||||
databaseAdminClient, err := database.NewDatabaseAdminClient(ctx, opts...) |
||||
if err != nil { |
||||
return fmt.Errorf("failed to create database admin client: %v", err) |
||||
} |
||||
//nolint:errcheck // If the databaseAdminClient.Close fails, we simply don't care.
|
||||
defer databaseAdminClient.Close() |
||||
|
||||
databaseName := fmt.Sprintf("projects/%s/instances/%s/databases/%s", cfg.Project, cfg.Instance, cfg.Database) |
||||
|
||||
op, err := databaseAdminClient.UpdateDatabaseDdl(ctx, &databasepb.UpdateDatabaseDdlRequest{ |
||||
Database: databaseName, |
||||
Statements: statements, |
||||
}, gax.WithTimeout(0)) /* disable default timeout */ |
||||
if err != nil { |
||||
return fmt.Errorf("failed to start database DDL update: %v", err) |
||||
} |
||||
|
||||
err = op.Wait(ctx, gax.WithTimeout(0)) /* disable default timeout */ |
||||
if err != nil { |
||||
return fmt.Errorf("failed to apply database DDL update: %v", err) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
func (s *SpannerDialect) UnionDistinct() string { |
||||
return "UNION DISTINCT" |
||||
} |
||||
|
||||
func (s *SpannerDialect) findChangeStreams(engine *xorm.Engine) ([]string, error) { |
||||
var result []string |
||||
query := `SELECT c.CHANGE_STREAM_NAME |
||||
FROM INFORMATION_SCHEMA.CHANGE_STREAMS AS C |
||||
WHERE C.CHANGE_STREAM_CATALOG='' |
||||
AND C.CHANGE_STREAM_SCHEMA=''` |
||||
rows, err := engine.DB().Query(query) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
//nolint:errcheck // If the rows.Close fails, we simply don't care.
|
||||
defer rows.Close() |
||||
for rows.Next() { |
||||
var name string |
||||
if err := rows.Scan(&name); err != nil { |
||||
return nil, err |
||||
} |
||||
result = append(result, name) |
||||
} |
||||
return result, nil |
||||
} |
||||
@ -1,32 +0,0 @@ |
||||
package sqltemplate |
||||
|
||||
// Spanner is an implementation of Dialect for the Google Spanner database.
|
||||
var Spanner = spanner{} |
||||
|
||||
var _ Dialect = Spanner |
||||
|
||||
type spanner struct{} |
||||
|
||||
func (s spanner) DialectName() string { |
||||
return "spanner" |
||||
} |
||||
|
||||
func (s spanner) Ident(a string) (string, error) { |
||||
return backtickIdent{}.Ident(a) |
||||
} |
||||
|
||||
func (s spanner) ArgPlaceholder(argNum int) string { |
||||
return argFmtSQL92.ArgPlaceholder(argNum) |
||||
} |
||||
|
||||
func (s spanner) SelectFor(a ...string) (string, error) { |
||||
return rowLockingClauseSpanner.SelectFor(a...) |
||||
} |
||||
|
||||
func (spanner) CurrentEpoch() string { |
||||
return "UNIX_MICROS(CURRENT_TIMESTAMP())" |
||||
} |
||||
|
||||
var rowLockingClauseSpanner = rowLockingClauseMap{ |
||||
SelectForUpdate: SelectForUpdate, |
||||
} |
||||
@ -1,40 +0,0 @@ |
||||
// Package spanner should only be used from tests, or from enterprise code (eg. protected by build tags).
|
||||
package spanner |
||||
|
||||
import ( |
||||
"strconv" |
||||
|
||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
||||
"google.golang.org/api/option" |
||||
"google.golang.org/grpc" |
||||
"google.golang.org/grpc/credentials/insecure" |
||||
) |
||||
|
||||
func UsePlainText(connectorConfig spannerdriver.ConnectorConfig) bool { |
||||
if strval, ok := connectorConfig.Params["useplaintext"]; ok { |
||||
if val, err := strconv.ParseBool(strval); err == nil { |
||||
return val |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// ConnectorConfigToClientOptions is adapted from https://github.com/googleapis/go-sql-spanner/blob/main/driver.go#L341-L477, from version 1.11.1.
|
||||
func ConnectorConfigToClientOptions(connectorConfig spannerdriver.ConnectorConfig) []option.ClientOption { |
||||
var opts []option.ClientOption |
||||
if connectorConfig.Host != "" { |
||||
opts = append(opts, option.WithEndpoint(connectorConfig.Host)) |
||||
} |
||||
if strval, ok := connectorConfig.Params["credentials"]; ok { |
||||
opts = append(opts, option.WithCredentialsFile(strval)) |
||||
} |
||||
if strval, ok := connectorConfig.Params["credentialsjson"]; ok { |
||||
opts = append(opts, option.WithCredentialsJSON([]byte(strval))) |
||||
} |
||||
if UsePlainText(connectorConfig) { |
||||
opts = append(opts, |
||||
option.WithGRPCDialOption(grpc.WithTransportCredentials(insecure.NewCredentials())), |
||||
option.WithoutAuthentication()) |
||||
} |
||||
return opts |
||||
} |
||||
@ -1,399 +0,0 @@ |
||||
//go:build enterprise || pro
|
||||
|
||||
package xorm |
||||
|
||||
import ( |
||||
"database/sql" |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
spannerclient "cloud.google.com/go/spanner" |
||||
_ "github.com/googleapis/go-sql-spanner" |
||||
spannerdriver "github.com/googleapis/go-sql-spanner" |
||||
"github.com/grafana/grafana/pkg/util/xorm/core" |
||||
"google.golang.org/grpc/codes" |
||||
) |
||||
|
||||
func init() { |
||||
core.RegisterDriver("spanner", &spannerDriver{}) |
||||
core.RegisterDialect("spanner", func() core.Dialect { return &spanner{} }) |
||||
} |
||||
|
||||
// https://cloud.google.com/spanner/docs/reference/standard-sql/lexical#reserved_keywords
|
||||
var spannerReservedKeywords = map[string]struct{}{ |
||||
"ALL": {}, |
||||
"AND": {}, |
||||
"ANY": {}, |
||||
"ARRAY": {}, |
||||
"AS": {}, |
||||
"ASC": {}, |
||||
"ASSERT_ROWS_MODIFIED": {}, |
||||
"AT": {}, |
||||
"BETWEEN": {}, |
||||
"BY": {}, |
||||
"CASE": {}, |
||||
"CAST": {}, |
||||
"COLLATE": {}, |
||||
"CONTAINS": {}, |
||||
"CREATE": {}, |
||||
"CROSS": {}, |
||||
"CUBE": {}, |
||||
"CURRENT": {}, |
||||
"DEFAULT": {}, |
||||
"DEFINE": {}, |
||||
"DESC": {}, |
||||
"DISTINCT": {}, |
||||
"ELSE": {}, |
||||
"END": {}, |
||||
"ENUM": {}, |
||||
"ESCAPE": {}, |
||||
"EXCEPT": {}, |
||||
"EXCLUDE": {}, |
||||
"EXISTS": {}, |
||||
"EXTRACT": {}, |
||||
"FALSE": {}, |
||||
"FETCH": {}, |
||||
"FOLLOWING": {}, |
||||
"FOR": {}, |
||||
"FROM": {}, |
||||
"FULL": {}, |
||||
"GROUP": {}, |
||||
"GROUPING": {}, |
||||
"GROUPS": {}, |
||||
"HASH": {}, |
||||
"HAVING": {}, |
||||
"IF": {}, |
||||
"IGNORE": {}, |
||||
"IN": {}, |
||||
"INNER": {}, |
||||
"INTERSECT": {}, |
||||
"INTERVAL": {}, |
||||
"INTO": {}, |
||||
"IS": {}, |
||||
"JOIN": {}, |
||||
"LATERAL": {}, |
||||
"LEFT": {}, |
||||
"LIKE": {}, |
||||
"LIMIT": {}, |
||||
"LOOKUP": {}, |
||||
"MERGE": {}, |
||||
"NATURAL": {}, |
||||
"NEW": {}, |
||||
"NO": {}, |
||||
"NOT": {}, |
||||
"NULL": {}, |
||||
"NULLS": {}, |
||||
"OF": {}, |
||||
"ON": {}, |
||||
"OR": {}, |
||||
"ORDER": {}, |
||||
"OUTER": {}, |
||||
"OVER": {}, |
||||
"PARTITION": {}, |
||||
"PRECEDING": {}, |
||||
"PROTO": {}, |
||||
"RANGE": {}, |
||||
"RECURSIVE": {}, |
||||
"RESPECT": {}, |
||||
"RIGHT": {}, |
||||
"ROLLUP": {}, |
||||
"ROWS": {}, |
||||
"SELECT": {}, |
||||
"SET": {}, |
||||
"SOME": {}, |
||||
"STRUCT": {}, |
||||
"TABLESAMPLE": {}, |
||||
"THEN": {}, |
||||
"TO": {}, |
||||
"TREAT": {}, |
||||
"TRUE": {}, |
||||
"UNBOUNDED": {}, |
||||
"UNION": {}, |
||||
"UNNEST": {}, |
||||
"USING": {}, |
||||
"WHEN": {}, |
||||
"WHERE": {}, |
||||
"WINDOW": {}, |
||||
"WITH": {}, |
||||
"WITHIN": {}, |
||||
} |
||||
|
||||
type spannerDriver struct{} |
||||
|
||||
func (d *spannerDriver) Parse(_driverName, datasourceName string) (*core.Uri, error) { |
||||
return &core.Uri{DbType: "spanner", DbName: datasourceName}, nil |
||||
} |
||||
|
||||
type spanner struct { |
||||
core.Base |
||||
} |
||||
|
||||
func (s *spanner) Init(db *core.DB, uri *core.Uri, driverName string, datasourceName string) error { |
||||
return s.Base.Init(db, s, uri, driverName, datasourceName) |
||||
} |
||||
func (s *spanner) Filters() []core.Filter { return []core.Filter{&core.IdFilter{}} } |
||||
func (s *spanner) IsReserved(name string) bool { |
||||
_, exists := spannerReservedKeywords[name] |
||||
return exists |
||||
} |
||||
func (s *spanner) AndStr() string { return "AND" } |
||||
func (s *spanner) OrStr() string { return "OR" } |
||||
func (s *spanner) EqStr() string { return "=" } |
||||
func (s *spanner) RollBackStr() string { return "ROLL BACK" } |
||||
func (s *spanner) AutoIncrStr() string { |
||||
// Spanner does not support auto-increment, but supports unique generated IDs (not sequential!).
|
||||
return "GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)" |
||||
} |
||||
func (s *spanner) SupportInsertMany() bool { return false } // Needs manual transaction batching
|
||||
func (s *spanner) SupportEngine() bool { return false } // No support for engine selection
|
||||
func (s *spanner) SupportCharset() bool { return false } // ...or charsets
|
||||
func (s *spanner) SupportDropIfExists() bool { return false } // Drop should be handled differently
|
||||
func (s *spanner) IndexOnTable() bool { return false } |
||||
func (s *spanner) ShowCreateNull() bool { return false } |
||||
func (s *spanner) Quote(name string) string { return "`" + name + "`" } |
||||
func (s *spanner) SqlType(col *core.Column) string { |
||||
switch col.SQLType.Name { |
||||
case core.Int, core.SmallInt, core.BigInt: |
||||
return "INT64" |
||||
case core.Varchar, core.Text, core.MediumText, core.LongText, core.Char, core.NVarchar, core.NChar, core.NText: |
||||
l := col.Length |
||||
if l == 0 { |
||||
l = col.SQLType.DefaultLength |
||||
} |
||||
if l > 0 { |
||||
return fmt.Sprintf("STRING(%d)", l) |
||||
} |
||||
return "STRING(MAX)" |
||||
case core.Jsonb: |
||||
return "STRING(MAX)" |
||||
case core.Bool, core.TinyInt: |
||||
return "BOOL" |
||||
case core.Float, core.Double: |
||||
return "FLOAT64" |
||||
case core.Bytea, core.Blob, core.MediumBlob, core.LongBlob: |
||||
l := col.Length |
||||
if l == 0 { |
||||
l = col.SQLType.DefaultLength |
||||
} |
||||
if l > 0 { |
||||
return fmt.Sprintf("BYTES(%d)", l) |
||||
} |
||||
return "BYTES(MAX)" |
||||
case core.DateTime, core.TimeStamp: |
||||
return "TIMESTAMP" |
||||
default: |
||||
panic("unknown column type: " + col.SQLType.Name) |
||||
//default:
|
||||
// return "STRING(MAX)" // XXX: more types to add
|
||||
} |
||||
} |
||||
|
||||
func (s *spanner) GetColumns(tableName string) ([]string, map[string]*core.Column, error) { |
||||
query := `SELECT COLUMN_NAME, SPANNER_TYPE, IS_NULLABLE, IS_IDENTITY, IDENTITY_GENERATION, IDENTITY_KIND, COLUMN_DEFAULT |
||||
FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = ? AND TABLE_SCHEMA="" ORDER BY ORDINAL_POSITION` |
||||
rows, err := s.DB().Query(query, tableName) |
||||
if err != nil { |
||||
return nil, nil, err |
||||
} |
||||
defer rows.Close() |
||||
|
||||
columns := make(map[string]*core.Column) |
||||
var colNames []string |
||||
|
||||
var name, sqlType, isNullable string |
||||
var isIdentity, identityGeneration, identityKind, columnDefault sql.NullString |
||||
for rows.Next() { |
||||
if err := rows.Scan(&name, &sqlType, &isNullable, &isIdentity, &identityGeneration, &identityKind, &columnDefault); err != nil { |
||||
return nil, nil, err |
||||
} |
||||
|
||||
var length int |
||||
switch { |
||||
case sqlType == "INT64": |
||||
sqlType = core.Int |
||||
case sqlType == "FLOAT32" || sqlType == "FLOAT64": |
||||
sqlType = core.Float |
||||
case sqlType == "BOOL": |
||||
sqlType = core.Bool |
||||
case sqlType == "BYTES(MAX)": |
||||
sqlType = core.Blob |
||||
case sqlType == "STRING(MAX)": |
||||
sqlType = core.NVarchar |
||||
case sqlType == "TIMESTAMP": |
||||
sqlType = core.DateTime |
||||
case strings.HasPrefix(sqlType, "BYTES("): |
||||
// 6 == len(`BYTES(`), we also remove ")" from the end.
|
||||
if l, err := strconv.Atoi(sqlType[6 : len(sqlType)-1]); err == nil { |
||||
length = l |
||||
} |
||||
sqlType = core.Blob |
||||
case strings.HasPrefix(sqlType, "STRING("): |
||||
// 7 == len(`STRING(`), we also remove ")" from the end.
|
||||
if l, err := strconv.Atoi(sqlType[7 : len(sqlType)-1]); err == nil { |
||||
length = l |
||||
} |
||||
sqlType = core.Varchar |
||||
default: |
||||
panic("unknown column type: " + sqlType) |
||||
} |
||||
|
||||
autoincrement := isIdentity.Valid && isIdentity.String == "YES" && |
||||
identityGeneration.Valid && identityGeneration.String == "BY DEFAULT" && |
||||
identityKind.Valid && identityKind.String == "BIT_REVERSED_POSITIVE_SEQUENCE" |
||||
|
||||
defValue := "" |
||||
defEmpty := true |
||||
if columnDefault.Valid { |
||||
defValue = columnDefault.String |
||||
defEmpty = false |
||||
} |
||||
|
||||
col := &core.Column{ |
||||
Name: name, |
||||
SQLType: core.SQLType{Name: sqlType}, |
||||
Length: length, |
||||
Nullable: isNullable == "YES", |
||||
IsAutoIncrement: autoincrement, |
||||
Indexes: map[string]int{}, |
||||
Default: defValue, |
||||
DefaultIsEmpty: defEmpty, |
||||
} |
||||
columns[name] = col |
||||
colNames = append(colNames, name) |
||||
} |
||||
|
||||
return colNames, columns, rows.Err() |
||||
} |
||||
|
||||
func (s *spanner) CreateTableSql(table *core.Table, tableName, _, charset string) string { |
||||
sql := "CREATE TABLE " + s.Quote(tableName) + " (" |
||||
|
||||
for i, col := range table.Columns() { |
||||
if i > 0 { |
||||
sql += ", " |
||||
} |
||||
|
||||
sql += s.Quote(col.Name) + " " + s.SqlType(col) |
||||
if !col.Nullable { |
||||
sql += " NOT NULL" |
||||
} |
||||
if col.Default != "" { |
||||
sql += " DEFAULT (" + col.Default + ")" |
||||
} |
||||
if col.IsAutoIncrement { |
||||
sql += " GENERATED BY DEFAULT AS IDENTITY (BIT_REVERSED_POSITIVE)" |
||||
} |
||||
} |
||||
|
||||
sql += ") PRIMARY KEY (" + strings.Join(table.PrimaryKeys, ",") + ")" |
||||
return sql |
||||
} |
||||
|
||||
func (s *spanner) CreateIndexSql(tableName string, index *core.Index) string { |
||||
sql := "CREATE " |
||||
if index.Type == core.UniqueType { |
||||
sql += "UNIQUE NULL_FILTERED " |
||||
} |
||||
sql += "INDEX " + s.Quote(index.XName(tableName)) + " ON " + s.Quote(tableName) + " (" + strings.Join(index.Cols, ", ") + ")" |
||||
return sql |
||||
} |
||||
|
||||
func (s *spanner) IndexCheckSql(tableName, indexName string) (string, []any) { |
||||
return `SELECT index_name FROM information_schema.indexes |
||||
WHERE table_name = ? AND table_schema = "" AND index_name = ?`, |
||||
[]any{tableName, indexName} |
||||
} |
||||
|
||||
func (s *spanner) TableCheckSql(tableName string) (string, []any) { |
||||
return `SELECT table_name FROM information_schema.tables |
||||
WHERE table_name = ? AND table_schema = ""`, |
||||
[]any{tableName} |
||||
} |
||||
|
||||
func (s *spanner) GetTables() ([]*core.Table, error) { |
||||
res, err := s.DB().Query(`SELECT table_name FROM information_schema.tables WHERE table_schema = ""`) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
defer res.Close() |
||||
|
||||
tables := []*core.Table{} |
||||
for res.Next() { |
||||
var name string |
||||
if err := res.Scan(&name); err != nil { |
||||
return nil, err |
||||
} |
||||
t := core.NewEmptyTable() |
||||
t.Name = name |
||||
tables = append(tables, t) |
||||
} |
||||
return tables, res.Err() |
||||
} |
||||
|
||||
func (s *spanner) GetIndexes(tableName string) (map[string]*core.Index, error) { |
||||
res, err := s.DB().Query(`SELECT ix.INDEX_NAME, ix.INDEX_TYPE, ix.IS_UNIQUE, c.COLUMN_NAME |
||||
FROM INFORMATION_SCHEMA.INDEXES ix |
||||
JOIN INFORMATION_SCHEMA.INDEX_COLUMNS c ON (ix.TABLE_NAME=c.TABLE_NAME AND ix.INDEX_NAME=c.INDEX_NAME) |
||||
WHERE ix.TABLE_SCHEMA = "" AND ix.TABLE_NAME=? |
||||
ORDER BY ix.INDEX_NAME, c.ORDINAL_POSITION`, tableName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
defer res.Close() |
||||
|
||||
indexes := map[string]*core.Index{} |
||||
var ixName, ixType, colName string |
||||
var isUnique bool |
||||
for res.Next() { |
||||
err := res.Scan(&ixName, &ixType, &isUnique, &colName) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
isRegular := false |
||||
if strings.HasPrefix(ixName, "IDX_"+tableName) || strings.HasPrefix(ixName, "UQE_"+tableName) { |
||||
ixName = ixName[5+len(tableName):] |
||||
isRegular = true |
||||
} |
||||
|
||||
var index *core.Index |
||||
var ok bool |
||||
if index, ok = indexes[ixName]; !ok { |
||||
t := core.IndexType // ixType == "INDEX" && !isUnique
|
||||
if ixType == "PRIMARY KEY" || isUnique { |
||||
t = core.UniqueType |
||||
} |
||||
|
||||
index = &core.Index{} |
||||
index.IsRegular = isRegular |
||||
index.Type = t |
||||
index.Name = ixName |
||||
indexes[ixName] = index |
||||
} |
||||
index.AddColumn(colName) |
||||
} |
||||
return indexes, res.Err() |
||||
} |
||||
|
||||
func (s *spanner) CreateSequenceGenerator(db *sql.DB) (SequenceGenerator, error) { |
||||
dsn := s.DataSourceName() |
||||
connectorConfig, err := spannerdriver.ExtractConnectorConfig(dsn) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if connectorConfig.Params[strings.ToLower("inMemSequenceGenerator")] == "true" { |
||||
// Switch to using in-memory sequence number generator.
|
||||
// Using database-based sequence generator doesn't work with emulator, as emulator
|
||||
// only supports single transaction. If there is already another transaction started
|
||||
// generating new ID via database-based sequence generator would always fail.
|
||||
return newInMemSequenceGenerator(), nil |
||||
} |
||||
|
||||
return newSequenceGenerator(db), nil |
||||
} |
||||
|
||||
func (s *spanner) RetryOnError(err error) bool { |
||||
return err != nil && spannerclient.ErrCode(spannerclient.ToSpannerError(err)) == codes.Aborted |
||||
} |
||||
@ -1,29 +0,0 @@ |
||||
//go:build enterprise || pro
|
||||
|
||||
package xorm |
||||
|
||||
import ( |
||||
"fmt" |
||||
"testing" |
||||
|
||||
"cloud.google.com/go/spanner/spannertest" |
||||
_ "github.com/mattn/go-sqlite3" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestBasicOperationsWithSpanner(t *testing.T) { |
||||
span, err := spannertest.NewServer("localhost:0") |
||||
require.NoError(t, err) |
||||
defer span.Close() |
||||
|
||||
eng, err := NewEngine("spanner", fmt.Sprintf("%s/projects/test/instances/test/databases/test;usePlainText=true", span.Addr)) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, eng) |
||||
require.Equal(t, "spanner", eng.DriverName()) |
||||
|
||||
_, err = eng.Exec("CREATE TABLE test_struct (id int64, comment string(max), json string(max)) primary key (id)") |
||||
require.NoError(t, err) |
||||
|
||||
// Currently broken because simple INSERT into spannertest doesn't work: https://github.com/googleapis/go-sql-spanner/issues/392
|
||||
// testBasicOperations(t, eng)
|
||||
} |
||||
Loading…
Reference in new issue