SecretsManager: outbox use message id (#107472)

* SecretsManager: outbox use message id

Co-authored-by: PoorlyDefinedBehaviour <brunotj2015@hotmail.com>

* Remove query timestamp

* Add missing query

---------

Co-authored-by: PoorlyDefinedBehaviour <brunotj2015@hotmail.com>
Co-authored-by: Matheus Macabu <macabu@users.noreply.github.com>
Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>

pkg/registry/apis/secret/contracts/outbox_queue.go

@@ -41,7 +41,7 @@ type AppendOutboxMessage struct {
 type OutboxMessage struct {
 	RequestID       string
 	Type            OutboxMessageType
-	MessageID       string
+	MessageID       int64
 	Name            string
 	Namespace       string
 	EncryptedSecret string
@@ -49,15 +49,16 @@ type OutboxMessage struct {
 	ExternalID      *string
 	// How many times this message has been received
 	ReceiveCount int
+	Created      int64
 }
 
 type OutboxQueue interface {
 	// Appends a message to the outbox queue
-	Append(ctx context.Context, message AppendOutboxMessage) (string, error)
+	Append(ctx context.Context, message AppendOutboxMessage) (int64, error)
 	// Receives at most n messages from the outbox queue
 	ReceiveN(ctx context.Context, n uint) ([]OutboxMessage, error)
 	// Deletes a message from the outbox queue
-	Delete(ctx context.Context, messageID string) error
+	Delete(ctx context.Context, messageID int64) error
 	// Increments the number of times each message has been received by 1. Must be atomic.
-	IncrementReceiveCount(ctx context.Context, messageIDs []string) error
+	IncrementReceiveCount(ctx context.Context, messageIDs []int64) error
 }

pkg/storage/secret/metadata/data/secure_value_outbox_append.sql

@@ -1,6 +1,5 @@
 INSERT INTO {{ .Ident "secret_secure_value_outbox" }} (
   {{ .Ident "request_id" }},
-  {{ .Ident "uid" }},
   {{ .Ident "message_type" }},
   {{ .Ident "name" }},
   {{ .Ident "namespace" }},
@@ -17,7 +16,6 @@ INSERT INTO {{ .Ident "secret_secure_value_outbox" }} (
   {{ .Ident "created" }}
 ) VALUES (
   {{ .Arg .Row.RequestID }},
-  {{ .Arg .Row.MessageID }},
   {{ .Arg .Row.MessageType }},
   {{ .Arg .Row.Name }},
   {{ .Arg .Row.Namespace }},

pkg/storage/secret/metadata/data/secure_value_outbox_delete.sql

@@ -1,5 +1,5 @@
 DELETE FROM
   {{ .Ident "secret_secure_value_outbox" }}
 WHERE
-  {{ .Ident "uid" }} = {{ .Arg .MessageID }}
+  {{ .Ident "id" }} = {{ .Arg .MessageID }}
 ;

pkg/storage/secret/metadata/data/secure_value_outbox_fetch_message_ids.sql

@@ -0,0 +1,6 @@
+SELECT
+  {{ .Ident "id" }}
+FROM {{ .Ident "secret_secure_value_outbox" }}
+ORDER BY id ASC
+LIMIT {{ .Arg .ReceiveLimit }}
+;

pkg/storage/secret/metadata/data/secure_value_outbox_query_timestamp.sql

@@ -0,0 +1,8 @@
+SELECT
+  {{ .Ident "created" }},
+  {{ .Ident "message_type" }}
+FROM
+  {{ .Ident "secret_secure_value_outbox" }}
+WHERE
+  {{ .Ident "id" }} = {{ .Arg .MessageID }}
+;

pkg/storage/secret/metadata/data/secure_value_outbox_receiveN.sql

@@ -1,6 +1,6 @@
 SELECT
   {{ .Ident "request_id" }},
-  {{ .Ident "uid" }},
+  {{ .Ident "id" }},
   {{ .Ident "message_type" }},
   {{ .Ident "name" }},
   {{ .Ident "namespace" }},
@@ -11,9 +11,9 @@ SELECT
   {{ .Ident "created" }}
 FROM
   {{ .Ident "secret_secure_value_outbox" }}
+WHERE 
+  {{ .Ident "id" }} IN ({{ .ArgList .MessageIDs }})
 ORDER BY
-  {{ .Ident "created" }} ASC
-LIMIT
-  {{ .Arg .ReceiveLimit }}
+  {{ .Ident "id" }} ASC
 {{ .SelectFor "UPDATE SKIP LOCKED" }}
 ;

pkg/storage/secret/metadata/data/secure_value_outbox_update_receive_count.sql

@@ -3,5 +3,5 @@ UPDATE
 SET
   {{ .Ident "receive_count" }} = {{ .Ident "receive_count" }} + 1
 WHERE 
-  {{ .Ident "uid" }} IN ({{ .ArgList .MessageIDs }})
+  {{ .Ident "id" }} IN ({{ .ArgList .MessageIDs }})
 ;

pkg/storage/secret/metadata/outbox_store.go

@@ -11,7 +11,6 @@ import (
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 
-	"github.com/google/uuid"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/assert"
 	"github.com/grafana/grafana/pkg/registry/apis/secret/contracts"
 	"github.com/grafana/grafana/pkg/storage/unified/sql/sqltemplate"
@@ -23,7 +22,10 @@ type outboxStore struct {
 	tracer  trace.Tracer
 }
 
-func ProvideOutboxQueue(db contracts.Database, tracer trace.Tracer) contracts.OutboxQueue {
+func ProvideOutboxQueue(
+	db contracts.Database,
+	tracer trace.Tracer,
+) contracts.OutboxQueue {
 	return &outboxStore{
 		db:      db,
 		dialect: sqltemplate.DialectForDriver(db.DriverName()),
@@ -33,7 +35,7 @@ func ProvideOutboxQueue(db contracts.Database, tracer trace.Tracer) contracts.Ou
 
 type outboxMessageDB struct {
 	RequestID       string
-	MessageID       string
+	MessageID       int64
 	MessageType     contracts.OutboxMessageType
 	Name            string
 	Namespace       string
@@ -44,7 +46,7 @@ type outboxMessageDB struct {
 	Created         int64
 }
 
-func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMessage) (messageID string, err error) {
+func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMessage) (messageID int64, err error) {
 	ctx, span := s.tracer.Start(ctx, "outboxStore.Append", trace.WithAttributes(
 		attribute.String("name", input.Name),
 		attribute.String("namespace", input.Namespace),
@@ -59,8 +61,8 @@ func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMe
 			span.RecordError(err)
 		}
 
-		if messageID != "" {
-			span.SetAttributes(attribute.String("messageID", messageID))
+		if messageID != 0 {
+			span.SetAttributes(attribute.Int64("messageID", messageID))
 		}
 	}()
 
@@ -74,7 +76,7 @@ func (s *outboxStore) Append(ctx context.Context, input contracts.AppendOutboxMe
 	return messageID, nil
 }
 
-func (s *outboxStore) insertMessage(ctx context.Context, input contracts.AppendOutboxMessage) (string, error) {
+func (s *outboxStore) insertMessage(ctx context.Context, input contracts.AppendOutboxMessage) (int64, error) {
 	keeperName := sql.NullString{}
 	if input.KeeperName != nil {
 		keeperName = sql.NullString{
@@ -99,13 +101,10 @@ func (s *outboxStore) insertMessage(ctx context.Context, input contracts.AppendO
 		}
 	}
 
-	messageID := uuid.New().String()
-
 	req := appendSecureValueOutbox{
 		SQLTemplate: sqltemplate.New(s.dialect),
 		Row: &outboxMessageDB{
 			RequestID:       input.RequestID,
-			MessageID:       messageID,
 			MessageType:     input.Type,
 			Name:            input.Name,
 			Namespace:       input.Namespace,
@@ -119,33 +118,46 @@ func (s *outboxStore) insertMessage(ctx context.Context, input contracts.AppendO
 
 	query, err := sqltemplate.Execute(sqlSecureValueOutboxAppend, req)
 	if err != nil {
-		return messageID, fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxAppend.Name(), err)
+		return 0, fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxAppend.Name(), err)
 	}
 
 	result, err := s.db.ExecContext(ctx, query, req.GetArgs()...)
 	if err != nil {
 		if unifiedsql.IsRowAlreadyExistsError(err) {
-			return messageID, contracts.ErrSecureValueOperationInProgress
+			return 0, contracts.ErrSecureValueOperationInProgress
 		}
-		return messageID, fmt.Errorf("inserting message into secure value outbox table: %w", err)
+		return 0, fmt.Errorf("inserting message into secure value outbox table: %w", err)
 	}
 
 	rowsAffected, err := result.RowsAffected()
 	if err != nil {
-		return messageID, fmt.Errorf("get rows affected: %w", err)
+		return 0, fmt.Errorf("get rows affected: %w", err)
 	}
 
 	if rowsAffected != 1 {
-		return messageID, fmt.Errorf("expected to affect 1 row, but affected %d", rowsAffected)
+		return 0, fmt.Errorf("expected to affect 1 row, but affected %d", rowsAffected)
 	}
 
-	return messageID, nil
+	id, err := result.LastInsertId()
+	if err != nil {
+		return id, fmt.Errorf("fetching last inserted id: %w", err)
+	}
+
+	return id, nil
 }
 
-func (s *outboxStore) ReceiveN(ctx context.Context, n uint) ([]contracts.OutboxMessage, error) {
+func (s *outboxStore) ReceiveN(ctx context.Context, limit uint) ([]contracts.OutboxMessage, error) {
+	messageIDs, err := s.fetchMessageIdsInQueue(ctx, limit)
+	if err != nil {
+		return nil, fmt.Errorf("fetching message ids from queue: %w", err)
+	}
+	// If queue is empty
+	if len(messageIDs) == 0 {
+		return nil, nil
+	}
 	req := receiveNSecureValueOutbox{
-		SQLTemplate:  sqltemplate.New(s.dialect),
-		ReceiveLimit: n,
+		SQLTemplate: sqltemplate.New(s.dialect),
+		MessageIDs:  messageIDs,
 	}
 
 	query, err := sqltemplate.Execute(sqlSecureValueOutboxReceiveN, req)
@@ -197,6 +209,7 @@ func (s *outboxStore) ReceiveN(ctx context.Context, n uint) ([]contracts.OutboxM
 			KeeperName:   keeperName,
 			ExternalID:   externalID,
 			ReceiveCount: row.ReceiveCount,
+			Created:      row.Created,
 		}
 
 		if row.MessageType != contracts.DeleteSecretOutboxMessage && row.EncryptedSecret.Valid {
@@ -213,9 +226,43 @@ func (s *outboxStore) ReceiveN(ctx context.Context, n uint) ([]contracts.OutboxM
 	return messages, nil
 }
 
-func (s *outboxStore) Delete(ctx context.Context, messageID string) (err error) {
+func (s *outboxStore) fetchMessageIdsInQueue(ctx context.Context, limit uint) ([]int64, error) {
+	req := fetchMessageIDsOutbox{
+		SQLTemplate:  sqltemplate.New(s.dialect),
+		ReceiveLimit: limit,
+	}
+
+	query, err := sqltemplate.Execute(sqlSecureValueOutboxFetchMessageIDs, req)
+	if err != nil {
+		return nil, fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxFetchMessageIDs.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, query, req.GetArgs()...)
+	if err != nil {
+		return nil, fmt.Errorf("fetching rows from secure value outbox table: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	messageIDs := make([]int64, 0, limit)
+
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, fmt.Errorf("scanning row; %w", err)
+		}
+		messageIDs = append(messageIDs, id)
+	}
+
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("reading rows: %w", err)
+	}
+
+	return messageIDs, nil
+}
+
+func (s *outboxStore) Delete(ctx context.Context, messageID int64) (err error) {
 	ctx, span := s.tracer.Start(ctx, "outboxStore.Append", trace.WithAttributes(
-		attribute.String("messageID", messageID),
+		attribute.Int64("messageID", messageID),
 	))
 	defer span.End()
 
@@ -226,7 +273,7 @@ func (s *outboxStore) Delete(ctx context.Context, messageID string) (err error)
 		}
 	}()
 
-	assert.True(messageID != "", "outboxStore.Delete: messageID is required")
+	assert.True(messageID != 0, "outboxStore.Delete: messageID is required")
 
 	if err := s.deleteMessage(ctx, messageID); err != nil {
 		return fmt.Errorf("deleting message from outbox table %+w", err)
@@ -235,18 +282,56 @@ func (s *outboxStore) Delete(ctx context.Context, messageID string) (err error)
 	return nil
 }
 
-func (s *outboxStore) deleteMessage(ctx context.Context, messageID string) error {
-	req := deleteSecureValueOutbox{
+func (s *outboxStore) deleteMessage(ctx context.Context, messageID int64) error {
+	tsReq := getOutboxMessageTimestamp{
 		SQLTemplate: sqltemplate.New(s.dialect),
 		MessageID:   messageID,
 	}
 
-	query, err := sqltemplate.Execute(sqlSecureValueOutboxDelete, req)
+	// First query the object so we can get the timestamp and calculate the total lifetime
+	timestampQuery, err := sqltemplate.Execute(sqlSecureValueOutboxQueryTimestamp, tsReq)
+	if err != nil {
+		return fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxQueryTimestamp.Name(), err)
+	}
+
+	rows, err := s.db.QueryContext(ctx, timestampQuery, tsReq.GetArgs()...)
+	if err != nil {
+		return fmt.Errorf("querying timestamp from secure value outbox table: %w", err)
+	}
+
+	if !rows.Next() {
+		_ = rows.Close()
+		return fmt.Errorf("no row found for message id=%v", messageID)
+	}
+
+	var timestamp int64
+	var messageType string
+	if err := rows.Scan(&timestamp, &messageType); err != nil {
+		_ = rows.Close()
+		return fmt.Errorf("scanning timestamp: %w", err)
+	}
+
+	// Explicitly close rows and check for errors before proceeding
+	if err := rows.Close(); err != nil {
+		return fmt.Errorf("closing rows: %w", err)
+	}
+
+	if err := rows.Err(); err != nil {
+		return fmt.Errorf("rows error: %w", err)
+	}
+
+	// Then delete the object
+	delReq := deleteSecureValueOutbox{
+		SQLTemplate: sqltemplate.New(s.dialect),
+		MessageID:   messageID,
+	}
+
+	query, err := sqltemplate.Execute(sqlSecureValueOutboxDelete, delReq)
 	if err != nil {
 		return fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxDelete.Name(), err)
 	}
 
-	result, err := s.db.ExecContext(ctx, query, req.GetArgs()...)
+	result, err := s.db.ExecContext(ctx, query, delReq.GetArgs()...)
 	if err != nil {
 		return fmt.Errorf("deleting message id=%v from secure value outbox table: %w", messageID, err)
 	}
@@ -256,6 +341,7 @@ func (s *outboxStore) deleteMessage(ctx context.Context, messageID string) error
 		return fmt.Errorf("get rows affected: %w", err)
 	}
 
+	// TODO: Presumably it's a bug if we delete 0 rows?
 	if rowsAffected > 1 {
 		return fmt.Errorf("bug: deleted more than one row from the outbox table, should delete only one at a time: deleted=%v", rowsAffected)
 	}
@@ -263,7 +349,7 @@ func (s *outboxStore) deleteMessage(ctx context.Context, messageID string) error
 	return nil
 }
 
-func (s *outboxStore) IncrementReceiveCount(ctx context.Context, messageIDs []string) error {
+func (s *outboxStore) IncrementReceiveCount(ctx context.Context, messageIDs []int64) error {
 	if len(messageIDs) == 0 {
 		return nil
 	}
@@ -272,6 +358,7 @@ func (s *outboxStore) IncrementReceiveCount(ctx context.Context, messageIDs []st
 		SQLTemplate: sqltemplate.New(s.dialect),
 		MessageIDs:  messageIDs,
 	}
+
 	query, err := sqltemplate.Execute(sqlSecureValueOutboxUpdateReceiveCount, req)
 	if err != nil {
 		return fmt.Errorf("execute template %q: %w", sqlSecureValueOutboxUpdateReceiveCount.Name(), err)

pkg/storage/secret/metadata/outbox_store_test.go

@@ -24,7 +24,7 @@ func newOutboxStoreModel() *outboxStoreModel {
 	return &outboxStoreModel{}
 }
 
-func (model *outboxStoreModel) Append(messageID string, message contracts.AppendOutboxMessage) {
+func (model *outboxStoreModel) Append(messageID int64, message contracts.AppendOutboxMessage) {
 	model.rows = append(model.rows, contracts.OutboxMessage{
 		Type:            message.Type,
 		MessageID:       messageID,
@@ -44,7 +44,7 @@ func (model *outboxStoreModel) ReceiveN(n uint) []contracts.OutboxMessage {
 	return model.rows[:maxMessages]
 }
 
-func (model *outboxStoreModel) Delete(messageID string) {
+func (model *outboxStoreModel) Delete(messageID int64) {
 	oldLen := len(model.rows)
 	model.rows = slices.DeleteFunc(model.rows, func(m contracts.OutboxMessage) bool {
 		return m.MessageID == messageID
@@ -70,7 +70,7 @@ func TestOutboxStoreModel(t *testing.T) {
 	}
 
 	outboxMessage1 := contracts.OutboxMessage{
-		MessageID:       "message_id_1",
+		MessageID:       1,
 		Type:            contracts.CreateSecretOutboxMessage,
 		Name:            "s-1",
 		Namespace:       "n-1",
@@ -79,7 +79,7 @@ func TestOutboxStoreModel(t *testing.T) {
 	}
 
 	outboxMessage2 := contracts.OutboxMessage{
-		MessageID:       "message_id_2",
+		MessageID:       2,
 		Type:            contracts.CreateSecretOutboxMessage,
 		Name:            "s-1",
 		Namespace:       "n-1",
@@ -87,11 +87,11 @@ func TestOutboxStoreModel(t *testing.T) {
 		ExternalID:      nil,
 	}
 
-	model.Append("message_id_1", appendOutboxMessage)
+	model.Append(1, appendOutboxMessage)
 
 	require.Equal(t, []contracts.OutboxMessage{outboxMessage1}, model.ReceiveN(10))
 
-	model.Append("message_id_2", appendOutboxMessage)
+	model.Append(2, appendOutboxMessage)
 
 	require.Equal(t, []contracts.OutboxMessage{outboxMessage1, outboxMessage2}, model.ReceiveN(10))
 
@@ -206,9 +206,8 @@ func TestOutboxStoreProperty(t *testing.T) {
 	rng := rand.New(rand.NewSource(seed))
 
 	defer func() {
-		if err := recover(); err != nil || t.Failed() {
-			fmt.Printf("TestOutboxStoreProperty: err=%+v\n\nSEED=%+v", err, seed)
-			t.FailNow()
+		if t.Failed() {
+			fmt.Printf("TestOutboxStoreProperty: SEED=%+v\n\n", seed)
 		}
 	}()
 
@@ -227,6 +226,7 @@ func TestOutboxStoreProperty(t *testing.T) {
 			n := rng.Intn(3)
 			switch n {
 			case 0:
+				time.Sleep(1 * time.Microsecond)
 				message := contracts.AppendOutboxMessage{
 					Type:            contracts.CreateSecretOutboxMessage,
 					Name:            fmt.Sprintf("s-%d", i),
@@ -247,7 +247,9 @@ func TestOutboxStoreProperty(t *testing.T) {
 				modelMessages := model.ReceiveN(n)
 
 				require.Equal(t, len(modelMessages), len(messages))
-				require.Equal(t, modelMessages, messages)
+				for i := range len(modelMessages) {
+					require.Equal(t, modelMessages[i].MessageID, messages[i].MessageID)
+				}
 
 			case 2:
 				if len(model.rows) == 0 {

pkg/storage/secret/metadata/query.go

@@ -34,9 +34,11 @@ var (
 	sqlSecureValueReadForDecrypt   = mustTemplate("secure_value_read_for_decrypt.sql")
 
 	sqlSecureValueOutboxAppend             = mustTemplate("secure_value_outbox_append.sql")
+	sqlSecureValueOutboxFetchMessageIDs    = mustTemplate("secure_value_outbox_fetch_message_ids.sql")
 	sqlSecureValueOutboxReceiveN           = mustTemplate("secure_value_outbox_receiveN.sql")
 	sqlSecureValueOutboxDelete             = mustTemplate("secure_value_outbox_delete.sql")
 	sqlSecureValueOutboxUpdateReceiveCount = mustTemplate("secure_value_outbox_update_receive_count.sql")
+	sqlSecureValueOutboxQueryTimestamp     = mustTemplate("secure_value_outbox_query_timestamp.sql")
 )
 
 func mustTemplate(filename string) *template.Template {
@@ -109,26 +111,26 @@ func (r deleteKeeper) Validate() error {
 }
 
 // This is used at keeper store to validate create & update operations
-type listByNameKeeper struct {
+type listByNameSecureValue struct {
 	sqltemplate.SQLTemplate
-	Namespace   string
-	KeeperNames []string
+	Namespace        string
+	UsedSecureValues []string
 }
 
 // Validate is only used if we use `dbutil` from `unifiedstorage`
-func (r listByNameKeeper) Validate() error {
+func (r listByNameSecureValue) Validate() error {
 	return nil // TODO
 }
 
 // This is used at keeper store to validate create & update operations
-type listByNameSecureValue struct {
+type listByNameKeeper struct {
 	sqltemplate.SQLTemplate
-	Namespace        string
-	UsedSecureValues []string
+	Namespace   string
+	KeeperNames []string
 }
 
 // Validate is only used if we use `dbutil` from `unifiedstorage`
-func (r listByNameSecureValue) Validate() error {
+func (r listByNameKeeper) Validate() error {
 	return nil // TODO
 }
 
@@ -240,21 +242,35 @@ func (appendSecureValueOutbox) Validate() error { return nil }
 
 type receiveNSecureValueOutbox struct {
 	sqltemplate.SQLTemplate
-	ReceiveLimit uint
+	MessageIDs []int64
 }
 
 func (receiveNSecureValueOutbox) Validate() error { return nil }
 
+type fetchMessageIDsOutbox struct {
+	sqltemplate.SQLTemplate
+	ReceiveLimit uint
+}
+
+func (fetchMessageIDsOutbox) Validate() error { return nil }
+
 type deleteSecureValueOutbox struct {
 	sqltemplate.SQLTemplate
-	MessageID string
+	MessageID int64
 }
 
 func (deleteSecureValueOutbox) Validate() error { return nil }
 
+type getOutboxMessageTimestamp struct {
+	sqltemplate.SQLTemplate
+	MessageID int64
+}
+
+func (getOutboxMessageTimestamp) Validate() error { return nil }
+
 type incrementReceiveCountOutbox struct {
 	sqltemplate.SQLTemplate
-	MessageIDs []string
+	MessageIDs []int64
 }
 
 func (incrementReceiveCountOutbox) Validate() error { return nil }

pkg/storage/secret/metadata/query_test.go

@@ -312,7 +312,7 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					Name: "update-receive-count",
 					Data: &incrementReceiveCountOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
-						MessageIDs:  []string{"id1", "id2", "id3"},
+						MessageIDs:  []int64{1, 2, 3},
 					},
 				},
 			},
@@ -322,7 +322,7 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					Data: &appendSecureValueOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
 						Row: &outboxMessageDB{
-							MessageID:   "my-uuid",
+							MessageID:   1,
 							MessageType: "some-type",
 							Name:        "name",
 							Namespace:   "namespace",
@@ -337,7 +337,7 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					Data: &appendSecureValueOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
 						Row: &outboxMessageDB{
-							MessageID:       "my-uuid",
+							MessageID:       1,
 							MessageType:     "some-type",
 							Name:            "name",
 							Namespace:       "namespace",
@@ -352,7 +352,7 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					Data: &appendSecureValueOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
 						Row: &outboxMessageDB{
-							MessageID:       "my-uuid",
+							MessageID:       1,
 							MessageType:     "some-type",
 							Name:            "name",
 							Namespace:       "namespace",
@@ -367,7 +367,7 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					Data: &appendSecureValueOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
 						Row: &outboxMessageDB{
-							MessageID:       "my-uuid",
+							MessageID:       1,
 							MessageType:     "some-type",
 							Name:            "name",
 							Namespace:       "namespace",
@@ -379,23 +379,30 @@ func TestSecureValueOutboxQueries(t *testing.T) {
 					},
 				},
 			},
-
-			sqlSecureValueOutboxReceiveN: {
+			sqlSecureValueOutboxFetchMessageIDs: {
 				{
 					Name: "basic",
-					Data: &receiveNSecureValueOutbox{
+					Data: &fetchMessageIDsOutbox{
 						SQLTemplate:  mocks.NewTestingSQLTemplate(),
 						ReceiveLimit: 10,
 					},
 				},
 			},
-
+			sqlSecureValueOutboxReceiveN: {
+				{
+					Name: "basic",
+					Data: &receiveNSecureValueOutbox{
+						SQLTemplate: mocks.NewTestingSQLTemplate(),
+						MessageIDs:  []int64{1, 2, 3},
+					},
+				},
+			},
 			sqlSecureValueOutboxDelete: {
 				{
 					Name: "basic",
 					Data: &deleteSecureValueOutbox{
 						SQLTemplate: mocks.NewTestingSQLTemplate(),
-						MessageID:   "my-uuid",
+						MessageID:   1,
 					},
 				},
 			},

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-all-fields-present.sql

@@ -1,6 +1,5 @@
 INSERT INTO `secret_secure_value_outbox` (
   `request_id`,
-  `uid`,
   `message_type`,
   `name`,
   `namespace`,
@@ -11,7 +10,6 @@ INSERT INTO `secret_secure_value_outbox` (
   `created`
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-encrypted-secret.sql

@@ -1,6 +1,5 @@
 INSERT INTO `secret_secure_value_outbox` (
   `request_id`,
-  `uid`,
   `message_type`,
   `name`,
   `namespace`,
@@ -10,7 +9,6 @@ INSERT INTO `secret_secure_value_outbox` (
   `created`
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-external-id.sql

@@ -1,6 +1,5 @@
 INSERT INTO `secret_secure_value_outbox` (
   `request_id`,
-  `uid`,
   `message_type`,
   `name`,
   `namespace`,
@@ -10,7 +9,6 @@ INSERT INTO `secret_secure_value_outbox` (
   `created`
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_append-no-keeper-name.sql

@@ -1,6 +1,5 @@
 INSERT INTO `secret_secure_value_outbox` (
   `request_id`,
-  `uid`,
   `message_type`,
   `name`,
   `namespace`,
@@ -10,7 +9,6 @@ INSERT INTO `secret_secure_value_outbox` (
   `created`
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_delete-basic.sql

@@ -1,5 +1,5 @@
 DELETE FROM
   `secret_secure_value_outbox`
 WHERE
-  `uid` = 'my-uuid'
+  `id` = 1
 ;

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_fetch_message_ids-basic.sql

@@ -0,0 +1,6 @@
+SELECT
+  `id`
+FROM `secret_secure_value_outbox`
+ORDER BY id ASC
+LIMIT 10
+;

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_receiveN-basic.sql

@@ -1,6 +1,6 @@
 SELECT
   `request_id`,
-  `uid`,
+  `id`,
   `message_type`,
   `name`,
   `namespace`,
@@ -11,9 +11,9 @@ SELECT
   `created`
 FROM
   `secret_secure_value_outbox`
+WHERE 
+  `id` IN (1, 2, 3)
 ORDER BY
-  `created` ASC
-LIMIT
-  10
+  `id` ASC
 FOR UPDATE SKIP LOCKED
 ;

pkg/storage/secret/metadata/testdata/mysql--secure_value_outbox_update_receive_count-update-receive-count.sql

@@ -3,5 +3,5 @@ UPDATE
 SET
   `receive_count` = `receive_count` + 1
 WHERE 
-  `uid` IN ('id1', 'id2', 'id3')
+  `id` IN (1, 2, 3)
 ;

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-all-fields-present.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -11,7 +10,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-encrypted-secret.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-external-id.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_append-no-keeper-name.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_delete-basic.sql

@@ -1,5 +1,5 @@
 DELETE FROM
   "secret_secure_value_outbox"
 WHERE
-  "uid" = 'my-uuid'
+  "id" = 1
 ;

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_fetch_message_ids-basic.sql

@@ -0,0 +1,6 @@
+SELECT
+  "id"
+FROM "secret_secure_value_outbox"
+ORDER BY id ASC
+LIMIT 10
+;

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_receiveN-basic.sql

@@ -1,6 +1,6 @@
 SELECT
   "request_id",
-  "uid",
+  "id",
   "message_type",
   "name",
   "namespace",
@@ -11,9 +11,9 @@ SELECT
   "created"
 FROM
   "secret_secure_value_outbox"
+WHERE 
+  "id" IN (1, 2, 3)
 ORDER BY
-  "created" ASC
-LIMIT
-  10
+  "id" ASC
 FOR UPDATE SKIP LOCKED
 ;

pkg/storage/secret/metadata/testdata/postgres--secure_value_outbox_update_receive_count-update-receive-count.sql

@@ -3,5 +3,5 @@ UPDATE
 SET
   "receive_count" = "receive_count" + 1
 WHERE 
-  "uid" IN ('id1', 'id2', 'id3')
+  "id" IN (1, 2, 3)
 ;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-all-fields-present.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -11,7 +10,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-encrypted-secret.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-external-id.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_append-no-keeper-name.sql

@@ -1,6 +1,5 @@
 INSERT INTO "secret_secure_value_outbox" (
   "request_id",
-  "uid",
   "message_type",
   "name",
   "namespace",
@@ -10,7 +9,6 @@ INSERT INTO "secret_secure_value_outbox" (
   "created"
 ) VALUES (
   '',
-  'my-uuid',
   'some-type',
   'name',
   'namespace',

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_delete-basic.sql

@@ -1,5 +1,5 @@
 DELETE FROM
   "secret_secure_value_outbox"
 WHERE
-  "uid" = 'my-uuid'
+  "id" = 1
 ;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_fetch_message_ids-basic.sql

@@ -0,0 +1,6 @@
+SELECT
+  "id"
+FROM "secret_secure_value_outbox"
+ORDER BY id ASC
+LIMIT 10
+;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_receiveN-basic.sql

@@ -1,6 +1,6 @@
 SELECT
   "request_id",
-  "uid",
+  "id",
   "message_type",
   "name",
   "namespace",
@@ -11,8 +11,8 @@ SELECT
   "created"
 FROM
   "secret_secure_value_outbox"
+WHERE 
+  "id" IN (1, 2, 3)
 ORDER BY
-  "created" ASC
-LIMIT
-  10
+  "id" ASC
 ;

pkg/storage/secret/metadata/testdata/sqlite--secure_value_outbox_update_receive_count-update-receive-count.sql

@@ -3,5 +3,5 @@ UPDATE
 SET
   "receive_count" = "receive_count" + 1
 WHERE 
-  "uid" IN ('id1', 'id2', 'id3')
+  "id" IN (1, 2, 3)
 ;

pkg/storage/secret/migrator/migrator.go

@@ -134,7 +134,7 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 		Name: TableNameSecureValueOutbox,
 		Columns: []*migrator.Column{
 			{Name: "request_id", Type: migrator.DB_NVarchar, Length: 253, Nullable: false},
-			{Name: "uid", Type: migrator.DB_NVarchar, Length: 36, IsPrimaryKey: true}, // Fixed size of a UUID.
+			{Name: "id", Type: migrator.DB_BigInt, Length: 36, IsPrimaryKey: true, IsAutoIncrement: true}, // Fixed size of a UUID.
 			{Name: "message_type", Type: migrator.DB_NVarchar, Length: 16, Nullable: false},
 			{Name: "name", Type: migrator.DB_NVarchar, Length: 253, Nullable: false},      // Limit enforced by K8s.
 			{Name: "namespace", Type: migrator.DB_NVarchar, Length: 253, Nullable: false}, // Limit enforced by K8s.
@@ -148,8 +148,6 @@ func (*SecretDB) AddMigration(mg *migrator.Migrator) {
 			// There's only one operation per secret in the queue at all times,
 			// meaning the namespace + name combination should be unique
 			{Cols: []string{"namespace", "name"}, Type: migrator.UniqueIndex},
-			// Used for sorting
-			{Cols: []string{"created"}, Type: migrator.IndexType},
 		},
 	})