feat(core): implement sync outbox mechanism and refactor provider validation

- Introduce `SyncOutboxService` and model to retry failed CP-to-Redis sync operations - Update `SyncService` to handle sync failures by enqueuing tasks to the outbox - Centralize provider group and API key validation logic into `ProviderGroupManager` - Refactor API handlers to utilize the new manager and robust sync methods - Add configuration options for sync outbox (interval, batch size, retries)
2026-01-14 02:37:52 +00:00 · 2025-12-25 01:24:19 +08:00
parent 44a82fa252
commit 6a16712b9d
12 changed files with 750 additions and 113 deletions
--- a/internal/service/sync.go
+++ b/internal/service/sync.go
@@ -17,16 +17,46 @@ import (
 )

 type SyncService struct {
-	rdb *redis.Client
+	rdb    *redis.Client
+	outbox *SyncOutboxService
 }

 func NewSyncService(rdb *redis.Client) *SyncService {
 	return &SyncService{rdb: rdb}
 }

+// SetOutbox enables sync outbox retry for this service.
+func (s *SyncService) SetOutbox(outbox *SyncOutboxService) {
+	s.outbox = outbox
+}
+
 // SyncKey writes a single key into Redis without rebuilding the entire snapshot.
 func (s *SyncService) SyncKey(key *model.Key) error {
-	ctx := context.Background()
+	if key == nil {
+		return fmt.Errorf("key required")
+	}
+	tokenHash := key.TokenHash
+	if strings.TrimSpace(tokenHash) == "" {
+		tokenHash = tokenhash.HashToken(key.KeySecret) // backward compatibility
+	}
+	if strings.TrimSpace(tokenHash) == "" {
+		return fmt.Errorf("token hash missing for key %d", key.ID)
+	}
+	return s.handleSyncError(s.SyncKeyNow(context.Background(), key), SyncOutboxEntry{
+		ResourceType: "key",
+		Action:       "upsert",
+		ResourceID:   &key.ID,
+	})
+}
+
+// SyncKeyNow writes key metadata to Redis without outbox handling.
+func (s *SyncService) SyncKeyNow(ctx context.Context, key *model.Key) error {
+	if key == nil {
+		return fmt.Errorf("key required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	tokenHash := key.TokenHash
 	if strings.TrimSpace(tokenHash) == "" {
 		tokenHash = tokenhash.HashToken(key.KeySecret) // backward compatibility
@@ -65,7 +95,24 @@ func (s *SyncService) SyncKey(key *model.Key) error {

 // SyncMaster writes master metadata into Redis used by the balancer for validation.
 func (s *SyncService) SyncMaster(master *model.Master) error {
-	ctx := context.Background()
+	if master == nil {
+		return fmt.Errorf("master required")
+	}
+	return s.handleSyncError(s.SyncMasterNow(context.Background(), master), SyncOutboxEntry{
+		ResourceType: "master",
+		Action:       "upsert",
+		ResourceID:   &master.ID,
+	})
+}
+
+// SyncMasterNow writes master metadata to Redis without outbox handling.
+func (s *SyncService) SyncMasterNow(ctx context.Context, master *model.Master) error {
+	if master == nil {
+		return fmt.Errorf("master required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	key := fmt.Sprintf("auth:master:%d", master.ID)
 	if err := s.rdb.HSet(ctx, key, map[string]interface{}{
 		"epoch":      master.Epoch,
@@ -79,10 +126,45 @@ func (s *SyncService) SyncMaster(master *model.Master) error {

 // SyncProviders rebuilds provider snapshots from ProviderGroup + APIKey tables.
 func (s *SyncService) SyncProviders(db *gorm.DB) error {
+	return s.syncProviders(db, SyncOutboxEntry{
+		ResourceType: "snapshot",
+		Action:       "sync_providers",
+	})
+}
+
+// SyncProvidersForGroup retries provider snapshot with provider_group context.
+func (s *SyncService) SyncProvidersForGroup(db *gorm.DB, groupID uint) error {
+	return s.syncProviders(db, SyncOutboxEntry{
+		ResourceType: "provider_group",
+		Action:       "sync_providers",
+		ResourceID:   &groupID,
+	})
+}
+
+// SyncProvidersForAPIKey retries provider snapshot with api_key context.
+func (s *SyncService) SyncProvidersForAPIKey(db *gorm.DB, apiKeyID uint) error {
+	return s.syncProviders(db, SyncOutboxEntry{
+		ResourceType: "api_key",
+		Action:       "sync_providers",
+		ResourceID:   &apiKeyID,
+	})
+}
+
+func (s *SyncService) syncProviders(db *gorm.DB, entry SyncOutboxEntry) error {
 	if db == nil {
 		return fmt.Errorf("db required")
 	}
-	ctx := context.Background()
+	return s.handleSyncError(s.SyncProvidersNow(context.Background(), db), entry)
+}
+
+// SyncProvidersNow rebuilds provider snapshots without outbox handling.
+func (s *SyncService) SyncProvidersNow(ctx context.Context, db *gorm.DB) error {
+	if db == nil {
+		return fmt.Errorf("db required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}

 	var groups []model.ProviderGroup
 	if err := db.Find(&groups).Error; err != nil {
@@ -106,7 +188,30 @@ func (s *SyncService) SyncProviders(db *gorm.DB) error {

 // SyncModel writes a single model metadata record.
 func (s *SyncService) SyncModel(m *model.Model) error {
-	ctx := context.Background()
+	if m == nil {
+		return fmt.Errorf("model required")
+	}
+	if strings.TrimSpace(m.Name) == "" {
+		return fmt.Errorf("model name required")
+	}
+	return s.handleSyncError(s.SyncModelNow(context.Background(), m), SyncOutboxEntry{
+		ResourceType: "model",
+		Action:       "upsert",
+		ResourceID:   &m.ID,
+	})
+}
+
+// SyncModelNow writes a single model metadata record without outbox handling.
+func (s *SyncService) SyncModelNow(ctx context.Context, m *model.Model) error {
+	if m == nil {
+		return fmt.Errorf("model required")
+	}
+	if strings.TrimSpace(m.Name) == "" {
+		return fmt.Errorf("model name required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	snap := modelcap.Model{
 		Name:               m.Name,
 		Kind:               string(modelcap.NormalizeKind(m.Kind)),
@@ -136,7 +241,28 @@ func (s *SyncService) SyncModelDelete(m *model.Model) error {
 	if name == "" {
 		return fmt.Errorf("model name required")
 	}
-	ctx := context.Background()
+	return s.handleSyncError(s.SyncModelDeleteNow(context.Background(), m), SyncOutboxEntry{
+		ResourceType: "model",
+		Action:       "delete",
+		ResourceID:   &m.ID,
+		Payload: map[string]any{
+			"name": name,
+		},
+	})
+}
+
+// SyncModelDeleteNow removes model metadata from Redis without outbox handling.
+func (s *SyncService) SyncModelDeleteNow(ctx context.Context, m *model.Model) error {
+	if m == nil {
+		return fmt.Errorf("model required")
+	}
+	name := strings.TrimSpace(m.Name)
+	if name == "" {
+		return fmt.Errorf("model name required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}
 	if err := s.rdb.HDel(ctx, "meta:models", name).Err(); err != nil {
 		return fmt.Errorf("delete meta:models: %w", err)
 	}
@@ -249,7 +375,23 @@ func (s *SyncService) writeProvidersSnapshot(ctx context.Context, pipe redis.Pip

 // SyncAll rebuilds Redis hashes from the database; use for cold starts or forced refreshes.
 func (s *SyncService) SyncAll(db *gorm.DB) error {
-	ctx := context.Background()
+	if db == nil {
+		return fmt.Errorf("db required")
+	}
+	return s.handleSyncError(s.SyncAllNow(context.Background(), db), SyncOutboxEntry{
+		ResourceType: "snapshot",
+		Action:       "sync_all",
+	})
+}
+
+// SyncAllNow rebuilds snapshots without outbox handling.
+func (s *SyncService) SyncAllNow(ctx context.Context, db *gorm.DB) error {
+	if db == nil {
+		return fmt.Errorf("db required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}

 	var groups []model.ProviderGroup
 	if err := db.Find(&groups).Error; err != nil {
@@ -388,7 +530,45 @@ func (s *SyncService) SyncAll(db *gorm.DB) error {
 // SyncBindings rebuilds the binding snapshot for DP routing.
 // This is intentionally a rebuild to avoid stale entries on deletes/updates.
 func (s *SyncService) SyncBindings(db *gorm.DB) error {
-	ctx := context.Background()
+	return s.syncBindings(db, SyncOutboxEntry{
+		ResourceType: "snapshot",
+		Action:       "sync_bindings",
+	})
+}
+
+// SyncBindingsForGroup retries binding snapshot with provider_group context.
+func (s *SyncService) SyncBindingsForGroup(db *gorm.DB, groupID uint) error {
+	return s.syncBindings(db, SyncOutboxEntry{
+		ResourceType: "provider_group",
+		Action:       "sync_bindings",
+		ResourceID:   &groupID,
+	})
+}
+
+// SyncBindingsForAPIKey retries binding snapshot with api_key context.
+func (s *SyncService) SyncBindingsForAPIKey(db *gorm.DB, apiKeyID uint) error {
+	return s.syncBindings(db, SyncOutboxEntry{
+		ResourceType: "api_key",
+		Action:       "sync_bindings",
+		ResourceID:   &apiKeyID,
+	})
+}
+
+func (s *SyncService) syncBindings(db *gorm.DB, entry SyncOutboxEntry) error {
+	if db == nil {
+		return fmt.Errorf("db required")
+	}
+	return s.handleSyncError(s.SyncBindingsNow(context.Background(), db), entry)
+}
+
+// SyncBindingsNow rebuilds binding snapshot without outbox handling.
+func (s *SyncService) SyncBindingsNow(ctx context.Context, db *gorm.DB) error {
+	if db == nil {
+		return fmt.Errorf("db required")
+	}
+	if ctx == nil {
+		ctx = context.Background()
+	}

 	var groups []model.ProviderGroup
 	if err := db.Find(&groups).Error; err != nil {
@@ -570,6 +750,19 @@ func (s *SyncService) hsetJSON(ctx context.Context, key, field string, val inter
 	return nil
 }

+func (s *SyncService) handleSyncError(err error, entry SyncOutboxEntry) error {
+	if err == nil {
+		return nil
+	}
+	if s == nil || s.outbox == nil || !s.outbox.Enabled() {
+		return err
+	}
+	if enqueueErr := s.outbox.Enqueue(entry); enqueueErr != nil {
+		return fmt.Errorf("sync failed: %w (outbox enqueue failed: %v)", err, enqueueErr)
+	}
+	return nil
+}
+
 func normalizeWeight(weight int) int {
 	if weight <= 0 {
 		return 1