mirror of
https://github.com/EZ-Api/ez-api.git
synced 2026-01-13 17:47:51 +00:00
feat(seeder): add control plane data seeder
Introduce a `cmd/seeder` CLI to generate deterministic demo datasets and seed them into the Control Plane via admin endpoints, supporting reset, dry-run, profiles, and usage sample generation. Add Cobra/Viper dependencies to support the new command.
This commit is contained in:
zenfun
433
cmd/seeder/seeder.go
Normal file
433
cmd/seeder/seeder.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// seededNamespaces stores created namespace names for later use
|
||||
var seededNamespaces []string
|
||||
|
||||
// seededProviderGroups stores created provider groups for later use
|
||||
var seededProviderGroups []ProviderGroupResponse
|
||||
|
||||
// seededMasters stores created masters for later use
|
||||
var seededMasters []MasterResponse
|
||||
|
||||
// seededKeys stores created keys per master for usage samples
|
||||
var seededKeys map[uint][]KeyResponse
|
||||
|
||||
func (s *Seeder) seedNamespaces() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
namespaces := gen.GenerateNamespaces(s.profile.Namespaces)
|
||||
|
||||
// Get existing namespaces
|
||||
existing, err := s.client.ListNamespaces()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
// In dry-run mode, continue without checking existing
|
||||
existing = []NamespaceResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing namespaces, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list namespaces: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]NamespaceResponse)
|
||||
for _, ns := range existing {
|
||||
existingMap[ns.Name] = ns
|
||||
}
|
||||
|
||||
seededNamespaces = make([]string, 0, len(namespaces))
|
||||
|
||||
for _, ns := range namespaces {
|
||||
if existingNs, exists := existingMap[ns.Name]; exists {
|
||||
// Check if we should reset
|
||||
if s.cfg.Reset && HasSeederTag(existingNs.Description, s.seederTag) {
|
||||
if err := s.client.DeleteNamespace(existingNs.ID); err != nil {
|
||||
return fmt.Errorf("delete namespace %s: %w", ns.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", ns.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Skipped++
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Create namespace
|
||||
created, err := s.client.CreateNamespace(ns)
|
||||
if err != nil {
|
||||
// Check if it's a conflict (already exists)
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Skipped++
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create namespace %s: %w", ns.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Created++
|
||||
if created != nil {
|
||||
seededNamespaces = append(seededNamespaces, created.Name)
|
||||
} else {
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedProviderGroups() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
groups := gen.GenerateProviderGroups(s.profile.ProviderGroups)
|
||||
|
||||
// Get existing provider groups
|
||||
existing, err := s.client.ListProviderGroups()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []ProviderGroupResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing provider groups, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list provider groups: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]ProviderGroupResponse)
|
||||
for _, g := range existing {
|
||||
existingMap[g.Name] = g
|
||||
}
|
||||
|
||||
seededProviderGroups = make([]ProviderGroupResponse, 0, len(groups))
|
||||
|
||||
for _, group := range groups {
|
||||
if existingGroup, exists := existingMap[group.Name]; exists {
|
||||
// Check if we should reset (check name prefix for seeder tag)
|
||||
if s.cfg.Reset && strings.HasPrefix(group.Name, "demo-") {
|
||||
if err := s.client.DeleteProviderGroup(existingGroup.ID); err != nil {
|
||||
return fmt.Errorf("delete provider group %s: %w", group.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", group.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Skipped++
|
||||
seededProviderGroups = append(seededProviderGroups, existingGroup)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Create provider group
|
||||
created, err := s.client.CreateProviderGroup(group)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create provider group %s: %w", group.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Created++
|
||||
if created != nil {
|
||||
seededProviderGroups = append(seededProviderGroups, *created)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedProviders() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
for _, group := range seededProviderGroups {
|
||||
providers := gen.GenerateProviders(group.ID, group.Name, s.profile.ProvidersPerGroup)
|
||||
|
||||
createdCount := 0
|
||||
for _, provider := range providers {
|
||||
_, err := s.client.CreateAPIKey(provider)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Providers.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create provider for group %s: %w", group.Name, err)
|
||||
}
|
||||
s.summary.Providers.Created++
|
||||
createdCount++
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s: %d providers created\n", group.Name, createdCount)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedModels() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
models := gen.GenerateModels(s.profile.Models)
|
||||
|
||||
// Get existing models
|
||||
existing, err := s.client.ListModels()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []ModelResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing models, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list models: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]ModelResponse)
|
||||
for _, m := range existing {
|
||||
existingMap[m.Name] = m
|
||||
}
|
||||
|
||||
for _, model := range models {
|
||||
if existingModel, exists := existingMap[model.Name]; exists {
|
||||
if s.cfg.Reset {
|
||||
if err := s.client.DeleteModel(existingModel.ID); err != nil {
|
||||
return fmt.Errorf("delete model %s: %w", model.Name, err)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.client.CreateModel(model)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create model %s: %w", model.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Created++
|
||||
}
|
||||
|
||||
if !s.cfg.Verbose && !s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d models created\n", s.summary.Models.Created)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedBindings() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
bindings := gen.GenerateBindings(seededNamespaces, seededProviderGroups, s.profile.Bindings)
|
||||
|
||||
// Get existing bindings
|
||||
existing, err := s.client.ListBindings()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []BindingResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing bindings, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list bindings: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a map of existing bindings by composite key
|
||||
existingMap := make(map[string]BindingResponse)
|
||||
for _, b := range existing {
|
||||
key := fmt.Sprintf("%s:%s:%d", b.Namespace, b.PublicModel, b.GroupID)
|
||||
existingMap[key] = b
|
||||
}
|
||||
|
||||
for _, binding := range bindings {
|
||||
key := fmt.Sprintf("%s:%s:%d", binding.Namespace, binding.PublicModel, binding.GroupID)
|
||||
if existingBinding, exists := existingMap[key]; exists {
|
||||
if s.cfg.Reset {
|
||||
if err := s.client.DeleteBinding(existingBinding.ID); err != nil {
|
||||
return fmt.Errorf("delete binding: %w", err)
|
||||
}
|
||||
} else {
|
||||
s.summary.Bindings.Skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.client.CreateBinding(binding)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Bindings.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create binding: %w", err)
|
||||
}
|
||||
s.summary.Bindings.Created++
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d bindings created\n", s.summary.Bindings.Created)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedMasters() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
masters := gen.GenerateMasters(s.profile.Masters)
|
||||
|
||||
// Get existing masters
|
||||
existing, err := s.client.ListMasters()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []MasterResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing masters, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list masters: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]MasterResponse)
|
||||
for _, m := range existing {
|
||||
existingMap[m.Name] = m
|
||||
}
|
||||
|
||||
seededMasters = make([]MasterResponse, 0, len(masters))
|
||||
|
||||
for _, master := range masters {
|
||||
if existingMaster, exists := existingMap[master.Name]; exists {
|
||||
if s.cfg.Reset && strings.HasSuffix(master.Name, "-demo") {
|
||||
if err := s.client.DeleteMaster(existingMaster.ID); err != nil {
|
||||
return fmt.Errorf("delete master %s: %w", master.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", master.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Skipped++
|
||||
seededMasters = append(seededMasters, existingMaster)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
created, err := s.client.CreateMaster(master)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create master %s: %w", master.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Created++
|
||||
if created != nil {
|
||||
seededMasters = append(seededMasters, *created)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedKeys() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
seededKeys = make(map[uint][]KeyResponse)
|
||||
|
||||
for _, master := range seededMasters {
|
||||
keys := gen.GenerateKeys(s.profile.KeysPerMaster)
|
||||
|
||||
masterKeys := make([]KeyResponse, 0, len(keys))
|
||||
createdCount := 0
|
||||
|
||||
for _, key := range keys {
|
||||
created, err := s.client.CreateKey(master.ID, key)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Keys.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create key for master %s: %w", master.Name, err)
|
||||
}
|
||||
s.summary.Keys.Created++
|
||||
createdCount++
|
||||
if created != nil {
|
||||
masterKeys = append(masterKeys, *created)
|
||||
}
|
||||
}
|
||||
|
||||
seededKeys[master.ID] = masterKeys
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s: %d keys created\n", master.Name, createdCount)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedUsageSamples() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
logs := gen.GenerateUsageSamples(seededMasters, seededKeys, seededProviderGroups, s.cfg.UsageDays)
|
||||
|
||||
if len(logs) == 0 {
|
||||
fmt.Printf(" ○ No masters or groups to generate samples for\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Batch insert logs
|
||||
batchSize := 100
|
||||
for i := 0; i < len(logs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(logs) {
|
||||
end = len(logs)
|
||||
}
|
||||
batch := logs[i:end]
|
||||
|
||||
if err := s.client.CreateLogsBatch(batch); err != nil {
|
||||
return fmt.Errorf("create logs batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.summary.UsageSamples = len(logs)
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d days of data generated (%d log entries)\n", s.cfg.UsageDays, len(logs))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user