diff --git a/cmd/seeder/client.go b/cmd/seeder/client.go new file mode 100644 index 0000000..2e288c6 --- /dev/null +++ b/cmd/seeder/client.go @@ -0,0 +1,498 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +// Client wraps HTTP calls to the Control Plane API +type Client struct { + baseURL string + adminToken string + dryRun bool + verbose bool + httpClient *http.Client +} + +// NewClient creates a new API client +func NewClient(baseURL, adminToken string, dryRun, verbose bool) *Client { + return &Client{ + baseURL: baseURL, + adminToken: adminToken, + dryRun: dryRun, + verbose: verbose, + httpClient: &http.Client{Timeout: 30 * time.Second}, + } +} + +// APIError represents an error response from the API +type APIError struct { + StatusCode int + Body string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API error %d: %s", e.StatusCode, e.Body) +} + +// IsNotFound returns true if the error is a 404 +func (e *APIError) IsNotFound() bool { + return e.StatusCode == http.StatusNotFound +} + +// IsConflict returns true if the error is a 409 +func (e *APIError) IsConflict() bool { + return e.StatusCode == http.StatusConflict +} + +// doRequest executes an HTTP request +func (c *Client) doRequest(method, path string, body any) ([]byte, error) { + var reqBody io.Reader + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + reqBody = bytes.NewReader(data) + if c.verbose { + fmt.Printf(" Request: %s %s\n", method, path) + fmt.Printf(" Body: %s\n", string(data)) + } + } + + req, err := http.NewRequest(method, c.baseURL+path, reqBody) + if err != nil { + return nil, fmt.Errorf("create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+c.adminToken) + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("execute request: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("read response: %w", err) + } + + if c.verbose { + fmt.Printf(" Response: %d %s\n", resp.StatusCode, string(respBody)) + } + + if resp.StatusCode >= 400 { + return nil, &APIError{StatusCode: resp.StatusCode, Body: string(respBody)} + } + + return respBody, nil +} + +// get performs a GET request +func (c *Client) get(path string) ([]byte, error) { + return c.doRequest(http.MethodGet, path, nil) +} + +// post performs a POST request (respects dry-run) +func (c *Client) post(path string, body any) ([]byte, error) { + if c.dryRun { + data, _ := json.Marshal(body) + fmt.Printf(" [DRY-RUN] POST %s: %s\n", path, string(data)) + return []byte("{}"), nil + } + return c.doRequest(http.MethodPost, path, body) +} + +// put performs a PUT request (respects dry-run) +func (c *Client) put(path string, body any) ([]byte, error) { + if c.dryRun { + data, _ := json.Marshal(body) + fmt.Printf(" [DRY-RUN] PUT %s: %s\n", path, string(data)) + return []byte("{}"), nil + } + return c.doRequest(http.MethodPut, path, body) +} + +// delete performs a DELETE request (respects dry-run) +func (c *Client) delete(path string) error { + if c.dryRun { + fmt.Printf(" [DRY-RUN] DELETE %s\n", path) + return nil + } + _, err := c.doRequest(http.MethodDelete, path, nil) + return err +} + +// --- Namespace API --- + +type NamespaceRequest struct { + Name string `json:"name"` + Status string `json:"status,omitempty"` + Description string `json:"description,omitempty"` +} + +type NamespaceResponse struct { + ID uint `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + Description string `json:"description"` +} + +func (c *Client) ListNamespaces() ([]NamespaceResponse, error) { + data, err := c.get("/admin/namespaces") + if err != nil { + return nil, err + } + var result []NamespaceResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal namespaces: %w", err) + } + return result, nil +} + +func (c *Client) CreateNamespace(req NamespaceRequest) (*NamespaceResponse, error) { + data, err := c.post("/admin/namespaces", req) + if err != nil { + return nil, err + } + var result NamespaceResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal namespace: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteNamespace(id uint) error { + return c.delete(fmt.Sprintf("/admin/namespaces/%d", id)) +} + +// --- ProviderGroup API --- + +type ProviderGroupRequest struct { + Name string `json:"name"` + Type string `json:"type"` + BaseURL string `json:"base_url,omitempty"` + GoogleProject string `json:"google_project,omitempty"` + GoogleLocation string `json:"google_location,omitempty"` + StaticHeaders string `json:"static_headers,omitempty"` + HeadersProfile string `json:"headers_profile,omitempty"` + Models []string `json:"models,omitempty"` + Status string `json:"status,omitempty"` +} + +type ProviderGroupResponse struct { + ID uint `json:"id"` + Name string `json:"name"` + Type string `json:"type"` + BaseURL string `json:"base_url"` + GoogleProject string `json:"google_project"` + GoogleLocation string `json:"google_location"` + Models []string `json:"models"` + Status string `json:"status"` +} + +func (c *Client) ListProviderGroups() ([]ProviderGroupResponse, error) { + data, err := c.get("/admin/provider-groups") + if err != nil { + return nil, err + } + var result []ProviderGroupResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal provider groups: %w", err) + } + return result, nil +} + +func (c *Client) CreateProviderGroup(req ProviderGroupRequest) (*ProviderGroupResponse, error) { + data, err := c.post("/admin/provider-groups", req) + if err != nil { + return nil, err + } + var result ProviderGroupResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal provider group: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteProviderGroup(id uint) error { + return c.delete(fmt.Sprintf("/admin/provider-groups/%d", id)) +} + +// --- APIKey (Provider) API --- + +type APIKeyRequest struct { + GroupID uint `json:"group_id"` + APIKey string `json:"api_key"` + AccessToken string `json:"access_token,omitempty"` + RefreshToken string `json:"refresh_token,omitempty"` + ExpiresAt *string `json:"expires_at,omitempty"` + AccountID string `json:"account_id,omitempty"` + ProjectID string `json:"project_id,omitempty"` + Weight int `json:"weight,omitempty"` + Status string `json:"status,omitempty"` + AutoBan *bool `json:"auto_ban,omitempty"` +} + +type APIKeyResponse struct { + ID uint `json:"id"` + GroupID uint `json:"group_id"` + APIKey string `json:"api_key"` + Weight int `json:"weight"` + Status string `json:"status"` + AutoBan bool `json:"auto_ban"` +} + +func (c *Client) ListAPIKeys() ([]APIKeyResponse, error) { + data, err := c.get("/admin/api-keys") + if err != nil { + return nil, err + } + var result []APIKeyResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal api keys: %w", err) + } + return result, nil +} + +func (c *Client) CreateAPIKey(req APIKeyRequest) (*APIKeyResponse, error) { + data, err := c.post("/admin/api-keys", req) + if err != nil { + return nil, err + } + var result APIKeyResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal api key: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteAPIKey(id uint) error { + return c.delete(fmt.Sprintf("/admin/api-keys/%d", id)) +} + +// --- Model API --- + +type ModelRequest struct { + Name string `json:"name"` + Kind string `json:"kind,omitempty"` + ContextWindow int `json:"context_window,omitempty"` + CostPerToken float64 `json:"cost_per_token,omitempty"` + SupportsVision bool `json:"supports_vision,omitempty"` + SupportsFunctions bool `json:"supports_functions,omitempty"` + SupportsToolChoice bool `json:"supports_tool_choice,omitempty"` + SupportsFIM bool `json:"supports_fim,omitempty"` + MaxOutputTokens int `json:"max_output_tokens,omitempty"` +} + +type ModelResponse struct { + ID uint `json:"id"` + Name string `json:"name"` + Kind string `json:"kind"` + ContextWindow int `json:"context_window"` + CostPerToken float64 `json:"cost_per_token"` + SupportsVision bool `json:"supports_vision"` + SupportsFunctions bool `json:"supports_functions"` + SupportsToolChoice bool `json:"supports_tool_choice"` + SupportsFIM bool `json:"supports_fim"` + MaxOutputTokens int `json:"max_output_tokens"` +} + +func (c *Client) ListModels() ([]ModelResponse, error) { + data, err := c.get("/admin/models") + if err != nil { + return nil, err + } + var result []ModelResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal models: %w", err) + } + return result, nil +} + +func (c *Client) CreateModel(req ModelRequest) (*ModelResponse, error) { + data, err := c.post("/admin/models", req) + if err != nil { + return nil, err + } + var result ModelResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal model: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteModel(id uint) error { + return c.delete(fmt.Sprintf("/admin/models/%d", id)) +} + +// --- Binding API --- + +type BindingRequest struct { + Namespace string `json:"namespace"` + PublicModel string `json:"public_model"` + GroupID uint `json:"group_id"` + Weight int `json:"weight,omitempty"` + SelectorType string `json:"selector_type,omitempty"` + SelectorValue string `json:"selector_value,omitempty"` + Status string `json:"status,omitempty"` +} + +type BindingResponse struct { + ID uint `json:"id"` + Namespace string `json:"namespace"` + PublicModel string `json:"public_model"` + GroupID uint `json:"group_id"` + Weight int `json:"weight"` + SelectorType string `json:"selector_type"` + SelectorValue string `json:"selector_value"` + Status string `json:"status"` +} + +func (c *Client) ListBindings() ([]BindingResponse, error) { + data, err := c.get("/admin/bindings") + if err != nil { + return nil, err + } + var result []BindingResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal bindings: %w", err) + } + return result, nil +} + +func (c *Client) CreateBinding(req BindingRequest) (*BindingResponse, error) { + data, err := c.post("/admin/bindings", req) + if err != nil { + return nil, err + } + var result BindingResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal binding: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteBinding(id uint) error { + return c.delete(fmt.Sprintf("/admin/bindings/%d", id)) +} + +// --- Master API --- + +type MasterRequest struct { + Name string `json:"name"` + Group string `json:"group"` + MaxChildKeys int `json:"max_child_keys,omitempty"` + GlobalQPS int `json:"global_qps,omitempty"` +} + +type MasterResponse struct { + ID uint `json:"id"` + Name string `json:"name"` + Group string `json:"group"` + MasterKey string `json:"master_key,omitempty"` // Only returned on create + MaxChildKeys int `json:"max_child_keys"` + GlobalQPS int `json:"global_qps"` + Status string `json:"status"` +} + +func (c *Client) ListMasters() ([]MasterResponse, error) { + data, err := c.get("/admin/masters") + if err != nil { + return nil, err + } + var result []MasterResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal masters: %w", err) + } + return result, nil +} + +func (c *Client) CreateMaster(req MasterRequest) (*MasterResponse, error) { + data, err := c.post("/admin/masters", req) + if err != nil { + return nil, err + } + var result MasterResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal master: %w", err) + } + return &result, nil +} + +func (c *Client) DeleteMaster(id uint) error { + return c.delete(fmt.Sprintf("/admin/masters/%d", id)) +} + +// --- Key API --- + +type KeyRequest struct { + Group string `json:"group,omitempty"` + Scopes string `json:"scopes,omitempty"` + ModelLimits string `json:"model_limits,omitempty"` + ModelLimitsEnabled *bool `json:"model_limits_enabled,omitempty"` + ExpiresAt *string `json:"expires_at,omitempty"` + AllowIPs string `json:"allow_ips,omitempty"` + DenyIPs string `json:"deny_ips,omitempty"` +} + +type KeyResponse struct { + ID uint `json:"id"` + KeySecret string `json:"key_secret,omitempty"` // Only returned on create + Group string `json:"group"` + Scopes string `json:"scopes"` + IssuedBy string `json:"issued_by"` + Status string `json:"status"` +} + +func (c *Client) CreateKey(masterID uint, req KeyRequest) (*KeyResponse, error) { + data, err := c.post(fmt.Sprintf("/admin/masters/%d/keys", masterID), req) + if err != nil { + return nil, err + } + var result KeyResponse + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("unmarshal key: %w", err) + } + return &result, nil +} + +// --- Log API --- + +type LogRequest struct { + Group string `json:"group,omitempty"` + MasterID uint `json:"master_id,omitempty"` + KeyID uint `json:"key_id,omitempty"` + ModelName string `json:"model_name,omitempty"` + ProviderID uint `json:"provider_id,omitempty"` + ProviderType string `json:"provider_type,omitempty"` + ProviderName string `json:"provider_name,omitempty"` + StatusCode int `json:"status_code,omitempty"` + LatencyMs int64 `json:"latency_ms,omitempty"` + TokensIn int64 `json:"tokens_in,omitempty"` + TokensOut int64 `json:"tokens_out,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` + ClientIP string `json:"client_ip,omitempty"` + RequestSize int64 `json:"request_size,omitempty"` + ResponseSize int64 `json:"response_size,omitempty"` +} + +func (c *Client) CreateLog(req LogRequest) error { + _, err := c.post("/logs", req) + return err +} + +func (c *Client) CreateLogsBatch(logs []LogRequest) error { + for _, log := range logs { + if err := c.CreateLog(log); err != nil { + return err + } + } + return nil +} diff --git a/cmd/seeder/generator.go b/cmd/seeder/generator.go new file mode 100644 index 0000000..7c13e94 --- /dev/null +++ b/cmd/seeder/generator.go @@ -0,0 +1,452 @@ +package main + +import ( + "fmt" + "math/rand" + "strings" + "time" +) + +// Generator creates deterministic demo data +type Generator struct { + rng *rand.Rand + seederTag string + profile string +} + +// NewGenerator creates a new data generator +func NewGenerator(rng *rand.Rand, seederTag, profile string) *Generator { + return &Generator{ + rng: rng, + seederTag: seederTag, + profile: profile, + } +} + +// --- Namespace Generation --- + +var namespaceTemplates = []struct { + name string + desc string +}{ + {"default", "Default namespace for general use"}, + {"premium", "Premium tier with higher limits"}, + {"staging", "Staging environment namespace"}, + {"internal", "Internal services namespace"}, + {"partner", "Partner integrations namespace"}, +} + +func (g *Generator) GenerateNamespaces(count int) []NamespaceRequest { + result := make([]NamespaceRequest, 0, count) + for i := 0; i < count && i < len(namespaceTemplates); i++ { + t := namespaceTemplates[i] + result = append(result, NamespaceRequest{ + Name: fmt.Sprintf("demo-%s", t.name), + Status: "active", + Description: fmt.Sprintf("%s [%s]", t.desc, g.seederTag), + }) + } + return result +} + +// --- Provider Group Generation --- + +var providerGroupTemplates = []struct { + name string + ptype string + baseURL string + models []string +}{ + { + name: "openai-demo", + ptype: "openai", + baseURL: "https://api.openai.com/v1", + models: []string{"gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-3.5-turbo"}, + }, + { + name: "anthropic-demo", + ptype: "anthropic", + baseURL: "https://api.anthropic.com", + models: []string{"claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022", "claude-3-opus-20240229"}, + }, + { + name: "gemini-demo", + ptype: "gemini", + baseURL: "https://generativelanguage.googleapis.com/v1beta", + models: []string{"gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"}, + }, + { + name: "deepseek-demo", + ptype: "openai", + baseURL: "https://api.deepseek.com/v1", + models: []string{"deepseek-chat", "deepseek-coder"}, + }, + { + name: "groq-demo", + ptype: "openai", + baseURL: "https://api.groq.com/openai/v1", + models: []string{"llama-3.1-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"}, + }, + { + name: "mistral-demo", + ptype: "openai", + baseURL: "https://api.mistral.ai/v1", + models: []string{"mistral-large-latest", "mistral-medium-latest", "mistral-small-latest"}, + }, + { + name: "cohere-demo", + ptype: "openai", + baseURL: "https://api.cohere.ai/v1", + models: []string{"command-r-plus", "command-r", "command"}, + }, + { + name: "together-demo", + ptype: "openai", + baseURL: "https://api.together.xyz/v1", + models: []string{"meta-llama/Llama-3-70b-chat-hf", "mistralai/Mixtral-8x22B-Instruct-v0.1"}, + }, + { + name: "openrouter-demo", + ptype: "openai", + baseURL: "https://openrouter.ai/api/v1", + models: []string{"openai/gpt-4o", "anthropic/claude-3.5-sonnet"}, + }, + { + name: "azure-demo", + ptype: "azure", + baseURL: "https://demo.openai.azure.com/openai/deployments", + models: []string{"gpt-4o-deploy", "gpt-35-turbo-deploy"}, + }, +} + +func (g *Generator) GenerateProviderGroups(count int) []ProviderGroupRequest { + result := make([]ProviderGroupRequest, 0, count) + for i := 0; i < count && i < len(providerGroupTemplates); i++ { + t := providerGroupTemplates[i] + result = append(result, ProviderGroupRequest{ + Name: t.name, + Type: t.ptype, + BaseURL: t.baseURL, + Models: t.models, + Status: "active", + }) + } + return result +} + +// --- Provider (APIKey) Generation --- + +func (g *Generator) GenerateProviders(groupID uint, groupName string, count int) []APIKeyRequest { + result := make([]APIKeyRequest, 0, count) + for i := 0; i < count; i++ { + // Generate a fake API key based on group name and index + fakeKey := fmt.Sprintf("sk-%s-%s-%04d", groupName, g.randomString(24), i+1) + result = append(result, APIKeyRequest{ + GroupID: groupID, + APIKey: fakeKey, + Weight: 100, + Status: "active", + }) + } + return result +} + +// --- Model Generation --- + +var modelTemplates = []struct { + name string + kind string + contextWindow int + maxOutput int + vision bool + functions bool + toolChoice bool + costPerToken float64 +}{ + {"gpt-4o", "chat", 128000, 16384, true, true, true, 0.000005}, + {"gpt-4o-mini", "chat", 128000, 16384, true, true, true, 0.00000015}, + {"gpt-4-turbo", "chat", 128000, 4096, true, true, true, 0.00001}, + {"gpt-3.5-turbo", "chat", 16385, 4096, false, true, true, 0.0000005}, + {"claude-3-5-sonnet-20241022", "chat", 200000, 8192, true, true, true, 0.000003}, + {"claude-3-5-haiku-20241022", "chat", 200000, 8192, true, true, true, 0.00000025}, + {"claude-3-opus-20240229", "chat", 200000, 4096, true, true, true, 0.000015}, + {"gemini-1.5-pro", "chat", 2097152, 8192, true, true, true, 0.00000125}, + {"gemini-1.5-flash", "chat", 1048576, 8192, true, true, true, 0.000000075}, + {"gemini-2.0-flash-exp", "chat", 1048576, 8192, true, true, true, 0.0000001}, + {"deepseek-chat", "chat", 64000, 4096, false, true, true, 0.00000014}, + {"deepseek-coder", "chat", 64000, 4096, false, true, true, 0.00000014}, + {"llama-3.1-70b-versatile", "chat", 131072, 8192, false, true, true, 0.00000059}, + {"llama-3.1-8b-instant", "chat", 131072, 8192, false, true, true, 0.00000005}, + {"mixtral-8x7b-32768", "chat", 32768, 4096, false, true, false, 0.00000027}, + {"mistral-large-latest", "chat", 128000, 8192, false, true, true, 0.000002}, + {"command-r-plus", "chat", 128000, 4096, false, true, true, 0.000003}, + {"command-r", "chat", 128000, 4096, false, true, true, 0.0000005}, + {"text-embedding-3-small", "embedding", 8191, 0, false, false, false, 0.00000002}, + {"text-embedding-3-large", "embedding", 8191, 0, false, false, false, 0.00000013}, + {"voyage-large-2", "embedding", 16000, 0, false, false, false, 0.00000012}, + {"rerank-english-v3.0", "rerank", 4096, 0, false, false, false, 0.000001}, + {"rerank-multilingual-v3.0", "rerank", 4096, 0, false, false, false, 0.000001}, + {"o1-preview", "chat", 128000, 32768, true, false, false, 0.000015}, + {"o1-mini", "chat", 128000, 65536, true, false, false, 0.000003}, + {"dall-e-3", "other", 0, 0, false, false, false, 0.04}, + {"whisper-1", "other", 0, 0, false, false, false, 0.006}, + {"tts-1", "other", 0, 0, false, false, false, 0.000015}, + {"tts-1-hd", "other", 0, 0, false, false, false, 0.00003}, + {"codestral-latest", "chat", 32000, 8192, false, true, true, 0.000001}, +} + +func (g *Generator) GenerateModels(count int) []ModelRequest { + result := make([]ModelRequest, 0, count) + for i := 0; i < count && i < len(modelTemplates); i++ { + t := modelTemplates[i] + result = append(result, ModelRequest{ + Name: t.name, + Kind: t.kind, + ContextWindow: t.contextWindow, + MaxOutputTokens: t.maxOutput, + SupportsVision: t.vision, + SupportsFunctions: t.functions, + SupportsToolChoice: t.toolChoice, + CostPerToken: t.costPerToken, + }) + } + return result +} + +// --- Binding Generation --- + +func (g *Generator) GenerateBindings(namespaces []string, groups []ProviderGroupResponse, count int) []BindingRequest { + result := make([]BindingRequest, 0, count) + + if len(namespaces) == 0 || len(groups) == 0 { + return result + } + + // Create bindings for common models + commonModels := []string{"gpt-4o", "gpt-4o-mini", "claude-3-5-sonnet-20241022", "gemini-1.5-pro"} + + for i := 0; i < count; i++ { + ns := namespaces[i%len(namespaces)] + group := groups[i%len(groups)] + + var modelName string + if i < len(commonModels) { + modelName = commonModels[i] + } else { + // Pick a model from the group's models + if len(group.Models) > 0 { + modelName = group.Models[g.rng.Intn(len(group.Models))] + } else { + modelName = fmt.Sprintf("model-%d", i) + } + } + + result = append(result, BindingRequest{ + Namespace: ns, + PublicModel: modelName, + GroupID: group.ID, + Weight: 100, + SelectorType: "exact", + SelectorValue: modelName, + Status: "active", + }) + } + return result +} + +// --- Master Generation --- + +var masterTemplates = []struct { + name string + group string + maxKeys int + qps int +}{ + {"admin-demo", "default", 10, 100}, + {"user-demo", "default", 5, 50}, + {"developer-demo", "default", 20, 200}, + {"partner-demo", "partner", 15, 150}, + {"staging-demo", "staging", 10, 100}, + {"internal-demo", "internal", 50, 500}, + {"test-demo", "default", 5, 25}, + {"premium-demo", "premium", 100, 1000}, + {"enterprise-demo", "enterprise", 200, 2000}, + {"trial-demo", "trial", 3, 10}, +} + +func (g *Generator) GenerateMasters(count int) []MasterRequest { + result := make([]MasterRequest, 0, count) + for i := 0; i < count && i < len(masterTemplates); i++ { + t := masterTemplates[i] + result = append(result, MasterRequest{ + Name: t.name, + Group: t.group, + MaxChildKeys: t.maxKeys, + GlobalQPS: t.qps, + }) + } + return result +} + +// --- Key Generation --- + +func (g *Generator) GenerateKeys(count int) []KeyRequest { + result := make([]KeyRequest, 0, count) + for i := 0; i < count; i++ { + result = append(result, KeyRequest{ + Scopes: "chat,embedding", + }) + } + return result +} + +// --- Usage Sample Generation --- + +func (g *Generator) GenerateUsageSamples( + masters []MasterResponse, + keys map[uint][]KeyResponse, + groups []ProviderGroupResponse, + usageDays int, +) []LogRequest { + result := make([]LogRequest, 0) + + if len(masters) == 0 || len(groups) == 0 { + return result + } + + now := time.Now() + startTime := now.AddDate(0, 0, -usageDays) + + // Generate samples for each day + for day := 0; day < usageDays; day++ { + dayTime := startTime.AddDate(0, 0, day) + + // More traffic on weekdays + isWeekday := dayTime.Weekday() >= time.Monday && dayTime.Weekday() <= time.Friday + baseCount := 50 + if isWeekday { + baseCount = 100 + } + + // Gradual growth trend + growthFactor := 1.0 + float64(day)/float64(usageDays)*0.5 + samplesPerDay := int(float64(baseCount) * growthFactor) + + for i := 0; i < samplesPerDay; i++ { + // Pick random master + master := masters[g.rng.Intn(len(masters))] + + // Pick random key for this master + masterKeys := keys[master.ID] + var keyID uint + if len(masterKeys) > 0 { + keyID = masterKeys[g.rng.Intn(len(masterKeys))].ID + } + + // Pick random group + group := groups[g.rng.Intn(len(groups))] + + // Pick random model from group + var modelName string + if len(group.Models) > 0 { + modelName = group.Models[g.rng.Intn(len(group.Models))] + } else { + modelName = "gpt-4o" + } + + // Random timestamp within the day + hour := g.rng.Intn(24) + minute := g.rng.Intn(60) + second := g.rng.Intn(60) + timestamp := time.Date( + dayTime.Year(), dayTime.Month(), dayTime.Day(), + hour, minute, second, 0, time.UTC, + ) + + // Generate realistic metrics + statusCode := 200 + errorMsg := "" + + // 2-5% error rate + if g.rng.Float64() < 0.03 { + errorCodes := []int{400, 401, 429, 500, 503} + statusCode = errorCodes[g.rng.Intn(len(errorCodes))] + errorMsgs := []string{ + "rate limit exceeded", + "invalid request", + "authentication failed", + "internal server error", + "service unavailable", + } + errorMsg = errorMsgs[g.rng.Intn(len(errorMsgs))] + } + + // Realistic latency (50ms - 5000ms, with most around 200-500ms) + latencyMs := int64(50 + g.rng.ExpFloat64()*300) + if latencyMs > 5000 { + latencyMs = 5000 + } + + // Realistic token counts + tokensIn := int64(100 + g.rng.Intn(2000)) + tokensOut := int64(50 + g.rng.Intn(1000)) + + // Request/response sizes + requestSize := tokensIn * 4 // ~4 bytes per token + responseSize := tokensOut * 4 + + // Random client IP + clientIP := fmt.Sprintf("192.168.%d.%d", g.rng.Intn(256), g.rng.Intn(256)) + + _ = timestamp // TODO: need to set created_at somehow + + result = append(result, LogRequest{ + Group: master.Group, + MasterID: master.ID, + KeyID: keyID, + ModelName: modelName, + ProviderID: group.ID, + ProviderType: group.Type, + ProviderName: group.Name, + StatusCode: statusCode, + LatencyMs: latencyMs, + TokensIn: tokensIn, + TokensOut: tokensOut, + ErrorMessage: errorMsg, + ClientIP: clientIP, + RequestSize: requestSize, + ResponseSize: responseSize, + }) + } + } + + return result +} + +// --- Helper functions --- + +func (g *Generator) randomString(length int) string { + const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, length) + for i := range b { + b[i] = charset[g.rng.Intn(len(charset))] + } + return string(b) +} + +// HasSeederTag checks if a string contains the seeder tag +func HasSeederTag(s, tag string) bool { + return strings.Contains(s, tag) +} + +// ExtractSeederTag extracts the seeder tag from a string +func ExtractSeederTag(s string) string { + if idx := strings.Index(s, "seeder:"); idx != -1 { + end := strings.Index(s[idx:], "]") + if end == -1 { + end = len(s) - idx + } + return s[idx : idx+end] + } + return "" +} diff --git a/cmd/seeder/main.go b/cmd/seeder/main.go new file mode 100644 index 0000000..7d99075 --- /dev/null +++ b/cmd/seeder/main.go @@ -0,0 +1,287 @@ +package main + +import ( + "fmt" + "math/rand" + "os" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +// Config holds seeder configuration +type Config struct { + BaseURL string + AdminToken string + Profile string + Seed int64 + UsageDays int + Env string + Reset bool + DryRun bool + Force bool + Verbose bool +} + +// ProfileConfig defines entity counts for a profile +type ProfileConfig struct { + Namespaces int + ProviderGroups int + ProvidersPerGroup int + Models int + Bindings int + Masters int + KeysPerMaster int + UsageDays int +} + +var profiles = map[string]ProfileConfig{ + "demo": { + Namespaces: 2, + ProviderGroups: 2, + ProvidersPerGroup: 2, + Models: 5, + Bindings: 4, + Masters: 2, + KeysPerMaster: 3, + UsageDays: 7, + }, + "load": { + Namespaces: 5, + ProviderGroups: 10, + ProvidersPerGroup: 3, + Models: 30, + Bindings: 50, + Masters: 50, + KeysPerMaster: 2, + UsageDays: 30, + }, + "dev": { + Namespaces: 3, + ProviderGroups: 5, + ProvidersPerGroup: 3, + Models: 20, + Bindings: 15, + Masters: 10, + KeysPerMaster: 2, + UsageDays: 14, + }, +} + +func main() { + var cfg Config + + rootCmd := &cobra.Command{ + Use: "seeder", + Short: "EZ-API Demo Data Seeder", + Long: `A CLI tool to seed demo data into EZ-API Control Plane. + +Creates namespaces, provider groups, providers, models, bindings, +masters, API keys, and usage samples for frontend development and demos.`, + RunE: func(cmd *cobra.Command, args []string) error { + return run(cfg) + }, + } + + // Bind flags + rootCmd.Flags().StringVar(&cfg.BaseURL, "base-url", "http://localhost:8080", "Control Plane API base URL") + rootCmd.Flags().StringVar(&cfg.AdminToken, "admin-token", "", "Admin authentication token (required)") + rootCmd.Flags().StringVar(&cfg.Profile, "profile", "demo", "Dataset profile: demo, load, dev") + rootCmd.Flags().Int64Var(&cfg.Seed, "seed", 0, "Fixed random seed for reproducibility (0 = random)") + rootCmd.Flags().IntVar(&cfg.UsageDays, "usage-days", 0, "Time window for usage samples (0 = profile default)") + rootCmd.Flags().StringVar(&cfg.Env, "env", "dev", "Environment: dev, test, staging, production") + rootCmd.Flags().BoolVar(&cfg.Reset, "reset", false, "Remove previous seeder data before recreating") + rootCmd.Flags().BoolVar(&cfg.DryRun, "dry-run", false, "Print actions without executing write API calls") + rootCmd.Flags().BoolVar(&cfg.Force, "force", false, "Allow running against production") + rootCmd.Flags().BoolVar(&cfg.Verbose, "verbose", false, "Print detailed progress") + + // Bind environment variables + viper.SetEnvPrefix("SEEDER") + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_")) + + // Override from env if not set via flag + rootCmd.PreRunE = func(cmd *cobra.Command, args []string) error { + if !cmd.Flags().Changed("base-url") && viper.IsSet("BASE_URL") { + cfg.BaseURL = viper.GetString("BASE_URL") + } + if !cmd.Flags().Changed("admin-token") && viper.IsSet("ADMIN_TOKEN") { + cfg.AdminToken = viper.GetString("ADMIN_TOKEN") + } + if !cmd.Flags().Changed("profile") && viper.IsSet("PROFILE") { + cfg.Profile = viper.GetString("PROFILE") + } + if !cmd.Flags().Changed("seed") && viper.IsSet("SEED") { + cfg.Seed = viper.GetInt64("SEED") + } + if !cmd.Flags().Changed("usage-days") && viper.IsSet("USAGE_DAYS") { + cfg.UsageDays = viper.GetInt("USAGE_DAYS") + } + if !cmd.Flags().Changed("env") && viper.IsSet("ENV") { + cfg.Env = viper.GetString("ENV") + } + return nil + } + + if err := rootCmd.Execute(); err != nil { + os.Exit(1) + } +} + +func run(cfg Config) error { + // Validate required fields + if cfg.AdminToken == "" { + return fmt.Errorf("admin-token is required (use --admin-token or SEEDER_ADMIN_TOKEN)") + } + + // Validate profile + profile, ok := profiles[cfg.Profile] + if !ok { + return fmt.Errorf("invalid profile: %s (valid: demo, load, dev)", cfg.Profile) + } + + // Environment guard + if isProduction(cfg.Env) && !cfg.Force { + return fmt.Errorf("refusing to run against production environment (env=%s). Use --force to override", cfg.Env) + } + + // Initialize seed + if cfg.Seed == 0 { + cfg.Seed = time.Now().UnixNano() + } + + // Use profile default for usage days if not specified + if cfg.UsageDays == 0 { + cfg.UsageDays = profile.UsageDays + } + + // Create seeder tag + seederTag := fmt.Sprintf("seeder:%s:seed-%d", cfg.Profile, cfg.Seed) + + // Print banner + printBanner(cfg, seederTag) + + // Warning for production + if isProduction(cfg.Env) && cfg.Force { + fmt.Println("\n⚠️ WARNING: Running against PRODUCTION environment!") + fmt.Println(" Press Ctrl+C within 5 seconds to abort...") + time.Sleep(5 * time.Second) + fmt.Println(" Proceeding...") + } + + // Create seeder instance + seeder := NewSeeder(cfg, profile, seederTag) + + // Run seeding + return seeder.Run() +} + +func isProduction(env string) bool { + env = strings.ToLower(env) + return env == "production" || env == "prod" +} + +func printBanner(cfg Config, seederTag string) { + mode := "create" + if cfg.Reset { + mode = "reset + create" + } + if cfg.DryRun { + mode += " (dry-run)" + } + + fmt.Println() + fmt.Println("EZ-API Demo Data Seeder") + fmt.Println("=======================") + fmt.Printf("Profile: %s\n", cfg.Profile) + fmt.Printf("Seed: %d\n", cfg.Seed) + fmt.Printf("Target: %s\n", cfg.BaseURL) + fmt.Printf("Environment: %s\n", cfg.Env) + fmt.Printf("Usage Window: %d days\n", cfg.UsageDays) + fmt.Printf("Mode: %s\n", mode) + fmt.Printf("Seeder Tag: %s\n", seederTag) + fmt.Println() +} + +// Seeder handles the seeding process +type Seeder struct { + cfg Config + profile ProfileConfig + seederTag string + client *Client + rng *rand.Rand + summary Summary +} + +// Summary tracks created/skipped counts +type Summary struct { + Namespaces CountPair + ProviderGroups CountPair + Providers CountPair + Models CountPair + Bindings CountPair + Masters CountPair + Keys CountPair + UsageSamples int +} + +type CountPair struct { + Created int + Skipped int +} + +// NewSeeder creates a new seeder instance +func NewSeeder(cfg Config, profile ProfileConfig, seederTag string) *Seeder { + return &Seeder{ + cfg: cfg, + profile: profile, + seederTag: seederTag, + client: NewClient(cfg.BaseURL, cfg.AdminToken, cfg.DryRun, cfg.Verbose), + rng: rand.New(rand.NewSource(cfg.Seed)), + } +} + +// Run executes the seeding process +func (s *Seeder) Run() error { + steps := []struct { + name string + fn func() error + }{ + {"Creating namespaces", s.seedNamespaces}, + {"Creating provider groups", s.seedProviderGroups}, + {"Creating providers", s.seedProviders}, + {"Creating models", s.seedModels}, + {"Creating bindings", s.seedBindings}, + {"Creating masters", s.seedMasters}, + {"Creating API keys", s.seedKeys}, + {"Generating usage samples", s.seedUsageSamples}, + } + + for i, step := range steps { + fmt.Printf("[%d/%d] %s...\n", i+1, len(steps), step.name) + if err := step.fn(); err != nil { + return fmt.Errorf("%s: %w", step.name, err) + } + } + + s.printSummary() + return nil +} + +func (s *Seeder) printSummary() { + fmt.Println("\nDone! Summary:") + fmt.Printf(" Namespaces: %d created, %d skipped\n", s.summary.Namespaces.Created, s.summary.Namespaces.Skipped) + fmt.Printf(" Provider Groups: %d created, %d skipped\n", s.summary.ProviderGroups.Created, s.summary.ProviderGroups.Skipped) + fmt.Printf(" Providers: %d created, %d skipped\n", s.summary.Providers.Created, s.summary.Providers.Skipped) + fmt.Printf(" Models: %d created, %d skipped\n", s.summary.Models.Created, s.summary.Models.Skipped) + fmt.Printf(" Bindings: %d created, %d skipped\n", s.summary.Bindings.Created, s.summary.Bindings.Skipped) + fmt.Printf(" Masters: %d created, %d skipped\n", s.summary.Masters.Created, s.summary.Masters.Skipped) + fmt.Printf(" API Keys: %d created, %d skipped\n", s.summary.Keys.Created, s.summary.Keys.Skipped) + fmt.Printf(" Usage Samples: %d entries\n", s.summary.UsageSamples) + fmt.Printf("\nSeeder tag: %s\n", s.seederTag) + fmt.Printf("\nDashboard should now show data at:\n %s/admin/dashboard\n", s.cfg.BaseURL) +} + +// Seed methods are implemented in seeder.go diff --git a/cmd/seeder/seeder.go b/cmd/seeder/seeder.go new file mode 100644 index 0000000..3338f93 --- /dev/null +++ b/cmd/seeder/seeder.go @@ -0,0 +1,433 @@ +package main + +import ( + "fmt" + "strings" +) + +// seededNamespaces stores created namespace names for later use +var seededNamespaces []string + +// seededProviderGroups stores created provider groups for later use +var seededProviderGroups []ProviderGroupResponse + +// seededMasters stores created masters for later use +var seededMasters []MasterResponse + +// seededKeys stores created keys per master for usage samples +var seededKeys map[uint][]KeyResponse + +func (s *Seeder) seedNamespaces() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + namespaces := gen.GenerateNamespaces(s.profile.Namespaces) + + // Get existing namespaces + existing, err := s.client.ListNamespaces() + if err != nil { + if s.cfg.DryRun { + // In dry-run mode, continue without checking existing + existing = []NamespaceResponse{} + fmt.Printf(" [DRY-RUN] Unable to list existing namespaces, proceeding anyway\n") + } else { + return fmt.Errorf("list namespaces: %w", err) + } + } + + existingMap := make(map[string]NamespaceResponse) + for _, ns := range existing { + existingMap[ns.Name] = ns + } + + seededNamespaces = make([]string, 0, len(namespaces)) + + for _, ns := range namespaces { + if existingNs, exists := existingMap[ns.Name]; exists { + // Check if we should reset + if s.cfg.Reset && HasSeederTag(existingNs.Description, s.seederTag) { + if err := s.client.DeleteNamespace(existingNs.ID); err != nil { + return fmt.Errorf("delete namespace %s: %w", ns.Name, err) + } + if s.cfg.Verbose { + fmt.Printf(" ✗ %s (deleted for reset)\n", ns.Name) + } + } else { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name) + } + s.summary.Namespaces.Skipped++ + seededNamespaces = append(seededNamespaces, ns.Name) + continue + } + } + + // Create namespace + created, err := s.client.CreateNamespace(ns) + if err != nil { + // Check if it's a conflict (already exists) + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name) + } + s.summary.Namespaces.Skipped++ + seededNamespaces = append(seededNamespaces, ns.Name) + continue + } + return fmt.Errorf("create namespace %s: %w", ns.Name, err) + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s (created)\n", ns.Name) + } + s.summary.Namespaces.Created++ + if created != nil { + seededNamespaces = append(seededNamespaces, created.Name) + } else { + seededNamespaces = append(seededNamespaces, ns.Name) + } + } + + return nil +} + +func (s *Seeder) seedProviderGroups() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + groups := gen.GenerateProviderGroups(s.profile.ProviderGroups) + + // Get existing provider groups + existing, err := s.client.ListProviderGroups() + if err != nil { + if s.cfg.DryRun { + existing = []ProviderGroupResponse{} + fmt.Printf(" [DRY-RUN] Unable to list existing provider groups, proceeding anyway\n") + } else { + return fmt.Errorf("list provider groups: %w", err) + } + } + + existingMap := make(map[string]ProviderGroupResponse) + for _, g := range existing { + existingMap[g.Name] = g + } + + seededProviderGroups = make([]ProviderGroupResponse, 0, len(groups)) + + for _, group := range groups { + if existingGroup, exists := existingMap[group.Name]; exists { + // Check if we should reset (check name prefix for seeder tag) + if s.cfg.Reset && strings.HasPrefix(group.Name, "demo-") { + if err := s.client.DeleteProviderGroup(existingGroup.ID); err != nil { + return fmt.Errorf("delete provider group %s: %w", group.Name, err) + } + if s.cfg.Verbose { + fmt.Printf(" ✗ %s (deleted for reset)\n", group.Name) + } + } else { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", group.Name) + } + s.summary.ProviderGroups.Skipped++ + seededProviderGroups = append(seededProviderGroups, existingGroup) + continue + } + } + + // Create provider group + created, err := s.client.CreateProviderGroup(group) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", group.Name) + } + s.summary.ProviderGroups.Skipped++ + continue + } + return fmt.Errorf("create provider group %s: %w", group.Name, err) + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s (created)\n", group.Name) + } + s.summary.ProviderGroups.Created++ + if created != nil { + seededProviderGroups = append(seededProviderGroups, *created) + } + } + + return nil +} + +func (s *Seeder) seedProviders() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + + for _, group := range seededProviderGroups { + providers := gen.GenerateProviders(group.ID, group.Name, s.profile.ProvidersPerGroup) + + createdCount := 0 + for _, provider := range providers { + _, err := s.client.CreateAPIKey(provider) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + s.summary.Providers.Skipped++ + continue + } + return fmt.Errorf("create provider for group %s: %w", group.Name, err) + } + s.summary.Providers.Created++ + createdCount++ + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s: %d providers created\n", group.Name, createdCount) + } + } + + return nil +} + +func (s *Seeder) seedModels() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + models := gen.GenerateModels(s.profile.Models) + + // Get existing models + existing, err := s.client.ListModels() + if err != nil { + if s.cfg.DryRun { + existing = []ModelResponse{} + fmt.Printf(" [DRY-RUN] Unable to list existing models, proceeding anyway\n") + } else { + return fmt.Errorf("list models: %w", err) + } + } + + existingMap := make(map[string]ModelResponse) + for _, m := range existing { + existingMap[m.Name] = m + } + + for _, model := range models { + if existingModel, exists := existingMap[model.Name]; exists { + if s.cfg.Reset { + if err := s.client.DeleteModel(existingModel.ID); err != nil { + return fmt.Errorf("delete model %s: %w", model.Name, err) + } + } else { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", model.Name) + } + s.summary.Models.Skipped++ + continue + } + } + + _, err := s.client.CreateModel(model) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", model.Name) + } + s.summary.Models.Skipped++ + continue + } + return fmt.Errorf("create model %s: %w", model.Name, err) + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s (created)\n", model.Name) + } + s.summary.Models.Created++ + } + + if !s.cfg.Verbose && !s.cfg.DryRun { + fmt.Printf(" ✓ %d models created\n", s.summary.Models.Created) + } + + return nil +} + +func (s *Seeder) seedBindings() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + bindings := gen.GenerateBindings(seededNamespaces, seededProviderGroups, s.profile.Bindings) + + // Get existing bindings + existing, err := s.client.ListBindings() + if err != nil { + if s.cfg.DryRun { + existing = []BindingResponse{} + fmt.Printf(" [DRY-RUN] Unable to list existing bindings, proceeding anyway\n") + } else { + return fmt.Errorf("list bindings: %w", err) + } + } + + // Create a map of existing bindings by composite key + existingMap := make(map[string]BindingResponse) + for _, b := range existing { + key := fmt.Sprintf("%s:%s:%d", b.Namespace, b.PublicModel, b.GroupID) + existingMap[key] = b + } + + for _, binding := range bindings { + key := fmt.Sprintf("%s:%s:%d", binding.Namespace, binding.PublicModel, binding.GroupID) + if existingBinding, exists := existingMap[key]; exists { + if s.cfg.Reset { + if err := s.client.DeleteBinding(existingBinding.ID); err != nil { + return fmt.Errorf("delete binding: %w", err) + } + } else { + s.summary.Bindings.Skipped++ + continue + } + } + + _, err := s.client.CreateBinding(binding) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + s.summary.Bindings.Skipped++ + continue + } + return fmt.Errorf("create binding: %w", err) + } + s.summary.Bindings.Created++ + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %d bindings created\n", s.summary.Bindings.Created) + } + + return nil +} + +func (s *Seeder) seedMasters() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + masters := gen.GenerateMasters(s.profile.Masters) + + // Get existing masters + existing, err := s.client.ListMasters() + if err != nil { + if s.cfg.DryRun { + existing = []MasterResponse{} + fmt.Printf(" [DRY-RUN] Unable to list existing masters, proceeding anyway\n") + } else { + return fmt.Errorf("list masters: %w", err) + } + } + + existingMap := make(map[string]MasterResponse) + for _, m := range existing { + existingMap[m.Name] = m + } + + seededMasters = make([]MasterResponse, 0, len(masters)) + + for _, master := range masters { + if existingMaster, exists := existingMap[master.Name]; exists { + if s.cfg.Reset && strings.HasSuffix(master.Name, "-demo") { + if err := s.client.DeleteMaster(existingMaster.ID); err != nil { + return fmt.Errorf("delete master %s: %w", master.Name, err) + } + if s.cfg.Verbose { + fmt.Printf(" ✗ %s (deleted for reset)\n", master.Name) + } + } else { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", master.Name) + } + s.summary.Masters.Skipped++ + seededMasters = append(seededMasters, existingMaster) + continue + } + } + + created, err := s.client.CreateMaster(master) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + if s.cfg.Verbose { + fmt.Printf(" ○ %s (exists, skipped)\n", master.Name) + } + s.summary.Masters.Skipped++ + continue + } + return fmt.Errorf("create master %s: %w", master.Name, err) + } + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s (created)\n", master.Name) + } + s.summary.Masters.Created++ + if created != nil { + seededMasters = append(seededMasters, *created) + } + } + + return nil +} + +func (s *Seeder) seedKeys() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + + seededKeys = make(map[uint][]KeyResponse) + + for _, master := range seededMasters { + keys := gen.GenerateKeys(s.profile.KeysPerMaster) + + masterKeys := make([]KeyResponse, 0, len(keys)) + createdCount := 0 + + for _, key := range keys { + created, err := s.client.CreateKey(master.ID, key) + if err != nil { + if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() { + s.summary.Keys.Skipped++ + continue + } + return fmt.Errorf("create key for master %s: %w", master.Name, err) + } + s.summary.Keys.Created++ + createdCount++ + if created != nil { + masterKeys = append(masterKeys, *created) + } + } + + seededKeys[master.ID] = masterKeys + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %s: %d keys created\n", master.Name, createdCount) + } + } + + return nil +} + +func (s *Seeder) seedUsageSamples() error { + gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile) + + logs := gen.GenerateUsageSamples(seededMasters, seededKeys, seededProviderGroups, s.cfg.UsageDays) + + if len(logs) == 0 { + fmt.Printf(" ○ No masters or groups to generate samples for\n") + return nil + } + + // Batch insert logs + batchSize := 100 + for i := 0; i < len(logs); i += batchSize { + end := i + batchSize + if end > len(logs) { + end = len(logs) + } + batch := logs[i:end] + + if err := s.client.CreateLogsBatch(batch); err != nil { + return fmt.Errorf("create logs batch: %w", err) + } + } + + s.summary.UsageSamples = len(logs) + + if s.cfg.Verbose || s.cfg.DryRun { + fmt.Printf(" ✓ %d days of data generated (%d log entries)\n", s.cfg.UsageDays, len(logs)) + } + + return nil +} diff --git a/go.mod b/go.mod index e3a3992..4326212 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/goccy/go-yaml v1.18.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.6.0 // indirect @@ -66,6 +67,7 @@ require ( github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 245814b..9a328d8 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,7 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -79,6 +80,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= @@ -132,6 +135,7 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= @@ -140,6 +144,9 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=