mirror of
https://github.com/EZ-Api/ez-api.git
synced 2026-01-13 17:47:51 +00:00
feat(seeder): add control plane data seeder
Introduce a `cmd/seeder` CLI to generate deterministic demo datasets and seed them into the Control Plane via admin endpoints, supporting reset, dry-run, profiles, and usage sample generation. Add Cobra/Viper dependencies to support the new command.
This commit is contained in:
zenfun
498
cmd/seeder/client.go
Normal file
498
cmd/seeder/client.go
Normal file
@@ -0,0 +1,498 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client wraps HTTP calls to the Control Plane API
|
||||
type Client struct {
|
||||
baseURL string
|
||||
adminToken string
|
||||
dryRun bool
|
||||
verbose bool
|
||||
httpClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new API client
|
||||
func NewClient(baseURL, adminToken string, dryRun, verbose bool) *Client {
|
||||
return &Client{
|
||||
baseURL: baseURL,
|
||||
adminToken: adminToken,
|
||||
dryRun: dryRun,
|
||||
verbose: verbose,
|
||||
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||
}
|
||||
}
|
||||
|
||||
// APIError represents an error response from the API
|
||||
type APIError struct {
|
||||
StatusCode int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *APIError) Error() string {
|
||||
return fmt.Sprintf("API error %d: %s", e.StatusCode, e.Body)
|
||||
}
|
||||
|
||||
// IsNotFound returns true if the error is a 404
|
||||
func (e *APIError) IsNotFound() bool {
|
||||
return e.StatusCode == http.StatusNotFound
|
||||
}
|
||||
|
||||
// IsConflict returns true if the error is a 409
|
||||
func (e *APIError) IsConflict() bool {
|
||||
return e.StatusCode == http.StatusConflict
|
||||
}
|
||||
|
||||
// doRequest executes an HTTP request
|
||||
func (c *Client) doRequest(method, path string, body any) ([]byte, error) {
|
||||
var reqBody io.Reader
|
||||
if body != nil {
|
||||
data, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("marshal request: %w", err)
|
||||
}
|
||||
reqBody = bytes.NewReader(data)
|
||||
if c.verbose {
|
||||
fmt.Printf(" Request: %s %s\n", method, path)
|
||||
fmt.Printf(" Body: %s\n", string(data))
|
||||
}
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(method, c.baseURL+path, reqBody)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create request: %w", err)
|
||||
}
|
||||
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+c.adminToken)
|
||||
|
||||
resp, err := c.httpClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("execute request: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read response: %w", err)
|
||||
}
|
||||
|
||||
if c.verbose {
|
||||
fmt.Printf(" Response: %d %s\n", resp.StatusCode, string(respBody))
|
||||
}
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, &APIError{StatusCode: resp.StatusCode, Body: string(respBody)}
|
||||
}
|
||||
|
||||
return respBody, nil
|
||||
}
|
||||
|
||||
// get performs a GET request
|
||||
func (c *Client) get(path string) ([]byte, error) {
|
||||
return c.doRequest(http.MethodGet, path, nil)
|
||||
}
|
||||
|
||||
// post performs a POST request (respects dry-run)
|
||||
func (c *Client) post(path string, body any) ([]byte, error) {
|
||||
if c.dryRun {
|
||||
data, _ := json.Marshal(body)
|
||||
fmt.Printf(" [DRY-RUN] POST %s: %s\n", path, string(data))
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return c.doRequest(http.MethodPost, path, body)
|
||||
}
|
||||
|
||||
// put performs a PUT request (respects dry-run)
|
||||
func (c *Client) put(path string, body any) ([]byte, error) {
|
||||
if c.dryRun {
|
||||
data, _ := json.Marshal(body)
|
||||
fmt.Printf(" [DRY-RUN] PUT %s: %s\n", path, string(data))
|
||||
return []byte("{}"), nil
|
||||
}
|
||||
return c.doRequest(http.MethodPut, path, body)
|
||||
}
|
||||
|
||||
// delete performs a DELETE request (respects dry-run)
|
||||
func (c *Client) delete(path string) error {
|
||||
if c.dryRun {
|
||||
fmt.Printf(" [DRY-RUN] DELETE %s\n", path)
|
||||
return nil
|
||||
}
|
||||
_, err := c.doRequest(http.MethodDelete, path, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// --- Namespace API ---
|
||||
|
||||
type NamespaceRequest struct {
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status,omitempty"`
|
||||
Description string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
type NamespaceResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Status string `json:"status"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
func (c *Client) ListNamespaces() ([]NamespaceResponse, error) {
|
||||
data, err := c.get("/admin/namespaces")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []NamespaceResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal namespaces: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateNamespace(req NamespaceRequest) (*NamespaceResponse, error) {
|
||||
data, err := c.post("/admin/namespaces", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result NamespaceResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal namespace: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteNamespace(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/namespaces/%d", id))
|
||||
}
|
||||
|
||||
// --- ProviderGroup API ---
|
||||
|
||||
type ProviderGroupRequest struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
BaseURL string `json:"base_url,omitempty"`
|
||||
GoogleProject string `json:"google_project,omitempty"`
|
||||
GoogleLocation string `json:"google_location,omitempty"`
|
||||
StaticHeaders string `json:"static_headers,omitempty"`
|
||||
HeadersProfile string `json:"headers_profile,omitempty"`
|
||||
Models []string `json:"models,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type ProviderGroupResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
BaseURL string `json:"base_url"`
|
||||
GoogleProject string `json:"google_project"`
|
||||
GoogleLocation string `json:"google_location"`
|
||||
Models []string `json:"models"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func (c *Client) ListProviderGroups() ([]ProviderGroupResponse, error) {
|
||||
data, err := c.get("/admin/provider-groups")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []ProviderGroupResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal provider groups: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateProviderGroup(req ProviderGroupRequest) (*ProviderGroupResponse, error) {
|
||||
data, err := c.post("/admin/provider-groups", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result ProviderGroupResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal provider group: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteProviderGroup(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/provider-groups/%d", id))
|
||||
}
|
||||
|
||||
// --- APIKey (Provider) API ---
|
||||
|
||||
type APIKeyRequest struct {
|
||||
GroupID uint `json:"group_id"`
|
||||
APIKey string `json:"api_key"`
|
||||
AccessToken string `json:"access_token,omitempty"`
|
||||
RefreshToken string `json:"refresh_token,omitempty"`
|
||||
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||
AccountID string `json:"account_id,omitempty"`
|
||||
ProjectID string `json:"project_id,omitempty"`
|
||||
Weight int `json:"weight,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
AutoBan *bool `json:"auto_ban,omitempty"`
|
||||
}
|
||||
|
||||
type APIKeyResponse struct {
|
||||
ID uint `json:"id"`
|
||||
GroupID uint `json:"group_id"`
|
||||
APIKey string `json:"api_key"`
|
||||
Weight int `json:"weight"`
|
||||
Status string `json:"status"`
|
||||
AutoBan bool `json:"auto_ban"`
|
||||
}
|
||||
|
||||
func (c *Client) ListAPIKeys() ([]APIKeyResponse, error) {
|
||||
data, err := c.get("/admin/api-keys")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []APIKeyResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal api keys: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateAPIKey(req APIKeyRequest) (*APIKeyResponse, error) {
|
||||
data, err := c.post("/admin/api-keys", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result APIKeyResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal api key: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteAPIKey(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/api-keys/%d", id))
|
||||
}
|
||||
|
||||
// --- Model API ---
|
||||
|
||||
type ModelRequest struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind,omitempty"`
|
||||
ContextWindow int `json:"context_window,omitempty"`
|
||||
CostPerToken float64 `json:"cost_per_token,omitempty"`
|
||||
SupportsVision bool `json:"supports_vision,omitempty"`
|
||||
SupportsFunctions bool `json:"supports_functions,omitempty"`
|
||||
SupportsToolChoice bool `json:"supports_tool_choice,omitempty"`
|
||||
SupportsFIM bool `json:"supports_fim,omitempty"`
|
||||
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
||||
}
|
||||
|
||||
type ModelResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"`
|
||||
ContextWindow int `json:"context_window"`
|
||||
CostPerToken float64 `json:"cost_per_token"`
|
||||
SupportsVision bool `json:"supports_vision"`
|
||||
SupportsFunctions bool `json:"supports_functions"`
|
||||
SupportsToolChoice bool `json:"supports_tool_choice"`
|
||||
SupportsFIM bool `json:"supports_fim"`
|
||||
MaxOutputTokens int `json:"max_output_tokens"`
|
||||
}
|
||||
|
||||
func (c *Client) ListModels() ([]ModelResponse, error) {
|
||||
data, err := c.get("/admin/models")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []ModelResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal models: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateModel(req ModelRequest) (*ModelResponse, error) {
|
||||
data, err := c.post("/admin/models", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result ModelResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal model: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteModel(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/models/%d", id))
|
||||
}
|
||||
|
||||
// --- Binding API ---
|
||||
|
||||
type BindingRequest struct {
|
||||
Namespace string `json:"namespace"`
|
||||
PublicModel string `json:"public_model"`
|
||||
GroupID uint `json:"group_id"`
|
||||
Weight int `json:"weight,omitempty"`
|
||||
SelectorType string `json:"selector_type,omitempty"`
|
||||
SelectorValue string `json:"selector_value,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
type BindingResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Namespace string `json:"namespace"`
|
||||
PublicModel string `json:"public_model"`
|
||||
GroupID uint `json:"group_id"`
|
||||
Weight int `json:"weight"`
|
||||
SelectorType string `json:"selector_type"`
|
||||
SelectorValue string `json:"selector_value"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func (c *Client) ListBindings() ([]BindingResponse, error) {
|
||||
data, err := c.get("/admin/bindings")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []BindingResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal bindings: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateBinding(req BindingRequest) (*BindingResponse, error) {
|
||||
data, err := c.post("/admin/bindings", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result BindingResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal binding: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteBinding(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/bindings/%d", id))
|
||||
}
|
||||
|
||||
// --- Master API ---
|
||||
|
||||
type MasterRequest struct {
|
||||
Name string `json:"name"`
|
||||
Group string `json:"group"`
|
||||
MaxChildKeys int `json:"max_child_keys,omitempty"`
|
||||
GlobalQPS int `json:"global_qps,omitempty"`
|
||||
}
|
||||
|
||||
type MasterResponse struct {
|
||||
ID uint `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Group string `json:"group"`
|
||||
MasterKey string `json:"master_key,omitempty"` // Only returned on create
|
||||
MaxChildKeys int `json:"max_child_keys"`
|
||||
GlobalQPS int `json:"global_qps"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func (c *Client) ListMasters() ([]MasterResponse, error) {
|
||||
data, err := c.get("/admin/masters")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result []MasterResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal masters: %w", err)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateMaster(req MasterRequest) (*MasterResponse, error) {
|
||||
data, err := c.post("/admin/masters", req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result MasterResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal master: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteMaster(id uint) error {
|
||||
return c.delete(fmt.Sprintf("/admin/masters/%d", id))
|
||||
}
|
||||
|
||||
// --- Key API ---
|
||||
|
||||
type KeyRequest struct {
|
||||
Group string `json:"group,omitempty"`
|
||||
Scopes string `json:"scopes,omitempty"`
|
||||
ModelLimits string `json:"model_limits,omitempty"`
|
||||
ModelLimitsEnabled *bool `json:"model_limits_enabled,omitempty"`
|
||||
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||
AllowIPs string `json:"allow_ips,omitempty"`
|
||||
DenyIPs string `json:"deny_ips,omitempty"`
|
||||
}
|
||||
|
||||
type KeyResponse struct {
|
||||
ID uint `json:"id"`
|
||||
KeySecret string `json:"key_secret,omitempty"` // Only returned on create
|
||||
Group string `json:"group"`
|
||||
Scopes string `json:"scopes"`
|
||||
IssuedBy string `json:"issued_by"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
func (c *Client) CreateKey(masterID uint, req KeyRequest) (*KeyResponse, error) {
|
||||
data, err := c.post(fmt.Sprintf("/admin/masters/%d/keys", masterID), req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var result KeyResponse
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal key: %w", err)
|
||||
}
|
||||
return &result, nil
|
||||
}
|
||||
|
||||
// --- Log API ---
|
||||
|
||||
type LogRequest struct {
|
||||
Group string `json:"group,omitempty"`
|
||||
MasterID uint `json:"master_id,omitempty"`
|
||||
KeyID uint `json:"key_id,omitempty"`
|
||||
ModelName string `json:"model_name,omitempty"`
|
||||
ProviderID uint `json:"provider_id,omitempty"`
|
||||
ProviderType string `json:"provider_type,omitempty"`
|
||||
ProviderName string `json:"provider_name,omitempty"`
|
||||
StatusCode int `json:"status_code,omitempty"`
|
||||
LatencyMs int64 `json:"latency_ms,omitempty"`
|
||||
TokensIn int64 `json:"tokens_in,omitempty"`
|
||||
TokensOut int64 `json:"tokens_out,omitempty"`
|
||||
ErrorMessage string `json:"error_message,omitempty"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
RequestSize int64 `json:"request_size,omitempty"`
|
||||
ResponseSize int64 `json:"response_size,omitempty"`
|
||||
}
|
||||
|
||||
func (c *Client) CreateLog(req LogRequest) error {
|
||||
_, err := c.post("/logs", req)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *Client) CreateLogsBatch(logs []LogRequest) error {
|
||||
for _, log := range logs {
|
||||
if err := c.CreateLog(log); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
452
cmd/seeder/generator.go
Normal file
452
cmd/seeder/generator.go
Normal file
@@ -0,0 +1,452 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Generator creates deterministic demo data
|
||||
type Generator struct {
|
||||
rng *rand.Rand
|
||||
seederTag string
|
||||
profile string
|
||||
}
|
||||
|
||||
// NewGenerator creates a new data generator
|
||||
func NewGenerator(rng *rand.Rand, seederTag, profile string) *Generator {
|
||||
return &Generator{
|
||||
rng: rng,
|
||||
seederTag: seederTag,
|
||||
profile: profile,
|
||||
}
|
||||
}
|
||||
|
||||
// --- Namespace Generation ---
|
||||
|
||||
var namespaceTemplates = []struct {
|
||||
name string
|
||||
desc string
|
||||
}{
|
||||
{"default", "Default namespace for general use"},
|
||||
{"premium", "Premium tier with higher limits"},
|
||||
{"staging", "Staging environment namespace"},
|
||||
{"internal", "Internal services namespace"},
|
||||
{"partner", "Partner integrations namespace"},
|
||||
}
|
||||
|
||||
func (g *Generator) GenerateNamespaces(count int) []NamespaceRequest {
|
||||
result := make([]NamespaceRequest, 0, count)
|
||||
for i := 0; i < count && i < len(namespaceTemplates); i++ {
|
||||
t := namespaceTemplates[i]
|
||||
result = append(result, NamespaceRequest{
|
||||
Name: fmt.Sprintf("demo-%s", t.name),
|
||||
Status: "active",
|
||||
Description: fmt.Sprintf("%s [%s]", t.desc, g.seederTag),
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Provider Group Generation ---
|
||||
|
||||
var providerGroupTemplates = []struct {
|
||||
name string
|
||||
ptype string
|
||||
baseURL string
|
||||
models []string
|
||||
}{
|
||||
{
|
||||
name: "openai-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.openai.com/v1",
|
||||
models: []string{"gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-3.5-turbo"},
|
||||
},
|
||||
{
|
||||
name: "anthropic-demo",
|
||||
ptype: "anthropic",
|
||||
baseURL: "https://api.anthropic.com",
|
||||
models: []string{"claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022", "claude-3-opus-20240229"},
|
||||
},
|
||||
{
|
||||
name: "gemini-demo",
|
||||
ptype: "gemini",
|
||||
baseURL: "https://generativelanguage.googleapis.com/v1beta",
|
||||
models: []string{"gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"},
|
||||
},
|
||||
{
|
||||
name: "deepseek-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.deepseek.com/v1",
|
||||
models: []string{"deepseek-chat", "deepseek-coder"},
|
||||
},
|
||||
{
|
||||
name: "groq-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.groq.com/openai/v1",
|
||||
models: []string{"llama-3.1-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"},
|
||||
},
|
||||
{
|
||||
name: "mistral-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.mistral.ai/v1",
|
||||
models: []string{"mistral-large-latest", "mistral-medium-latest", "mistral-small-latest"},
|
||||
},
|
||||
{
|
||||
name: "cohere-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.cohere.ai/v1",
|
||||
models: []string{"command-r-plus", "command-r", "command"},
|
||||
},
|
||||
{
|
||||
name: "together-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://api.together.xyz/v1",
|
||||
models: []string{"meta-llama/Llama-3-70b-chat-hf", "mistralai/Mixtral-8x22B-Instruct-v0.1"},
|
||||
},
|
||||
{
|
||||
name: "openrouter-demo",
|
||||
ptype: "openai",
|
||||
baseURL: "https://openrouter.ai/api/v1",
|
||||
models: []string{"openai/gpt-4o", "anthropic/claude-3.5-sonnet"},
|
||||
},
|
||||
{
|
||||
name: "azure-demo",
|
||||
ptype: "azure",
|
||||
baseURL: "https://demo.openai.azure.com/openai/deployments",
|
||||
models: []string{"gpt-4o-deploy", "gpt-35-turbo-deploy"},
|
||||
},
|
||||
}
|
||||
|
||||
func (g *Generator) GenerateProviderGroups(count int) []ProviderGroupRequest {
|
||||
result := make([]ProviderGroupRequest, 0, count)
|
||||
for i := 0; i < count && i < len(providerGroupTemplates); i++ {
|
||||
t := providerGroupTemplates[i]
|
||||
result = append(result, ProviderGroupRequest{
|
||||
Name: t.name,
|
||||
Type: t.ptype,
|
||||
BaseURL: t.baseURL,
|
||||
Models: t.models,
|
||||
Status: "active",
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Provider (APIKey) Generation ---
|
||||
|
||||
func (g *Generator) GenerateProviders(groupID uint, groupName string, count int) []APIKeyRequest {
|
||||
result := make([]APIKeyRequest, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
// Generate a fake API key based on group name and index
|
||||
fakeKey := fmt.Sprintf("sk-%s-%s-%04d", groupName, g.randomString(24), i+1)
|
||||
result = append(result, APIKeyRequest{
|
||||
GroupID: groupID,
|
||||
APIKey: fakeKey,
|
||||
Weight: 100,
|
||||
Status: "active",
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Model Generation ---
|
||||
|
||||
var modelTemplates = []struct {
|
||||
name string
|
||||
kind string
|
||||
contextWindow int
|
||||
maxOutput int
|
||||
vision bool
|
||||
functions bool
|
||||
toolChoice bool
|
||||
costPerToken float64
|
||||
}{
|
||||
{"gpt-4o", "chat", 128000, 16384, true, true, true, 0.000005},
|
||||
{"gpt-4o-mini", "chat", 128000, 16384, true, true, true, 0.00000015},
|
||||
{"gpt-4-turbo", "chat", 128000, 4096, true, true, true, 0.00001},
|
||||
{"gpt-3.5-turbo", "chat", 16385, 4096, false, true, true, 0.0000005},
|
||||
{"claude-3-5-sonnet-20241022", "chat", 200000, 8192, true, true, true, 0.000003},
|
||||
{"claude-3-5-haiku-20241022", "chat", 200000, 8192, true, true, true, 0.00000025},
|
||||
{"claude-3-opus-20240229", "chat", 200000, 4096, true, true, true, 0.000015},
|
||||
{"gemini-1.5-pro", "chat", 2097152, 8192, true, true, true, 0.00000125},
|
||||
{"gemini-1.5-flash", "chat", 1048576, 8192, true, true, true, 0.000000075},
|
||||
{"gemini-2.0-flash-exp", "chat", 1048576, 8192, true, true, true, 0.0000001},
|
||||
{"deepseek-chat", "chat", 64000, 4096, false, true, true, 0.00000014},
|
||||
{"deepseek-coder", "chat", 64000, 4096, false, true, true, 0.00000014},
|
||||
{"llama-3.1-70b-versatile", "chat", 131072, 8192, false, true, true, 0.00000059},
|
||||
{"llama-3.1-8b-instant", "chat", 131072, 8192, false, true, true, 0.00000005},
|
||||
{"mixtral-8x7b-32768", "chat", 32768, 4096, false, true, false, 0.00000027},
|
||||
{"mistral-large-latest", "chat", 128000, 8192, false, true, true, 0.000002},
|
||||
{"command-r-plus", "chat", 128000, 4096, false, true, true, 0.000003},
|
||||
{"command-r", "chat", 128000, 4096, false, true, true, 0.0000005},
|
||||
{"text-embedding-3-small", "embedding", 8191, 0, false, false, false, 0.00000002},
|
||||
{"text-embedding-3-large", "embedding", 8191, 0, false, false, false, 0.00000013},
|
||||
{"voyage-large-2", "embedding", 16000, 0, false, false, false, 0.00000012},
|
||||
{"rerank-english-v3.0", "rerank", 4096, 0, false, false, false, 0.000001},
|
||||
{"rerank-multilingual-v3.0", "rerank", 4096, 0, false, false, false, 0.000001},
|
||||
{"o1-preview", "chat", 128000, 32768, true, false, false, 0.000015},
|
||||
{"o1-mini", "chat", 128000, 65536, true, false, false, 0.000003},
|
||||
{"dall-e-3", "other", 0, 0, false, false, false, 0.04},
|
||||
{"whisper-1", "other", 0, 0, false, false, false, 0.006},
|
||||
{"tts-1", "other", 0, 0, false, false, false, 0.000015},
|
||||
{"tts-1-hd", "other", 0, 0, false, false, false, 0.00003},
|
||||
{"codestral-latest", "chat", 32000, 8192, false, true, true, 0.000001},
|
||||
}
|
||||
|
||||
func (g *Generator) GenerateModels(count int) []ModelRequest {
|
||||
result := make([]ModelRequest, 0, count)
|
||||
for i := 0; i < count && i < len(modelTemplates); i++ {
|
||||
t := modelTemplates[i]
|
||||
result = append(result, ModelRequest{
|
||||
Name: t.name,
|
||||
Kind: t.kind,
|
||||
ContextWindow: t.contextWindow,
|
||||
MaxOutputTokens: t.maxOutput,
|
||||
SupportsVision: t.vision,
|
||||
SupportsFunctions: t.functions,
|
||||
SupportsToolChoice: t.toolChoice,
|
||||
CostPerToken: t.costPerToken,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Binding Generation ---
|
||||
|
||||
func (g *Generator) GenerateBindings(namespaces []string, groups []ProviderGroupResponse, count int) []BindingRequest {
|
||||
result := make([]BindingRequest, 0, count)
|
||||
|
||||
if len(namespaces) == 0 || len(groups) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
// Create bindings for common models
|
||||
commonModels := []string{"gpt-4o", "gpt-4o-mini", "claude-3-5-sonnet-20241022", "gemini-1.5-pro"}
|
||||
|
||||
for i := 0; i < count; i++ {
|
||||
ns := namespaces[i%len(namespaces)]
|
||||
group := groups[i%len(groups)]
|
||||
|
||||
var modelName string
|
||||
if i < len(commonModels) {
|
||||
modelName = commonModels[i]
|
||||
} else {
|
||||
// Pick a model from the group's models
|
||||
if len(group.Models) > 0 {
|
||||
modelName = group.Models[g.rng.Intn(len(group.Models))]
|
||||
} else {
|
||||
modelName = fmt.Sprintf("model-%d", i)
|
||||
}
|
||||
}
|
||||
|
||||
result = append(result, BindingRequest{
|
||||
Namespace: ns,
|
||||
PublicModel: modelName,
|
||||
GroupID: group.ID,
|
||||
Weight: 100,
|
||||
SelectorType: "exact",
|
||||
SelectorValue: modelName,
|
||||
Status: "active",
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Master Generation ---
|
||||
|
||||
var masterTemplates = []struct {
|
||||
name string
|
||||
group string
|
||||
maxKeys int
|
||||
qps int
|
||||
}{
|
||||
{"admin-demo", "default", 10, 100},
|
||||
{"user-demo", "default", 5, 50},
|
||||
{"developer-demo", "default", 20, 200},
|
||||
{"partner-demo", "partner", 15, 150},
|
||||
{"staging-demo", "staging", 10, 100},
|
||||
{"internal-demo", "internal", 50, 500},
|
||||
{"test-demo", "default", 5, 25},
|
||||
{"premium-demo", "premium", 100, 1000},
|
||||
{"enterprise-demo", "enterprise", 200, 2000},
|
||||
{"trial-demo", "trial", 3, 10},
|
||||
}
|
||||
|
||||
func (g *Generator) GenerateMasters(count int) []MasterRequest {
|
||||
result := make([]MasterRequest, 0, count)
|
||||
for i := 0; i < count && i < len(masterTemplates); i++ {
|
||||
t := masterTemplates[i]
|
||||
result = append(result, MasterRequest{
|
||||
Name: t.name,
|
||||
Group: t.group,
|
||||
MaxChildKeys: t.maxKeys,
|
||||
GlobalQPS: t.qps,
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Key Generation ---
|
||||
|
||||
func (g *Generator) GenerateKeys(count int) []KeyRequest {
|
||||
result := make([]KeyRequest, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
result = append(result, KeyRequest{
|
||||
Scopes: "chat,embedding",
|
||||
})
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Usage Sample Generation ---
|
||||
|
||||
func (g *Generator) GenerateUsageSamples(
|
||||
masters []MasterResponse,
|
||||
keys map[uint][]KeyResponse,
|
||||
groups []ProviderGroupResponse,
|
||||
usageDays int,
|
||||
) []LogRequest {
|
||||
result := make([]LogRequest, 0)
|
||||
|
||||
if len(masters) == 0 || len(groups) == 0 {
|
||||
return result
|
||||
}
|
||||
|
||||
now := time.Now()
|
||||
startTime := now.AddDate(0, 0, -usageDays)
|
||||
|
||||
// Generate samples for each day
|
||||
for day := 0; day < usageDays; day++ {
|
||||
dayTime := startTime.AddDate(0, 0, day)
|
||||
|
||||
// More traffic on weekdays
|
||||
isWeekday := dayTime.Weekday() >= time.Monday && dayTime.Weekday() <= time.Friday
|
||||
baseCount := 50
|
||||
if isWeekday {
|
||||
baseCount = 100
|
||||
}
|
||||
|
||||
// Gradual growth trend
|
||||
growthFactor := 1.0 + float64(day)/float64(usageDays)*0.5
|
||||
samplesPerDay := int(float64(baseCount) * growthFactor)
|
||||
|
||||
for i := 0; i < samplesPerDay; i++ {
|
||||
// Pick random master
|
||||
master := masters[g.rng.Intn(len(masters))]
|
||||
|
||||
// Pick random key for this master
|
||||
masterKeys := keys[master.ID]
|
||||
var keyID uint
|
||||
if len(masterKeys) > 0 {
|
||||
keyID = masterKeys[g.rng.Intn(len(masterKeys))].ID
|
||||
}
|
||||
|
||||
// Pick random group
|
||||
group := groups[g.rng.Intn(len(groups))]
|
||||
|
||||
// Pick random model from group
|
||||
var modelName string
|
||||
if len(group.Models) > 0 {
|
||||
modelName = group.Models[g.rng.Intn(len(group.Models))]
|
||||
} else {
|
||||
modelName = "gpt-4o"
|
||||
}
|
||||
|
||||
// Random timestamp within the day
|
||||
hour := g.rng.Intn(24)
|
||||
minute := g.rng.Intn(60)
|
||||
second := g.rng.Intn(60)
|
||||
timestamp := time.Date(
|
||||
dayTime.Year(), dayTime.Month(), dayTime.Day(),
|
||||
hour, minute, second, 0, time.UTC,
|
||||
)
|
||||
|
||||
// Generate realistic metrics
|
||||
statusCode := 200
|
||||
errorMsg := ""
|
||||
|
||||
// 2-5% error rate
|
||||
if g.rng.Float64() < 0.03 {
|
||||
errorCodes := []int{400, 401, 429, 500, 503}
|
||||
statusCode = errorCodes[g.rng.Intn(len(errorCodes))]
|
||||
errorMsgs := []string{
|
||||
"rate limit exceeded",
|
||||
"invalid request",
|
||||
"authentication failed",
|
||||
"internal server error",
|
||||
"service unavailable",
|
||||
}
|
||||
errorMsg = errorMsgs[g.rng.Intn(len(errorMsgs))]
|
||||
}
|
||||
|
||||
// Realistic latency (50ms - 5000ms, with most around 200-500ms)
|
||||
latencyMs := int64(50 + g.rng.ExpFloat64()*300)
|
||||
if latencyMs > 5000 {
|
||||
latencyMs = 5000
|
||||
}
|
||||
|
||||
// Realistic token counts
|
||||
tokensIn := int64(100 + g.rng.Intn(2000))
|
||||
tokensOut := int64(50 + g.rng.Intn(1000))
|
||||
|
||||
// Request/response sizes
|
||||
requestSize := tokensIn * 4 // ~4 bytes per token
|
||||
responseSize := tokensOut * 4
|
||||
|
||||
// Random client IP
|
||||
clientIP := fmt.Sprintf("192.168.%d.%d", g.rng.Intn(256), g.rng.Intn(256))
|
||||
|
||||
_ = timestamp // TODO: need to set created_at somehow
|
||||
|
||||
result = append(result, LogRequest{
|
||||
Group: master.Group,
|
||||
MasterID: master.ID,
|
||||
KeyID: keyID,
|
||||
ModelName: modelName,
|
||||
ProviderID: group.ID,
|
||||
ProviderType: group.Type,
|
||||
ProviderName: group.Name,
|
||||
StatusCode: statusCode,
|
||||
LatencyMs: latencyMs,
|
||||
TokensIn: tokensIn,
|
||||
TokensOut: tokensOut,
|
||||
ErrorMessage: errorMsg,
|
||||
ClientIP: clientIP,
|
||||
RequestSize: requestSize,
|
||||
ResponseSize: responseSize,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// --- Helper functions ---
|
||||
|
||||
func (g *Generator) randomString(length int) string {
|
||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||
b := make([]byte, length)
|
||||
for i := range b {
|
||||
b[i] = charset[g.rng.Intn(len(charset))]
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// HasSeederTag checks if a string contains the seeder tag
|
||||
func HasSeederTag(s, tag string) bool {
|
||||
return strings.Contains(s, tag)
|
||||
}
|
||||
|
||||
// ExtractSeederTag extracts the seeder tag from a string
|
||||
func ExtractSeederTag(s string) string {
|
||||
if idx := strings.Index(s, "seeder:"); idx != -1 {
|
||||
end := strings.Index(s[idx:], "]")
|
||||
if end == -1 {
|
||||
end = len(s) - idx
|
||||
}
|
||||
return s[idx : idx+end]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
287
cmd/seeder/main.go
Normal file
287
cmd/seeder/main.go
Normal file
@@ -0,0 +1,287 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Config holds seeder configuration
|
||||
type Config struct {
|
||||
BaseURL string
|
||||
AdminToken string
|
||||
Profile string
|
||||
Seed int64
|
||||
UsageDays int
|
||||
Env string
|
||||
Reset bool
|
||||
DryRun bool
|
||||
Force bool
|
||||
Verbose bool
|
||||
}
|
||||
|
||||
// ProfileConfig defines entity counts for a profile
|
||||
type ProfileConfig struct {
|
||||
Namespaces int
|
||||
ProviderGroups int
|
||||
ProvidersPerGroup int
|
||||
Models int
|
||||
Bindings int
|
||||
Masters int
|
||||
KeysPerMaster int
|
||||
UsageDays int
|
||||
}
|
||||
|
||||
var profiles = map[string]ProfileConfig{
|
||||
"demo": {
|
||||
Namespaces: 2,
|
||||
ProviderGroups: 2,
|
||||
ProvidersPerGroup: 2,
|
||||
Models: 5,
|
||||
Bindings: 4,
|
||||
Masters: 2,
|
||||
KeysPerMaster: 3,
|
||||
UsageDays: 7,
|
||||
},
|
||||
"load": {
|
||||
Namespaces: 5,
|
||||
ProviderGroups: 10,
|
||||
ProvidersPerGroup: 3,
|
||||
Models: 30,
|
||||
Bindings: 50,
|
||||
Masters: 50,
|
||||
KeysPerMaster: 2,
|
||||
UsageDays: 30,
|
||||
},
|
||||
"dev": {
|
||||
Namespaces: 3,
|
||||
ProviderGroups: 5,
|
||||
ProvidersPerGroup: 3,
|
||||
Models: 20,
|
||||
Bindings: 15,
|
||||
Masters: 10,
|
||||
KeysPerMaster: 2,
|
||||
UsageDays: 14,
|
||||
},
|
||||
}
|
||||
|
||||
func main() {
|
||||
var cfg Config
|
||||
|
||||
rootCmd := &cobra.Command{
|
||||
Use: "seeder",
|
||||
Short: "EZ-API Demo Data Seeder",
|
||||
Long: `A CLI tool to seed demo data into EZ-API Control Plane.
|
||||
|
||||
Creates namespaces, provider groups, providers, models, bindings,
|
||||
masters, API keys, and usage samples for frontend development and demos.`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
return run(cfg)
|
||||
},
|
||||
}
|
||||
|
||||
// Bind flags
|
||||
rootCmd.Flags().StringVar(&cfg.BaseURL, "base-url", "http://localhost:8080", "Control Plane API base URL")
|
||||
rootCmd.Flags().StringVar(&cfg.AdminToken, "admin-token", "", "Admin authentication token (required)")
|
||||
rootCmd.Flags().StringVar(&cfg.Profile, "profile", "demo", "Dataset profile: demo, load, dev")
|
||||
rootCmd.Flags().Int64Var(&cfg.Seed, "seed", 0, "Fixed random seed for reproducibility (0 = random)")
|
||||
rootCmd.Flags().IntVar(&cfg.UsageDays, "usage-days", 0, "Time window for usage samples (0 = profile default)")
|
||||
rootCmd.Flags().StringVar(&cfg.Env, "env", "dev", "Environment: dev, test, staging, production")
|
||||
rootCmd.Flags().BoolVar(&cfg.Reset, "reset", false, "Remove previous seeder data before recreating")
|
||||
rootCmd.Flags().BoolVar(&cfg.DryRun, "dry-run", false, "Print actions without executing write API calls")
|
||||
rootCmd.Flags().BoolVar(&cfg.Force, "force", false, "Allow running against production")
|
||||
rootCmd.Flags().BoolVar(&cfg.Verbose, "verbose", false, "Print detailed progress")
|
||||
|
||||
// Bind environment variables
|
||||
viper.SetEnvPrefix("SEEDER")
|
||||
viper.AutomaticEnv()
|
||||
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||
|
||||
// Override from env if not set via flag
|
||||
rootCmd.PreRunE = func(cmd *cobra.Command, args []string) error {
|
||||
if !cmd.Flags().Changed("base-url") && viper.IsSet("BASE_URL") {
|
||||
cfg.BaseURL = viper.GetString("BASE_URL")
|
||||
}
|
||||
if !cmd.Flags().Changed("admin-token") && viper.IsSet("ADMIN_TOKEN") {
|
||||
cfg.AdminToken = viper.GetString("ADMIN_TOKEN")
|
||||
}
|
||||
if !cmd.Flags().Changed("profile") && viper.IsSet("PROFILE") {
|
||||
cfg.Profile = viper.GetString("PROFILE")
|
||||
}
|
||||
if !cmd.Flags().Changed("seed") && viper.IsSet("SEED") {
|
||||
cfg.Seed = viper.GetInt64("SEED")
|
||||
}
|
||||
if !cmd.Flags().Changed("usage-days") && viper.IsSet("USAGE_DAYS") {
|
||||
cfg.UsageDays = viper.GetInt("USAGE_DAYS")
|
||||
}
|
||||
if !cmd.Flags().Changed("env") && viper.IsSet("ENV") {
|
||||
cfg.Env = viper.GetString("ENV")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := rootCmd.Execute(); err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(cfg Config) error {
|
||||
// Validate required fields
|
||||
if cfg.AdminToken == "" {
|
||||
return fmt.Errorf("admin-token is required (use --admin-token or SEEDER_ADMIN_TOKEN)")
|
||||
}
|
||||
|
||||
// Validate profile
|
||||
profile, ok := profiles[cfg.Profile]
|
||||
if !ok {
|
||||
return fmt.Errorf("invalid profile: %s (valid: demo, load, dev)", cfg.Profile)
|
||||
}
|
||||
|
||||
// Environment guard
|
||||
if isProduction(cfg.Env) && !cfg.Force {
|
||||
return fmt.Errorf("refusing to run against production environment (env=%s). Use --force to override", cfg.Env)
|
||||
}
|
||||
|
||||
// Initialize seed
|
||||
if cfg.Seed == 0 {
|
||||
cfg.Seed = time.Now().UnixNano()
|
||||
}
|
||||
|
||||
// Use profile default for usage days if not specified
|
||||
if cfg.UsageDays == 0 {
|
||||
cfg.UsageDays = profile.UsageDays
|
||||
}
|
||||
|
||||
// Create seeder tag
|
||||
seederTag := fmt.Sprintf("seeder:%s:seed-%d", cfg.Profile, cfg.Seed)
|
||||
|
||||
// Print banner
|
||||
printBanner(cfg, seederTag)
|
||||
|
||||
// Warning for production
|
||||
if isProduction(cfg.Env) && cfg.Force {
|
||||
fmt.Println("\n⚠️ WARNING: Running against PRODUCTION environment!")
|
||||
fmt.Println(" Press Ctrl+C within 5 seconds to abort...")
|
||||
time.Sleep(5 * time.Second)
|
||||
fmt.Println(" Proceeding...")
|
||||
}
|
||||
|
||||
// Create seeder instance
|
||||
seeder := NewSeeder(cfg, profile, seederTag)
|
||||
|
||||
// Run seeding
|
||||
return seeder.Run()
|
||||
}
|
||||
|
||||
func isProduction(env string) bool {
|
||||
env = strings.ToLower(env)
|
||||
return env == "production" || env == "prod"
|
||||
}
|
||||
|
||||
func printBanner(cfg Config, seederTag string) {
|
||||
mode := "create"
|
||||
if cfg.Reset {
|
||||
mode = "reset + create"
|
||||
}
|
||||
if cfg.DryRun {
|
||||
mode += " (dry-run)"
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
fmt.Println("EZ-API Demo Data Seeder")
|
||||
fmt.Println("=======================")
|
||||
fmt.Printf("Profile: %s\n", cfg.Profile)
|
||||
fmt.Printf("Seed: %d\n", cfg.Seed)
|
||||
fmt.Printf("Target: %s\n", cfg.BaseURL)
|
||||
fmt.Printf("Environment: %s\n", cfg.Env)
|
||||
fmt.Printf("Usage Window: %d days\n", cfg.UsageDays)
|
||||
fmt.Printf("Mode: %s\n", mode)
|
||||
fmt.Printf("Seeder Tag: %s\n", seederTag)
|
||||
fmt.Println()
|
||||
}
|
||||
|
||||
// Seeder handles the seeding process
|
||||
type Seeder struct {
|
||||
cfg Config
|
||||
profile ProfileConfig
|
||||
seederTag string
|
||||
client *Client
|
||||
rng *rand.Rand
|
||||
summary Summary
|
||||
}
|
||||
|
||||
// Summary tracks created/skipped counts
|
||||
type Summary struct {
|
||||
Namespaces CountPair
|
||||
ProviderGroups CountPair
|
||||
Providers CountPair
|
||||
Models CountPair
|
||||
Bindings CountPair
|
||||
Masters CountPair
|
||||
Keys CountPair
|
||||
UsageSamples int
|
||||
}
|
||||
|
||||
type CountPair struct {
|
||||
Created int
|
||||
Skipped int
|
||||
}
|
||||
|
||||
// NewSeeder creates a new seeder instance
|
||||
func NewSeeder(cfg Config, profile ProfileConfig, seederTag string) *Seeder {
|
||||
return &Seeder{
|
||||
cfg: cfg,
|
||||
profile: profile,
|
||||
seederTag: seederTag,
|
||||
client: NewClient(cfg.BaseURL, cfg.AdminToken, cfg.DryRun, cfg.Verbose),
|
||||
rng: rand.New(rand.NewSource(cfg.Seed)),
|
||||
}
|
||||
}
|
||||
|
||||
// Run executes the seeding process
|
||||
func (s *Seeder) Run() error {
|
||||
steps := []struct {
|
||||
name string
|
||||
fn func() error
|
||||
}{
|
||||
{"Creating namespaces", s.seedNamespaces},
|
||||
{"Creating provider groups", s.seedProviderGroups},
|
||||
{"Creating providers", s.seedProviders},
|
||||
{"Creating models", s.seedModels},
|
||||
{"Creating bindings", s.seedBindings},
|
||||
{"Creating masters", s.seedMasters},
|
||||
{"Creating API keys", s.seedKeys},
|
||||
{"Generating usage samples", s.seedUsageSamples},
|
||||
}
|
||||
|
||||
for i, step := range steps {
|
||||
fmt.Printf("[%d/%d] %s...\n", i+1, len(steps), step.name)
|
||||
if err := step.fn(); err != nil {
|
||||
return fmt.Errorf("%s: %w", step.name, err)
|
||||
}
|
||||
}
|
||||
|
||||
s.printSummary()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) printSummary() {
|
||||
fmt.Println("\nDone! Summary:")
|
||||
fmt.Printf(" Namespaces: %d created, %d skipped\n", s.summary.Namespaces.Created, s.summary.Namespaces.Skipped)
|
||||
fmt.Printf(" Provider Groups: %d created, %d skipped\n", s.summary.ProviderGroups.Created, s.summary.ProviderGroups.Skipped)
|
||||
fmt.Printf(" Providers: %d created, %d skipped\n", s.summary.Providers.Created, s.summary.Providers.Skipped)
|
||||
fmt.Printf(" Models: %d created, %d skipped\n", s.summary.Models.Created, s.summary.Models.Skipped)
|
||||
fmt.Printf(" Bindings: %d created, %d skipped\n", s.summary.Bindings.Created, s.summary.Bindings.Skipped)
|
||||
fmt.Printf(" Masters: %d created, %d skipped\n", s.summary.Masters.Created, s.summary.Masters.Skipped)
|
||||
fmt.Printf(" API Keys: %d created, %d skipped\n", s.summary.Keys.Created, s.summary.Keys.Skipped)
|
||||
fmt.Printf(" Usage Samples: %d entries\n", s.summary.UsageSamples)
|
||||
fmt.Printf("\nSeeder tag: %s\n", s.seederTag)
|
||||
fmt.Printf("\nDashboard should now show data at:\n %s/admin/dashboard\n", s.cfg.BaseURL)
|
||||
}
|
||||
|
||||
// Seed methods are implemented in seeder.go
|
||||
433
cmd/seeder/seeder.go
Normal file
433
cmd/seeder/seeder.go
Normal file
@@ -0,0 +1,433 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// seededNamespaces stores created namespace names for later use
|
||||
var seededNamespaces []string
|
||||
|
||||
// seededProviderGroups stores created provider groups for later use
|
||||
var seededProviderGroups []ProviderGroupResponse
|
||||
|
||||
// seededMasters stores created masters for later use
|
||||
var seededMasters []MasterResponse
|
||||
|
||||
// seededKeys stores created keys per master for usage samples
|
||||
var seededKeys map[uint][]KeyResponse
|
||||
|
||||
func (s *Seeder) seedNamespaces() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
namespaces := gen.GenerateNamespaces(s.profile.Namespaces)
|
||||
|
||||
// Get existing namespaces
|
||||
existing, err := s.client.ListNamespaces()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
// In dry-run mode, continue without checking existing
|
||||
existing = []NamespaceResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing namespaces, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list namespaces: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]NamespaceResponse)
|
||||
for _, ns := range existing {
|
||||
existingMap[ns.Name] = ns
|
||||
}
|
||||
|
||||
seededNamespaces = make([]string, 0, len(namespaces))
|
||||
|
||||
for _, ns := range namespaces {
|
||||
if existingNs, exists := existingMap[ns.Name]; exists {
|
||||
// Check if we should reset
|
||||
if s.cfg.Reset && HasSeederTag(existingNs.Description, s.seederTag) {
|
||||
if err := s.client.DeleteNamespace(existingNs.ID); err != nil {
|
||||
return fmt.Errorf("delete namespace %s: %w", ns.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", ns.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Skipped++
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Create namespace
|
||||
created, err := s.client.CreateNamespace(ns)
|
||||
if err != nil {
|
||||
// Check if it's a conflict (already exists)
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Skipped++
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create namespace %s: %w", ns.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", ns.Name)
|
||||
}
|
||||
s.summary.Namespaces.Created++
|
||||
if created != nil {
|
||||
seededNamespaces = append(seededNamespaces, created.Name)
|
||||
} else {
|
||||
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedProviderGroups() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
groups := gen.GenerateProviderGroups(s.profile.ProviderGroups)
|
||||
|
||||
// Get existing provider groups
|
||||
existing, err := s.client.ListProviderGroups()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []ProviderGroupResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing provider groups, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list provider groups: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]ProviderGroupResponse)
|
||||
for _, g := range existing {
|
||||
existingMap[g.Name] = g
|
||||
}
|
||||
|
||||
seededProviderGroups = make([]ProviderGroupResponse, 0, len(groups))
|
||||
|
||||
for _, group := range groups {
|
||||
if existingGroup, exists := existingMap[group.Name]; exists {
|
||||
// Check if we should reset (check name prefix for seeder tag)
|
||||
if s.cfg.Reset && strings.HasPrefix(group.Name, "demo-") {
|
||||
if err := s.client.DeleteProviderGroup(existingGroup.ID); err != nil {
|
||||
return fmt.Errorf("delete provider group %s: %w", group.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", group.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Skipped++
|
||||
seededProviderGroups = append(seededProviderGroups, existingGroup)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Create provider group
|
||||
created, err := s.client.CreateProviderGroup(group)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create provider group %s: %w", group.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", group.Name)
|
||||
}
|
||||
s.summary.ProviderGroups.Created++
|
||||
if created != nil {
|
||||
seededProviderGroups = append(seededProviderGroups, *created)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedProviders() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
for _, group := range seededProviderGroups {
|
||||
providers := gen.GenerateProviders(group.ID, group.Name, s.profile.ProvidersPerGroup)
|
||||
|
||||
createdCount := 0
|
||||
for _, provider := range providers {
|
||||
_, err := s.client.CreateAPIKey(provider)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Providers.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create provider for group %s: %w", group.Name, err)
|
||||
}
|
||||
s.summary.Providers.Created++
|
||||
createdCount++
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s: %d providers created\n", group.Name, createdCount)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedModels() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
models := gen.GenerateModels(s.profile.Models)
|
||||
|
||||
// Get existing models
|
||||
existing, err := s.client.ListModels()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []ModelResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing models, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list models: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]ModelResponse)
|
||||
for _, m := range existing {
|
||||
existingMap[m.Name] = m
|
||||
}
|
||||
|
||||
for _, model := range models {
|
||||
if existingModel, exists := existingMap[model.Name]; exists {
|
||||
if s.cfg.Reset {
|
||||
if err := s.client.DeleteModel(existingModel.ID); err != nil {
|
||||
return fmt.Errorf("delete model %s: %w", model.Name, err)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.client.CreateModel(model)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create model %s: %w", model.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", model.Name)
|
||||
}
|
||||
s.summary.Models.Created++
|
||||
}
|
||||
|
||||
if !s.cfg.Verbose && !s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d models created\n", s.summary.Models.Created)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedBindings() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
bindings := gen.GenerateBindings(seededNamespaces, seededProviderGroups, s.profile.Bindings)
|
||||
|
||||
// Get existing bindings
|
||||
existing, err := s.client.ListBindings()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []BindingResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing bindings, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list bindings: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Create a map of existing bindings by composite key
|
||||
existingMap := make(map[string]BindingResponse)
|
||||
for _, b := range existing {
|
||||
key := fmt.Sprintf("%s:%s:%d", b.Namespace, b.PublicModel, b.GroupID)
|
||||
existingMap[key] = b
|
||||
}
|
||||
|
||||
for _, binding := range bindings {
|
||||
key := fmt.Sprintf("%s:%s:%d", binding.Namespace, binding.PublicModel, binding.GroupID)
|
||||
if existingBinding, exists := existingMap[key]; exists {
|
||||
if s.cfg.Reset {
|
||||
if err := s.client.DeleteBinding(existingBinding.ID); err != nil {
|
||||
return fmt.Errorf("delete binding: %w", err)
|
||||
}
|
||||
} else {
|
||||
s.summary.Bindings.Skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := s.client.CreateBinding(binding)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Bindings.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create binding: %w", err)
|
||||
}
|
||||
s.summary.Bindings.Created++
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d bindings created\n", s.summary.Bindings.Created)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedMasters() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
masters := gen.GenerateMasters(s.profile.Masters)
|
||||
|
||||
// Get existing masters
|
||||
existing, err := s.client.ListMasters()
|
||||
if err != nil {
|
||||
if s.cfg.DryRun {
|
||||
existing = []MasterResponse{}
|
||||
fmt.Printf(" [DRY-RUN] Unable to list existing masters, proceeding anyway\n")
|
||||
} else {
|
||||
return fmt.Errorf("list masters: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
existingMap := make(map[string]MasterResponse)
|
||||
for _, m := range existing {
|
||||
existingMap[m.Name] = m
|
||||
}
|
||||
|
||||
seededMasters = make([]MasterResponse, 0, len(masters))
|
||||
|
||||
for _, master := range masters {
|
||||
if existingMaster, exists := existingMap[master.Name]; exists {
|
||||
if s.cfg.Reset && strings.HasSuffix(master.Name, "-demo") {
|
||||
if err := s.client.DeleteMaster(existingMaster.ID); err != nil {
|
||||
return fmt.Errorf("delete master %s: %w", master.Name, err)
|
||||
}
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ✗ %s (deleted for reset)\n", master.Name)
|
||||
}
|
||||
} else {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Skipped++
|
||||
seededMasters = append(seededMasters, existingMaster)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
created, err := s.client.CreateMaster(master)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
if s.cfg.Verbose {
|
||||
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create master %s: %w", master.Name, err)
|
||||
}
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s (created)\n", master.Name)
|
||||
}
|
||||
s.summary.Masters.Created++
|
||||
if created != nil {
|
||||
seededMasters = append(seededMasters, *created)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedKeys() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
seededKeys = make(map[uint][]KeyResponse)
|
||||
|
||||
for _, master := range seededMasters {
|
||||
keys := gen.GenerateKeys(s.profile.KeysPerMaster)
|
||||
|
||||
masterKeys := make([]KeyResponse, 0, len(keys))
|
||||
createdCount := 0
|
||||
|
||||
for _, key := range keys {
|
||||
created, err := s.client.CreateKey(master.ID, key)
|
||||
if err != nil {
|
||||
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||
s.summary.Keys.Skipped++
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("create key for master %s: %w", master.Name, err)
|
||||
}
|
||||
s.summary.Keys.Created++
|
||||
createdCount++
|
||||
if created != nil {
|
||||
masterKeys = append(masterKeys, *created)
|
||||
}
|
||||
}
|
||||
|
||||
seededKeys[master.ID] = masterKeys
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %s: %d keys created\n", master.Name, createdCount)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Seeder) seedUsageSamples() error {
|
||||
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||
|
||||
logs := gen.GenerateUsageSamples(seededMasters, seededKeys, seededProviderGroups, s.cfg.UsageDays)
|
||||
|
||||
if len(logs) == 0 {
|
||||
fmt.Printf(" ○ No masters or groups to generate samples for\n")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Batch insert logs
|
||||
batchSize := 100
|
||||
for i := 0; i < len(logs); i += batchSize {
|
||||
end := i + batchSize
|
||||
if end > len(logs) {
|
||||
end = len(logs)
|
||||
}
|
||||
batch := logs[i:end]
|
||||
|
||||
if err := s.client.CreateLogsBatch(batch); err != nil {
|
||||
return fmt.Errorf("create logs batch: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
s.summary.UsageSamples = len(logs)
|
||||
|
||||
if s.cfg.Verbose || s.cfg.DryRun {
|
||||
fmt.Printf(" ✓ %d days of data generated (%d log entries)\n", s.cfg.UsageDays, len(logs))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user