mirror of
https://github.com/EZ-Api/ez-api.git
synced 2026-01-13 09:37:53 +00:00
feat(seeder): add control plane data seeder
Introduce a `cmd/seeder` CLI to generate deterministic demo datasets and seed them into the Control Plane via admin endpoints, supporting reset, dry-run, profiles, and usage sample generation. Add Cobra/Viper dependencies to support the new command.
This commit is contained in:
498
cmd/seeder/client.go
Normal file
498
cmd/seeder/client.go
Normal file
@@ -0,0 +1,498 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Client wraps HTTP calls to the Control Plane API
|
||||||
|
type Client struct {
|
||||||
|
baseURL string
|
||||||
|
adminToken string
|
||||||
|
dryRun bool
|
||||||
|
verbose bool
|
||||||
|
httpClient *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewClient creates a new API client
|
||||||
|
func NewClient(baseURL, adminToken string, dryRun, verbose bool) *Client {
|
||||||
|
return &Client{
|
||||||
|
baseURL: baseURL,
|
||||||
|
adminToken: adminToken,
|
||||||
|
dryRun: dryRun,
|
||||||
|
verbose: verbose,
|
||||||
|
httpClient: &http.Client{Timeout: 30 * time.Second},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// APIError represents an error response from the API
|
||||||
|
type APIError struct {
|
||||||
|
StatusCode int
|
||||||
|
Body string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *APIError) Error() string {
|
||||||
|
return fmt.Sprintf("API error %d: %s", e.StatusCode, e.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsNotFound returns true if the error is a 404
|
||||||
|
func (e *APIError) IsNotFound() bool {
|
||||||
|
return e.StatusCode == http.StatusNotFound
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsConflict returns true if the error is a 409
|
||||||
|
func (e *APIError) IsConflict() bool {
|
||||||
|
return e.StatusCode == http.StatusConflict
|
||||||
|
}
|
||||||
|
|
||||||
|
// doRequest executes an HTTP request
|
||||||
|
func (c *Client) doRequest(method, path string, body any) ([]byte, error) {
|
||||||
|
var reqBody io.Reader
|
||||||
|
if body != nil {
|
||||||
|
data, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("marshal request: %w", err)
|
||||||
|
}
|
||||||
|
reqBody = bytes.NewReader(data)
|
||||||
|
if c.verbose {
|
||||||
|
fmt.Printf(" Request: %s %s\n", method, path)
|
||||||
|
fmt.Printf(" Body: %s\n", string(data))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest(method, c.baseURL+path, reqBody)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("create request: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", "Bearer "+c.adminToken)
|
||||||
|
|
||||||
|
resp, err := c.httpClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("execute request: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
respBody, err := io.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("read response: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.verbose {
|
||||||
|
fmt.Printf(" Response: %d %s\n", resp.StatusCode, string(respBody))
|
||||||
|
}
|
||||||
|
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
return nil, &APIError{StatusCode: resp.StatusCode, Body: string(respBody)}
|
||||||
|
}
|
||||||
|
|
||||||
|
return respBody, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// get performs a GET request
|
||||||
|
func (c *Client) get(path string) ([]byte, error) {
|
||||||
|
return c.doRequest(http.MethodGet, path, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
// post performs a POST request (respects dry-run)
|
||||||
|
func (c *Client) post(path string, body any) ([]byte, error) {
|
||||||
|
if c.dryRun {
|
||||||
|
data, _ := json.Marshal(body)
|
||||||
|
fmt.Printf(" [DRY-RUN] POST %s: %s\n", path, string(data))
|
||||||
|
return []byte("{}"), nil
|
||||||
|
}
|
||||||
|
return c.doRequest(http.MethodPost, path, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// put performs a PUT request (respects dry-run)
|
||||||
|
func (c *Client) put(path string, body any) ([]byte, error) {
|
||||||
|
if c.dryRun {
|
||||||
|
data, _ := json.Marshal(body)
|
||||||
|
fmt.Printf(" [DRY-RUN] PUT %s: %s\n", path, string(data))
|
||||||
|
return []byte("{}"), nil
|
||||||
|
}
|
||||||
|
return c.doRequest(http.MethodPut, path, body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// delete performs a DELETE request (respects dry-run)
|
||||||
|
func (c *Client) delete(path string) error {
|
||||||
|
if c.dryRun {
|
||||||
|
fmt.Printf(" [DRY-RUN] DELETE %s\n", path)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
_, err := c.doRequest(http.MethodDelete, path, nil)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Namespace API ---
|
||||||
|
|
||||||
|
type NamespaceRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
Description string `json:"description,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type NamespaceResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
Description string `json:"description"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListNamespaces() ([]NamespaceResponse, error) {
|
||||||
|
data, err := c.get("/admin/namespaces")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []NamespaceResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal namespaces: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateNamespace(req NamespaceRequest) (*NamespaceResponse, error) {
|
||||||
|
data, err := c.post("/admin/namespaces", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result NamespaceResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal namespace: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteNamespace(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/namespaces/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- ProviderGroup API ---
|
||||||
|
|
||||||
|
type ProviderGroupRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
BaseURL string `json:"base_url,omitempty"`
|
||||||
|
GoogleProject string `json:"google_project,omitempty"`
|
||||||
|
GoogleLocation string `json:"google_location,omitempty"`
|
||||||
|
StaticHeaders string `json:"static_headers,omitempty"`
|
||||||
|
HeadersProfile string `json:"headers_profile,omitempty"`
|
||||||
|
Models []string `json:"models,omitempty"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ProviderGroupResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
BaseURL string `json:"base_url"`
|
||||||
|
GoogleProject string `json:"google_project"`
|
||||||
|
GoogleLocation string `json:"google_location"`
|
||||||
|
Models []string `json:"models"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListProviderGroups() ([]ProviderGroupResponse, error) {
|
||||||
|
data, err := c.get("/admin/provider-groups")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []ProviderGroupResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal provider groups: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateProviderGroup(req ProviderGroupRequest) (*ProviderGroupResponse, error) {
|
||||||
|
data, err := c.post("/admin/provider-groups", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result ProviderGroupResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal provider group: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteProviderGroup(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/provider-groups/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- APIKey (Provider) API ---
|
||||||
|
|
||||||
|
type APIKeyRequest struct {
|
||||||
|
GroupID uint `json:"group_id"`
|
||||||
|
APIKey string `json:"api_key"`
|
||||||
|
AccessToken string `json:"access_token,omitempty"`
|
||||||
|
RefreshToken string `json:"refresh_token,omitempty"`
|
||||||
|
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||||
|
AccountID string `json:"account_id,omitempty"`
|
||||||
|
ProjectID string `json:"project_id,omitempty"`
|
||||||
|
Weight int `json:"weight,omitempty"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
AutoBan *bool `json:"auto_ban,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type APIKeyResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
GroupID uint `json:"group_id"`
|
||||||
|
APIKey string `json:"api_key"`
|
||||||
|
Weight int `json:"weight"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
AutoBan bool `json:"auto_ban"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListAPIKeys() ([]APIKeyResponse, error) {
|
||||||
|
data, err := c.get("/admin/api-keys")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []APIKeyResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal api keys: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateAPIKey(req APIKeyRequest) (*APIKeyResponse, error) {
|
||||||
|
data, err := c.post("/admin/api-keys", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result APIKeyResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal api key: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteAPIKey(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/api-keys/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Model API ---
|
||||||
|
|
||||||
|
type ModelRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind,omitempty"`
|
||||||
|
ContextWindow int `json:"context_window,omitempty"`
|
||||||
|
CostPerToken float64 `json:"cost_per_token,omitempty"`
|
||||||
|
SupportsVision bool `json:"supports_vision,omitempty"`
|
||||||
|
SupportsFunctions bool `json:"supports_functions,omitempty"`
|
||||||
|
SupportsToolChoice bool `json:"supports_tool_choice,omitempty"`
|
||||||
|
SupportsFIM bool `json:"supports_fim,omitempty"`
|
||||||
|
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ModelResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Kind string `json:"kind"`
|
||||||
|
ContextWindow int `json:"context_window"`
|
||||||
|
CostPerToken float64 `json:"cost_per_token"`
|
||||||
|
SupportsVision bool `json:"supports_vision"`
|
||||||
|
SupportsFunctions bool `json:"supports_functions"`
|
||||||
|
SupportsToolChoice bool `json:"supports_tool_choice"`
|
||||||
|
SupportsFIM bool `json:"supports_fim"`
|
||||||
|
MaxOutputTokens int `json:"max_output_tokens"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListModels() ([]ModelResponse, error) {
|
||||||
|
data, err := c.get("/admin/models")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []ModelResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal models: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateModel(req ModelRequest) (*ModelResponse, error) {
|
||||||
|
data, err := c.post("/admin/models", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result ModelResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal model: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteModel(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/models/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Binding API ---
|
||||||
|
|
||||||
|
type BindingRequest struct {
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
PublicModel string `json:"public_model"`
|
||||||
|
GroupID uint `json:"group_id"`
|
||||||
|
Weight int `json:"weight,omitempty"`
|
||||||
|
SelectorType string `json:"selector_type,omitempty"`
|
||||||
|
SelectorValue string `json:"selector_value,omitempty"`
|
||||||
|
Status string `json:"status,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type BindingResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
Namespace string `json:"namespace"`
|
||||||
|
PublicModel string `json:"public_model"`
|
||||||
|
GroupID uint `json:"group_id"`
|
||||||
|
Weight int `json:"weight"`
|
||||||
|
SelectorType string `json:"selector_type"`
|
||||||
|
SelectorValue string `json:"selector_value"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListBindings() ([]BindingResponse, error) {
|
||||||
|
data, err := c.get("/admin/bindings")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []BindingResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal bindings: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateBinding(req BindingRequest) (*BindingResponse, error) {
|
||||||
|
data, err := c.post("/admin/bindings", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result BindingResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal binding: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteBinding(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/bindings/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Master API ---
|
||||||
|
|
||||||
|
type MasterRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
Group string `json:"group"`
|
||||||
|
MaxChildKeys int `json:"max_child_keys,omitempty"`
|
||||||
|
GlobalQPS int `json:"global_qps,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type MasterResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Group string `json:"group"`
|
||||||
|
MasterKey string `json:"master_key,omitempty"` // Only returned on create
|
||||||
|
MaxChildKeys int `json:"max_child_keys"`
|
||||||
|
GlobalQPS int `json:"global_qps"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) ListMasters() ([]MasterResponse, error) {
|
||||||
|
data, err := c.get("/admin/masters")
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result []MasterResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal masters: %w", err)
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateMaster(req MasterRequest) (*MasterResponse, error) {
|
||||||
|
data, err := c.post("/admin/masters", req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result MasterResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal master: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) DeleteMaster(id uint) error {
|
||||||
|
return c.delete(fmt.Sprintf("/admin/masters/%d", id))
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Key API ---
|
||||||
|
|
||||||
|
type KeyRequest struct {
|
||||||
|
Group string `json:"group,omitempty"`
|
||||||
|
Scopes string `json:"scopes,omitempty"`
|
||||||
|
ModelLimits string `json:"model_limits,omitempty"`
|
||||||
|
ModelLimitsEnabled *bool `json:"model_limits_enabled,omitempty"`
|
||||||
|
ExpiresAt *string `json:"expires_at,omitempty"`
|
||||||
|
AllowIPs string `json:"allow_ips,omitempty"`
|
||||||
|
DenyIPs string `json:"deny_ips,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type KeyResponse struct {
|
||||||
|
ID uint `json:"id"`
|
||||||
|
KeySecret string `json:"key_secret,omitempty"` // Only returned on create
|
||||||
|
Group string `json:"group"`
|
||||||
|
Scopes string `json:"scopes"`
|
||||||
|
IssuedBy string `json:"issued_by"`
|
||||||
|
Status string `json:"status"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateKey(masterID uint, req KeyRequest) (*KeyResponse, error) {
|
||||||
|
data, err := c.post(fmt.Sprintf("/admin/masters/%d/keys", masterID), req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var result KeyResponse
|
||||||
|
if err := json.Unmarshal(data, &result); err != nil {
|
||||||
|
return nil, fmt.Errorf("unmarshal key: %w", err)
|
||||||
|
}
|
||||||
|
return &result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Log API ---
|
||||||
|
|
||||||
|
type LogRequest struct {
|
||||||
|
Group string `json:"group,omitempty"`
|
||||||
|
MasterID uint `json:"master_id,omitempty"`
|
||||||
|
KeyID uint `json:"key_id,omitempty"`
|
||||||
|
ModelName string `json:"model_name,omitempty"`
|
||||||
|
ProviderID uint `json:"provider_id,omitempty"`
|
||||||
|
ProviderType string `json:"provider_type,omitempty"`
|
||||||
|
ProviderName string `json:"provider_name,omitempty"`
|
||||||
|
StatusCode int `json:"status_code,omitempty"`
|
||||||
|
LatencyMs int64 `json:"latency_ms,omitempty"`
|
||||||
|
TokensIn int64 `json:"tokens_in,omitempty"`
|
||||||
|
TokensOut int64 `json:"tokens_out,omitempty"`
|
||||||
|
ErrorMessage string `json:"error_message,omitempty"`
|
||||||
|
ClientIP string `json:"client_ip,omitempty"`
|
||||||
|
RequestSize int64 `json:"request_size,omitempty"`
|
||||||
|
ResponseSize int64 `json:"response_size,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateLog(req LogRequest) error {
|
||||||
|
_, err := c.post("/logs", req)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) CreateLogsBatch(logs []LogRequest) error {
|
||||||
|
for _, log := range logs {
|
||||||
|
if err := c.CreateLog(log); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
452
cmd/seeder/generator.go
Normal file
452
cmd/seeder/generator.go
Normal file
@@ -0,0 +1,452 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Generator creates deterministic demo data
|
||||||
|
type Generator struct {
|
||||||
|
rng *rand.Rand
|
||||||
|
seederTag string
|
||||||
|
profile string
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewGenerator creates a new data generator
|
||||||
|
func NewGenerator(rng *rand.Rand, seederTag, profile string) *Generator {
|
||||||
|
return &Generator{
|
||||||
|
rng: rng,
|
||||||
|
seederTag: seederTag,
|
||||||
|
profile: profile,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Namespace Generation ---
|
||||||
|
|
||||||
|
var namespaceTemplates = []struct {
|
||||||
|
name string
|
||||||
|
desc string
|
||||||
|
}{
|
||||||
|
{"default", "Default namespace for general use"},
|
||||||
|
{"premium", "Premium tier with higher limits"},
|
||||||
|
{"staging", "Staging environment namespace"},
|
||||||
|
{"internal", "Internal services namespace"},
|
||||||
|
{"partner", "Partner integrations namespace"},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Generator) GenerateNamespaces(count int) []NamespaceRequest {
|
||||||
|
result := make([]NamespaceRequest, 0, count)
|
||||||
|
for i := 0; i < count && i < len(namespaceTemplates); i++ {
|
||||||
|
t := namespaceTemplates[i]
|
||||||
|
result = append(result, NamespaceRequest{
|
||||||
|
Name: fmt.Sprintf("demo-%s", t.name),
|
||||||
|
Status: "active",
|
||||||
|
Description: fmt.Sprintf("%s [%s]", t.desc, g.seederTag),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Provider Group Generation ---
|
||||||
|
|
||||||
|
var providerGroupTemplates = []struct {
|
||||||
|
name string
|
||||||
|
ptype string
|
||||||
|
baseURL string
|
||||||
|
models []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "openai-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.openai.com/v1",
|
||||||
|
models: []string{"gpt-4o", "gpt-4o-mini", "gpt-4-turbo", "gpt-3.5-turbo"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "anthropic-demo",
|
||||||
|
ptype: "anthropic",
|
||||||
|
baseURL: "https://api.anthropic.com",
|
||||||
|
models: []string{"claude-3-5-sonnet-20241022", "claude-3-5-haiku-20241022", "claude-3-opus-20240229"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "gemini-demo",
|
||||||
|
ptype: "gemini",
|
||||||
|
baseURL: "https://generativelanguage.googleapis.com/v1beta",
|
||||||
|
models: []string{"gemini-1.5-pro", "gemini-1.5-flash", "gemini-2.0-flash-exp"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "deepseek-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.deepseek.com/v1",
|
||||||
|
models: []string{"deepseek-chat", "deepseek-coder"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "groq-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.groq.com/openai/v1",
|
||||||
|
models: []string{"llama-3.1-70b-versatile", "llama-3.1-8b-instant", "mixtral-8x7b-32768"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "mistral-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.mistral.ai/v1",
|
||||||
|
models: []string{"mistral-large-latest", "mistral-medium-latest", "mistral-small-latest"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cohere-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.cohere.ai/v1",
|
||||||
|
models: []string{"command-r-plus", "command-r", "command"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "together-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://api.together.xyz/v1",
|
||||||
|
models: []string{"meta-llama/Llama-3-70b-chat-hf", "mistralai/Mixtral-8x22B-Instruct-v0.1"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "openrouter-demo",
|
||||||
|
ptype: "openai",
|
||||||
|
baseURL: "https://openrouter.ai/api/v1",
|
||||||
|
models: []string{"openai/gpt-4o", "anthropic/claude-3.5-sonnet"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "azure-demo",
|
||||||
|
ptype: "azure",
|
||||||
|
baseURL: "https://demo.openai.azure.com/openai/deployments",
|
||||||
|
models: []string{"gpt-4o-deploy", "gpt-35-turbo-deploy"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Generator) GenerateProviderGroups(count int) []ProviderGroupRequest {
|
||||||
|
result := make([]ProviderGroupRequest, 0, count)
|
||||||
|
for i := 0; i < count && i < len(providerGroupTemplates); i++ {
|
||||||
|
t := providerGroupTemplates[i]
|
||||||
|
result = append(result, ProviderGroupRequest{
|
||||||
|
Name: t.name,
|
||||||
|
Type: t.ptype,
|
||||||
|
BaseURL: t.baseURL,
|
||||||
|
Models: t.models,
|
||||||
|
Status: "active",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Provider (APIKey) Generation ---
|
||||||
|
|
||||||
|
func (g *Generator) GenerateProviders(groupID uint, groupName string, count int) []APIKeyRequest {
|
||||||
|
result := make([]APIKeyRequest, 0, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
// Generate a fake API key based on group name and index
|
||||||
|
fakeKey := fmt.Sprintf("sk-%s-%s-%04d", groupName, g.randomString(24), i+1)
|
||||||
|
result = append(result, APIKeyRequest{
|
||||||
|
GroupID: groupID,
|
||||||
|
APIKey: fakeKey,
|
||||||
|
Weight: 100,
|
||||||
|
Status: "active",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Model Generation ---
|
||||||
|
|
||||||
|
var modelTemplates = []struct {
|
||||||
|
name string
|
||||||
|
kind string
|
||||||
|
contextWindow int
|
||||||
|
maxOutput int
|
||||||
|
vision bool
|
||||||
|
functions bool
|
||||||
|
toolChoice bool
|
||||||
|
costPerToken float64
|
||||||
|
}{
|
||||||
|
{"gpt-4o", "chat", 128000, 16384, true, true, true, 0.000005},
|
||||||
|
{"gpt-4o-mini", "chat", 128000, 16384, true, true, true, 0.00000015},
|
||||||
|
{"gpt-4-turbo", "chat", 128000, 4096, true, true, true, 0.00001},
|
||||||
|
{"gpt-3.5-turbo", "chat", 16385, 4096, false, true, true, 0.0000005},
|
||||||
|
{"claude-3-5-sonnet-20241022", "chat", 200000, 8192, true, true, true, 0.000003},
|
||||||
|
{"claude-3-5-haiku-20241022", "chat", 200000, 8192, true, true, true, 0.00000025},
|
||||||
|
{"claude-3-opus-20240229", "chat", 200000, 4096, true, true, true, 0.000015},
|
||||||
|
{"gemini-1.5-pro", "chat", 2097152, 8192, true, true, true, 0.00000125},
|
||||||
|
{"gemini-1.5-flash", "chat", 1048576, 8192, true, true, true, 0.000000075},
|
||||||
|
{"gemini-2.0-flash-exp", "chat", 1048576, 8192, true, true, true, 0.0000001},
|
||||||
|
{"deepseek-chat", "chat", 64000, 4096, false, true, true, 0.00000014},
|
||||||
|
{"deepseek-coder", "chat", 64000, 4096, false, true, true, 0.00000014},
|
||||||
|
{"llama-3.1-70b-versatile", "chat", 131072, 8192, false, true, true, 0.00000059},
|
||||||
|
{"llama-3.1-8b-instant", "chat", 131072, 8192, false, true, true, 0.00000005},
|
||||||
|
{"mixtral-8x7b-32768", "chat", 32768, 4096, false, true, false, 0.00000027},
|
||||||
|
{"mistral-large-latest", "chat", 128000, 8192, false, true, true, 0.000002},
|
||||||
|
{"command-r-plus", "chat", 128000, 4096, false, true, true, 0.000003},
|
||||||
|
{"command-r", "chat", 128000, 4096, false, true, true, 0.0000005},
|
||||||
|
{"text-embedding-3-small", "embedding", 8191, 0, false, false, false, 0.00000002},
|
||||||
|
{"text-embedding-3-large", "embedding", 8191, 0, false, false, false, 0.00000013},
|
||||||
|
{"voyage-large-2", "embedding", 16000, 0, false, false, false, 0.00000012},
|
||||||
|
{"rerank-english-v3.0", "rerank", 4096, 0, false, false, false, 0.000001},
|
||||||
|
{"rerank-multilingual-v3.0", "rerank", 4096, 0, false, false, false, 0.000001},
|
||||||
|
{"o1-preview", "chat", 128000, 32768, true, false, false, 0.000015},
|
||||||
|
{"o1-mini", "chat", 128000, 65536, true, false, false, 0.000003},
|
||||||
|
{"dall-e-3", "other", 0, 0, false, false, false, 0.04},
|
||||||
|
{"whisper-1", "other", 0, 0, false, false, false, 0.006},
|
||||||
|
{"tts-1", "other", 0, 0, false, false, false, 0.000015},
|
||||||
|
{"tts-1-hd", "other", 0, 0, false, false, false, 0.00003},
|
||||||
|
{"codestral-latest", "chat", 32000, 8192, false, true, true, 0.000001},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Generator) GenerateModels(count int) []ModelRequest {
|
||||||
|
result := make([]ModelRequest, 0, count)
|
||||||
|
for i := 0; i < count && i < len(modelTemplates); i++ {
|
||||||
|
t := modelTemplates[i]
|
||||||
|
result = append(result, ModelRequest{
|
||||||
|
Name: t.name,
|
||||||
|
Kind: t.kind,
|
||||||
|
ContextWindow: t.contextWindow,
|
||||||
|
MaxOutputTokens: t.maxOutput,
|
||||||
|
SupportsVision: t.vision,
|
||||||
|
SupportsFunctions: t.functions,
|
||||||
|
SupportsToolChoice: t.toolChoice,
|
||||||
|
CostPerToken: t.costPerToken,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Binding Generation ---
|
||||||
|
|
||||||
|
func (g *Generator) GenerateBindings(namespaces []string, groups []ProviderGroupResponse, count int) []BindingRequest {
|
||||||
|
result := make([]BindingRequest, 0, count)
|
||||||
|
|
||||||
|
if len(namespaces) == 0 || len(groups) == 0 {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create bindings for common models
|
||||||
|
commonModels := []string{"gpt-4o", "gpt-4o-mini", "claude-3-5-sonnet-20241022", "gemini-1.5-pro"}
|
||||||
|
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
ns := namespaces[i%len(namespaces)]
|
||||||
|
group := groups[i%len(groups)]
|
||||||
|
|
||||||
|
var modelName string
|
||||||
|
if i < len(commonModels) {
|
||||||
|
modelName = commonModels[i]
|
||||||
|
} else {
|
||||||
|
// Pick a model from the group's models
|
||||||
|
if len(group.Models) > 0 {
|
||||||
|
modelName = group.Models[g.rng.Intn(len(group.Models))]
|
||||||
|
} else {
|
||||||
|
modelName = fmt.Sprintf("model-%d", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
result = append(result, BindingRequest{
|
||||||
|
Namespace: ns,
|
||||||
|
PublicModel: modelName,
|
||||||
|
GroupID: group.ID,
|
||||||
|
Weight: 100,
|
||||||
|
SelectorType: "exact",
|
||||||
|
SelectorValue: modelName,
|
||||||
|
Status: "active",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Master Generation ---
|
||||||
|
|
||||||
|
var masterTemplates = []struct {
|
||||||
|
name string
|
||||||
|
group string
|
||||||
|
maxKeys int
|
||||||
|
qps int
|
||||||
|
}{
|
||||||
|
{"admin-demo", "default", 10, 100},
|
||||||
|
{"user-demo", "default", 5, 50},
|
||||||
|
{"developer-demo", "default", 20, 200},
|
||||||
|
{"partner-demo", "partner", 15, 150},
|
||||||
|
{"staging-demo", "staging", 10, 100},
|
||||||
|
{"internal-demo", "internal", 50, 500},
|
||||||
|
{"test-demo", "default", 5, 25},
|
||||||
|
{"premium-demo", "premium", 100, 1000},
|
||||||
|
{"enterprise-demo", "enterprise", 200, 2000},
|
||||||
|
{"trial-demo", "trial", 3, 10},
|
||||||
|
}
|
||||||
|
|
||||||
|
func (g *Generator) GenerateMasters(count int) []MasterRequest {
|
||||||
|
result := make([]MasterRequest, 0, count)
|
||||||
|
for i := 0; i < count && i < len(masterTemplates); i++ {
|
||||||
|
t := masterTemplates[i]
|
||||||
|
result = append(result, MasterRequest{
|
||||||
|
Name: t.name,
|
||||||
|
Group: t.group,
|
||||||
|
MaxChildKeys: t.maxKeys,
|
||||||
|
GlobalQPS: t.qps,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Key Generation ---
|
||||||
|
|
||||||
|
func (g *Generator) GenerateKeys(count int) []KeyRequest {
|
||||||
|
result := make([]KeyRequest, 0, count)
|
||||||
|
for i := 0; i < count; i++ {
|
||||||
|
result = append(result, KeyRequest{
|
||||||
|
Scopes: "chat,embedding",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Usage Sample Generation ---
|
||||||
|
|
||||||
|
func (g *Generator) GenerateUsageSamples(
|
||||||
|
masters []MasterResponse,
|
||||||
|
keys map[uint][]KeyResponse,
|
||||||
|
groups []ProviderGroupResponse,
|
||||||
|
usageDays int,
|
||||||
|
) []LogRequest {
|
||||||
|
result := make([]LogRequest, 0)
|
||||||
|
|
||||||
|
if len(masters) == 0 || len(groups) == 0 {
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
startTime := now.AddDate(0, 0, -usageDays)
|
||||||
|
|
||||||
|
// Generate samples for each day
|
||||||
|
for day := 0; day < usageDays; day++ {
|
||||||
|
dayTime := startTime.AddDate(0, 0, day)
|
||||||
|
|
||||||
|
// More traffic on weekdays
|
||||||
|
isWeekday := dayTime.Weekday() >= time.Monday && dayTime.Weekday() <= time.Friday
|
||||||
|
baseCount := 50
|
||||||
|
if isWeekday {
|
||||||
|
baseCount = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
// Gradual growth trend
|
||||||
|
growthFactor := 1.0 + float64(day)/float64(usageDays)*0.5
|
||||||
|
samplesPerDay := int(float64(baseCount) * growthFactor)
|
||||||
|
|
||||||
|
for i := 0; i < samplesPerDay; i++ {
|
||||||
|
// Pick random master
|
||||||
|
master := masters[g.rng.Intn(len(masters))]
|
||||||
|
|
||||||
|
// Pick random key for this master
|
||||||
|
masterKeys := keys[master.ID]
|
||||||
|
var keyID uint
|
||||||
|
if len(masterKeys) > 0 {
|
||||||
|
keyID = masterKeys[g.rng.Intn(len(masterKeys))].ID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pick random group
|
||||||
|
group := groups[g.rng.Intn(len(groups))]
|
||||||
|
|
||||||
|
// Pick random model from group
|
||||||
|
var modelName string
|
||||||
|
if len(group.Models) > 0 {
|
||||||
|
modelName = group.Models[g.rng.Intn(len(group.Models))]
|
||||||
|
} else {
|
||||||
|
modelName = "gpt-4o"
|
||||||
|
}
|
||||||
|
|
||||||
|
// Random timestamp within the day
|
||||||
|
hour := g.rng.Intn(24)
|
||||||
|
minute := g.rng.Intn(60)
|
||||||
|
second := g.rng.Intn(60)
|
||||||
|
timestamp := time.Date(
|
||||||
|
dayTime.Year(), dayTime.Month(), dayTime.Day(),
|
||||||
|
hour, minute, second, 0, time.UTC,
|
||||||
|
)
|
||||||
|
|
||||||
|
// Generate realistic metrics
|
||||||
|
statusCode := 200
|
||||||
|
errorMsg := ""
|
||||||
|
|
||||||
|
// 2-5% error rate
|
||||||
|
if g.rng.Float64() < 0.03 {
|
||||||
|
errorCodes := []int{400, 401, 429, 500, 503}
|
||||||
|
statusCode = errorCodes[g.rng.Intn(len(errorCodes))]
|
||||||
|
errorMsgs := []string{
|
||||||
|
"rate limit exceeded",
|
||||||
|
"invalid request",
|
||||||
|
"authentication failed",
|
||||||
|
"internal server error",
|
||||||
|
"service unavailable",
|
||||||
|
}
|
||||||
|
errorMsg = errorMsgs[g.rng.Intn(len(errorMsgs))]
|
||||||
|
}
|
||||||
|
|
||||||
|
// Realistic latency (50ms - 5000ms, with most around 200-500ms)
|
||||||
|
latencyMs := int64(50 + g.rng.ExpFloat64()*300)
|
||||||
|
if latencyMs > 5000 {
|
||||||
|
latencyMs = 5000
|
||||||
|
}
|
||||||
|
|
||||||
|
// Realistic token counts
|
||||||
|
tokensIn := int64(100 + g.rng.Intn(2000))
|
||||||
|
tokensOut := int64(50 + g.rng.Intn(1000))
|
||||||
|
|
||||||
|
// Request/response sizes
|
||||||
|
requestSize := tokensIn * 4 // ~4 bytes per token
|
||||||
|
responseSize := tokensOut * 4
|
||||||
|
|
||||||
|
// Random client IP
|
||||||
|
clientIP := fmt.Sprintf("192.168.%d.%d", g.rng.Intn(256), g.rng.Intn(256))
|
||||||
|
|
||||||
|
_ = timestamp // TODO: need to set created_at somehow
|
||||||
|
|
||||||
|
result = append(result, LogRequest{
|
||||||
|
Group: master.Group,
|
||||||
|
MasterID: master.ID,
|
||||||
|
KeyID: keyID,
|
||||||
|
ModelName: modelName,
|
||||||
|
ProviderID: group.ID,
|
||||||
|
ProviderType: group.Type,
|
||||||
|
ProviderName: group.Name,
|
||||||
|
StatusCode: statusCode,
|
||||||
|
LatencyMs: latencyMs,
|
||||||
|
TokensIn: tokensIn,
|
||||||
|
TokensOut: tokensOut,
|
||||||
|
ErrorMessage: errorMsg,
|
||||||
|
ClientIP: clientIP,
|
||||||
|
RequestSize: requestSize,
|
||||||
|
ResponseSize: responseSize,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Helper functions ---
|
||||||
|
|
||||||
|
func (g *Generator) randomString(length int) string {
|
||||||
|
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
|
||||||
|
b := make([]byte, length)
|
||||||
|
for i := range b {
|
||||||
|
b[i] = charset[g.rng.Intn(len(charset))]
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
|
// HasSeederTag checks if a string contains the seeder tag
|
||||||
|
func HasSeederTag(s, tag string) bool {
|
||||||
|
return strings.Contains(s, tag)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExtractSeederTag extracts the seeder tag from a string
|
||||||
|
func ExtractSeederTag(s string) string {
|
||||||
|
if idx := strings.Index(s, "seeder:"); idx != -1 {
|
||||||
|
end := strings.Index(s[idx:], "]")
|
||||||
|
if end == -1 {
|
||||||
|
end = len(s) - idx
|
||||||
|
}
|
||||||
|
return s[idx : idx+end]
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
287
cmd/seeder/main.go
Normal file
287
cmd/seeder/main.go
Normal file
@@ -0,0 +1,287 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"math/rand"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/spf13/cobra"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Config holds seeder configuration
|
||||||
|
type Config struct {
|
||||||
|
BaseURL string
|
||||||
|
AdminToken string
|
||||||
|
Profile string
|
||||||
|
Seed int64
|
||||||
|
UsageDays int
|
||||||
|
Env string
|
||||||
|
Reset bool
|
||||||
|
DryRun bool
|
||||||
|
Force bool
|
||||||
|
Verbose bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProfileConfig defines entity counts for a profile
|
||||||
|
type ProfileConfig struct {
|
||||||
|
Namespaces int
|
||||||
|
ProviderGroups int
|
||||||
|
ProvidersPerGroup int
|
||||||
|
Models int
|
||||||
|
Bindings int
|
||||||
|
Masters int
|
||||||
|
KeysPerMaster int
|
||||||
|
UsageDays int
|
||||||
|
}
|
||||||
|
|
||||||
|
var profiles = map[string]ProfileConfig{
|
||||||
|
"demo": {
|
||||||
|
Namespaces: 2,
|
||||||
|
ProviderGroups: 2,
|
||||||
|
ProvidersPerGroup: 2,
|
||||||
|
Models: 5,
|
||||||
|
Bindings: 4,
|
||||||
|
Masters: 2,
|
||||||
|
KeysPerMaster: 3,
|
||||||
|
UsageDays: 7,
|
||||||
|
},
|
||||||
|
"load": {
|
||||||
|
Namespaces: 5,
|
||||||
|
ProviderGroups: 10,
|
||||||
|
ProvidersPerGroup: 3,
|
||||||
|
Models: 30,
|
||||||
|
Bindings: 50,
|
||||||
|
Masters: 50,
|
||||||
|
KeysPerMaster: 2,
|
||||||
|
UsageDays: 30,
|
||||||
|
},
|
||||||
|
"dev": {
|
||||||
|
Namespaces: 3,
|
||||||
|
ProviderGroups: 5,
|
||||||
|
ProvidersPerGroup: 3,
|
||||||
|
Models: 20,
|
||||||
|
Bindings: 15,
|
||||||
|
Masters: 10,
|
||||||
|
KeysPerMaster: 2,
|
||||||
|
UsageDays: 14,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
var cfg Config
|
||||||
|
|
||||||
|
rootCmd := &cobra.Command{
|
||||||
|
Use: "seeder",
|
||||||
|
Short: "EZ-API Demo Data Seeder",
|
||||||
|
Long: `A CLI tool to seed demo data into EZ-API Control Plane.
|
||||||
|
|
||||||
|
Creates namespaces, provider groups, providers, models, bindings,
|
||||||
|
masters, API keys, and usage samples for frontend development and demos.`,
|
||||||
|
RunE: func(cmd *cobra.Command, args []string) error {
|
||||||
|
return run(cfg)
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bind flags
|
||||||
|
rootCmd.Flags().StringVar(&cfg.BaseURL, "base-url", "http://localhost:8080", "Control Plane API base URL")
|
||||||
|
rootCmd.Flags().StringVar(&cfg.AdminToken, "admin-token", "", "Admin authentication token (required)")
|
||||||
|
rootCmd.Flags().StringVar(&cfg.Profile, "profile", "demo", "Dataset profile: demo, load, dev")
|
||||||
|
rootCmd.Flags().Int64Var(&cfg.Seed, "seed", 0, "Fixed random seed for reproducibility (0 = random)")
|
||||||
|
rootCmd.Flags().IntVar(&cfg.UsageDays, "usage-days", 0, "Time window for usage samples (0 = profile default)")
|
||||||
|
rootCmd.Flags().StringVar(&cfg.Env, "env", "dev", "Environment: dev, test, staging, production")
|
||||||
|
rootCmd.Flags().BoolVar(&cfg.Reset, "reset", false, "Remove previous seeder data before recreating")
|
||||||
|
rootCmd.Flags().BoolVar(&cfg.DryRun, "dry-run", false, "Print actions without executing write API calls")
|
||||||
|
rootCmd.Flags().BoolVar(&cfg.Force, "force", false, "Allow running against production")
|
||||||
|
rootCmd.Flags().BoolVar(&cfg.Verbose, "verbose", false, "Print detailed progress")
|
||||||
|
|
||||||
|
// Bind environment variables
|
||||||
|
viper.SetEnvPrefix("SEEDER")
|
||||||
|
viper.AutomaticEnv()
|
||||||
|
viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
|
||||||
|
|
||||||
|
// Override from env if not set via flag
|
||||||
|
rootCmd.PreRunE = func(cmd *cobra.Command, args []string) error {
|
||||||
|
if !cmd.Flags().Changed("base-url") && viper.IsSet("BASE_URL") {
|
||||||
|
cfg.BaseURL = viper.GetString("BASE_URL")
|
||||||
|
}
|
||||||
|
if !cmd.Flags().Changed("admin-token") && viper.IsSet("ADMIN_TOKEN") {
|
||||||
|
cfg.AdminToken = viper.GetString("ADMIN_TOKEN")
|
||||||
|
}
|
||||||
|
if !cmd.Flags().Changed("profile") && viper.IsSet("PROFILE") {
|
||||||
|
cfg.Profile = viper.GetString("PROFILE")
|
||||||
|
}
|
||||||
|
if !cmd.Flags().Changed("seed") && viper.IsSet("SEED") {
|
||||||
|
cfg.Seed = viper.GetInt64("SEED")
|
||||||
|
}
|
||||||
|
if !cmd.Flags().Changed("usage-days") && viper.IsSet("USAGE_DAYS") {
|
||||||
|
cfg.UsageDays = viper.GetInt("USAGE_DAYS")
|
||||||
|
}
|
||||||
|
if !cmd.Flags().Changed("env") && viper.IsSet("ENV") {
|
||||||
|
cfg.Env = viper.GetString("ENV")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := rootCmd.Execute(); err != nil {
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func run(cfg Config) error {
|
||||||
|
// Validate required fields
|
||||||
|
if cfg.AdminToken == "" {
|
||||||
|
return fmt.Errorf("admin-token is required (use --admin-token or SEEDER_ADMIN_TOKEN)")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate profile
|
||||||
|
profile, ok := profiles[cfg.Profile]
|
||||||
|
if !ok {
|
||||||
|
return fmt.Errorf("invalid profile: %s (valid: demo, load, dev)", cfg.Profile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Environment guard
|
||||||
|
if isProduction(cfg.Env) && !cfg.Force {
|
||||||
|
return fmt.Errorf("refusing to run against production environment (env=%s). Use --force to override", cfg.Env)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize seed
|
||||||
|
if cfg.Seed == 0 {
|
||||||
|
cfg.Seed = time.Now().UnixNano()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Use profile default for usage days if not specified
|
||||||
|
if cfg.UsageDays == 0 {
|
||||||
|
cfg.UsageDays = profile.UsageDays
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create seeder tag
|
||||||
|
seederTag := fmt.Sprintf("seeder:%s:seed-%d", cfg.Profile, cfg.Seed)
|
||||||
|
|
||||||
|
// Print banner
|
||||||
|
printBanner(cfg, seederTag)
|
||||||
|
|
||||||
|
// Warning for production
|
||||||
|
if isProduction(cfg.Env) && cfg.Force {
|
||||||
|
fmt.Println("\n⚠️ WARNING: Running against PRODUCTION environment!")
|
||||||
|
fmt.Println(" Press Ctrl+C within 5 seconds to abort...")
|
||||||
|
time.Sleep(5 * time.Second)
|
||||||
|
fmt.Println(" Proceeding...")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create seeder instance
|
||||||
|
seeder := NewSeeder(cfg, profile, seederTag)
|
||||||
|
|
||||||
|
// Run seeding
|
||||||
|
return seeder.Run()
|
||||||
|
}
|
||||||
|
|
||||||
|
func isProduction(env string) bool {
|
||||||
|
env = strings.ToLower(env)
|
||||||
|
return env == "production" || env == "prod"
|
||||||
|
}
|
||||||
|
|
||||||
|
func printBanner(cfg Config, seederTag string) {
|
||||||
|
mode := "create"
|
||||||
|
if cfg.Reset {
|
||||||
|
mode = "reset + create"
|
||||||
|
}
|
||||||
|
if cfg.DryRun {
|
||||||
|
mode += " (dry-run)"
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println()
|
||||||
|
fmt.Println("EZ-API Demo Data Seeder")
|
||||||
|
fmt.Println("=======================")
|
||||||
|
fmt.Printf("Profile: %s\n", cfg.Profile)
|
||||||
|
fmt.Printf("Seed: %d\n", cfg.Seed)
|
||||||
|
fmt.Printf("Target: %s\n", cfg.BaseURL)
|
||||||
|
fmt.Printf("Environment: %s\n", cfg.Env)
|
||||||
|
fmt.Printf("Usage Window: %d days\n", cfg.UsageDays)
|
||||||
|
fmt.Printf("Mode: %s\n", mode)
|
||||||
|
fmt.Printf("Seeder Tag: %s\n", seederTag)
|
||||||
|
fmt.Println()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seeder handles the seeding process
|
||||||
|
type Seeder struct {
|
||||||
|
cfg Config
|
||||||
|
profile ProfileConfig
|
||||||
|
seederTag string
|
||||||
|
client *Client
|
||||||
|
rng *rand.Rand
|
||||||
|
summary Summary
|
||||||
|
}
|
||||||
|
|
||||||
|
// Summary tracks created/skipped counts
|
||||||
|
type Summary struct {
|
||||||
|
Namespaces CountPair
|
||||||
|
ProviderGroups CountPair
|
||||||
|
Providers CountPair
|
||||||
|
Models CountPair
|
||||||
|
Bindings CountPair
|
||||||
|
Masters CountPair
|
||||||
|
Keys CountPair
|
||||||
|
UsageSamples int
|
||||||
|
}
|
||||||
|
|
||||||
|
type CountPair struct {
|
||||||
|
Created int
|
||||||
|
Skipped int
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewSeeder creates a new seeder instance
|
||||||
|
func NewSeeder(cfg Config, profile ProfileConfig, seederTag string) *Seeder {
|
||||||
|
return &Seeder{
|
||||||
|
cfg: cfg,
|
||||||
|
profile: profile,
|
||||||
|
seederTag: seederTag,
|
||||||
|
client: NewClient(cfg.BaseURL, cfg.AdminToken, cfg.DryRun, cfg.Verbose),
|
||||||
|
rng: rand.New(rand.NewSource(cfg.Seed)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Run executes the seeding process
|
||||||
|
func (s *Seeder) Run() error {
|
||||||
|
steps := []struct {
|
||||||
|
name string
|
||||||
|
fn func() error
|
||||||
|
}{
|
||||||
|
{"Creating namespaces", s.seedNamespaces},
|
||||||
|
{"Creating provider groups", s.seedProviderGroups},
|
||||||
|
{"Creating providers", s.seedProviders},
|
||||||
|
{"Creating models", s.seedModels},
|
||||||
|
{"Creating bindings", s.seedBindings},
|
||||||
|
{"Creating masters", s.seedMasters},
|
||||||
|
{"Creating API keys", s.seedKeys},
|
||||||
|
{"Generating usage samples", s.seedUsageSamples},
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, step := range steps {
|
||||||
|
fmt.Printf("[%d/%d] %s...\n", i+1, len(steps), step.name)
|
||||||
|
if err := step.fn(); err != nil {
|
||||||
|
return fmt.Errorf("%s: %w", step.name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.printSummary()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) printSummary() {
|
||||||
|
fmt.Println("\nDone! Summary:")
|
||||||
|
fmt.Printf(" Namespaces: %d created, %d skipped\n", s.summary.Namespaces.Created, s.summary.Namespaces.Skipped)
|
||||||
|
fmt.Printf(" Provider Groups: %d created, %d skipped\n", s.summary.ProviderGroups.Created, s.summary.ProviderGroups.Skipped)
|
||||||
|
fmt.Printf(" Providers: %d created, %d skipped\n", s.summary.Providers.Created, s.summary.Providers.Skipped)
|
||||||
|
fmt.Printf(" Models: %d created, %d skipped\n", s.summary.Models.Created, s.summary.Models.Skipped)
|
||||||
|
fmt.Printf(" Bindings: %d created, %d skipped\n", s.summary.Bindings.Created, s.summary.Bindings.Skipped)
|
||||||
|
fmt.Printf(" Masters: %d created, %d skipped\n", s.summary.Masters.Created, s.summary.Masters.Skipped)
|
||||||
|
fmt.Printf(" API Keys: %d created, %d skipped\n", s.summary.Keys.Created, s.summary.Keys.Skipped)
|
||||||
|
fmt.Printf(" Usage Samples: %d entries\n", s.summary.UsageSamples)
|
||||||
|
fmt.Printf("\nSeeder tag: %s\n", s.seederTag)
|
||||||
|
fmt.Printf("\nDashboard should now show data at:\n %s/admin/dashboard\n", s.cfg.BaseURL)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Seed methods are implemented in seeder.go
|
||||||
433
cmd/seeder/seeder.go
Normal file
433
cmd/seeder/seeder.go
Normal file
@@ -0,0 +1,433 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// seededNamespaces stores created namespace names for later use
|
||||||
|
var seededNamespaces []string
|
||||||
|
|
||||||
|
// seededProviderGroups stores created provider groups for later use
|
||||||
|
var seededProviderGroups []ProviderGroupResponse
|
||||||
|
|
||||||
|
// seededMasters stores created masters for later use
|
||||||
|
var seededMasters []MasterResponse
|
||||||
|
|
||||||
|
// seededKeys stores created keys per master for usage samples
|
||||||
|
var seededKeys map[uint][]KeyResponse
|
||||||
|
|
||||||
|
func (s *Seeder) seedNamespaces() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
namespaces := gen.GenerateNamespaces(s.profile.Namespaces)
|
||||||
|
|
||||||
|
// Get existing namespaces
|
||||||
|
existing, err := s.client.ListNamespaces()
|
||||||
|
if err != nil {
|
||||||
|
if s.cfg.DryRun {
|
||||||
|
// In dry-run mode, continue without checking existing
|
||||||
|
existing = []NamespaceResponse{}
|
||||||
|
fmt.Printf(" [DRY-RUN] Unable to list existing namespaces, proceeding anyway\n")
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("list namespaces: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
existingMap := make(map[string]NamespaceResponse)
|
||||||
|
for _, ns := range existing {
|
||||||
|
existingMap[ns.Name] = ns
|
||||||
|
}
|
||||||
|
|
||||||
|
seededNamespaces = make([]string, 0, len(namespaces))
|
||||||
|
|
||||||
|
for _, ns := range namespaces {
|
||||||
|
if existingNs, exists := existingMap[ns.Name]; exists {
|
||||||
|
// Check if we should reset
|
||||||
|
if s.cfg.Reset && HasSeederTag(existingNs.Description, s.seederTag) {
|
||||||
|
if err := s.client.DeleteNamespace(existingNs.ID); err != nil {
|
||||||
|
return fmt.Errorf("delete namespace %s: %w", ns.Name, err)
|
||||||
|
}
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ✗ %s (deleted for reset)\n", ns.Name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||||
|
}
|
||||||
|
s.summary.Namespaces.Skipped++
|
||||||
|
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create namespace
|
||||||
|
created, err := s.client.CreateNamespace(ns)
|
||||||
|
if err != nil {
|
||||||
|
// Check if it's a conflict (already exists)
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", ns.Name)
|
||||||
|
}
|
||||||
|
s.summary.Namespaces.Skipped++
|
||||||
|
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create namespace %s: %w", ns.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s (created)\n", ns.Name)
|
||||||
|
}
|
||||||
|
s.summary.Namespaces.Created++
|
||||||
|
if created != nil {
|
||||||
|
seededNamespaces = append(seededNamespaces, created.Name)
|
||||||
|
} else {
|
||||||
|
seededNamespaces = append(seededNamespaces, ns.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedProviderGroups() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
groups := gen.GenerateProviderGroups(s.profile.ProviderGroups)
|
||||||
|
|
||||||
|
// Get existing provider groups
|
||||||
|
existing, err := s.client.ListProviderGroups()
|
||||||
|
if err != nil {
|
||||||
|
if s.cfg.DryRun {
|
||||||
|
existing = []ProviderGroupResponse{}
|
||||||
|
fmt.Printf(" [DRY-RUN] Unable to list existing provider groups, proceeding anyway\n")
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("list provider groups: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
existingMap := make(map[string]ProviderGroupResponse)
|
||||||
|
for _, g := range existing {
|
||||||
|
existingMap[g.Name] = g
|
||||||
|
}
|
||||||
|
|
||||||
|
seededProviderGroups = make([]ProviderGroupResponse, 0, len(groups))
|
||||||
|
|
||||||
|
for _, group := range groups {
|
||||||
|
if existingGroup, exists := existingMap[group.Name]; exists {
|
||||||
|
// Check if we should reset (check name prefix for seeder tag)
|
||||||
|
if s.cfg.Reset && strings.HasPrefix(group.Name, "demo-") {
|
||||||
|
if err := s.client.DeleteProviderGroup(existingGroup.ID); err != nil {
|
||||||
|
return fmt.Errorf("delete provider group %s: %w", group.Name, err)
|
||||||
|
}
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ✗ %s (deleted for reset)\n", group.Name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||||
|
}
|
||||||
|
s.summary.ProviderGroups.Skipped++
|
||||||
|
seededProviderGroups = append(seededProviderGroups, existingGroup)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create provider group
|
||||||
|
created, err := s.client.CreateProviderGroup(group)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", group.Name)
|
||||||
|
}
|
||||||
|
s.summary.ProviderGroups.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create provider group %s: %w", group.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s (created)\n", group.Name)
|
||||||
|
}
|
||||||
|
s.summary.ProviderGroups.Created++
|
||||||
|
if created != nil {
|
||||||
|
seededProviderGroups = append(seededProviderGroups, *created)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedProviders() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
|
||||||
|
for _, group := range seededProviderGroups {
|
||||||
|
providers := gen.GenerateProviders(group.ID, group.Name, s.profile.ProvidersPerGroup)
|
||||||
|
|
||||||
|
createdCount := 0
|
||||||
|
for _, provider := range providers {
|
||||||
|
_, err := s.client.CreateAPIKey(provider)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
s.summary.Providers.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create provider for group %s: %w", group.Name, err)
|
||||||
|
}
|
||||||
|
s.summary.Providers.Created++
|
||||||
|
createdCount++
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s: %d providers created\n", group.Name, createdCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedModels() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
models := gen.GenerateModels(s.profile.Models)
|
||||||
|
|
||||||
|
// Get existing models
|
||||||
|
existing, err := s.client.ListModels()
|
||||||
|
if err != nil {
|
||||||
|
if s.cfg.DryRun {
|
||||||
|
existing = []ModelResponse{}
|
||||||
|
fmt.Printf(" [DRY-RUN] Unable to list existing models, proceeding anyway\n")
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("list models: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
existingMap := make(map[string]ModelResponse)
|
||||||
|
for _, m := range existing {
|
||||||
|
existingMap[m.Name] = m
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, model := range models {
|
||||||
|
if existingModel, exists := existingMap[model.Name]; exists {
|
||||||
|
if s.cfg.Reset {
|
||||||
|
if err := s.client.DeleteModel(existingModel.ID); err != nil {
|
||||||
|
return fmt.Errorf("delete model %s: %w", model.Name, err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||||
|
}
|
||||||
|
s.summary.Models.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := s.client.CreateModel(model)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", model.Name)
|
||||||
|
}
|
||||||
|
s.summary.Models.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create model %s: %w", model.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s (created)\n", model.Name)
|
||||||
|
}
|
||||||
|
s.summary.Models.Created++
|
||||||
|
}
|
||||||
|
|
||||||
|
if !s.cfg.Verbose && !s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %d models created\n", s.summary.Models.Created)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedBindings() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
bindings := gen.GenerateBindings(seededNamespaces, seededProviderGroups, s.profile.Bindings)
|
||||||
|
|
||||||
|
// Get existing bindings
|
||||||
|
existing, err := s.client.ListBindings()
|
||||||
|
if err != nil {
|
||||||
|
if s.cfg.DryRun {
|
||||||
|
existing = []BindingResponse{}
|
||||||
|
fmt.Printf(" [DRY-RUN] Unable to list existing bindings, proceeding anyway\n")
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("list bindings: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a map of existing bindings by composite key
|
||||||
|
existingMap := make(map[string]BindingResponse)
|
||||||
|
for _, b := range existing {
|
||||||
|
key := fmt.Sprintf("%s:%s:%d", b.Namespace, b.PublicModel, b.GroupID)
|
||||||
|
existingMap[key] = b
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, binding := range bindings {
|
||||||
|
key := fmt.Sprintf("%s:%s:%d", binding.Namespace, binding.PublicModel, binding.GroupID)
|
||||||
|
if existingBinding, exists := existingMap[key]; exists {
|
||||||
|
if s.cfg.Reset {
|
||||||
|
if err := s.client.DeleteBinding(existingBinding.ID); err != nil {
|
||||||
|
return fmt.Errorf("delete binding: %w", err)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
s.summary.Bindings.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := s.client.CreateBinding(binding)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
s.summary.Bindings.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create binding: %w", err)
|
||||||
|
}
|
||||||
|
s.summary.Bindings.Created++
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %d bindings created\n", s.summary.Bindings.Created)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedMasters() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
masters := gen.GenerateMasters(s.profile.Masters)
|
||||||
|
|
||||||
|
// Get existing masters
|
||||||
|
existing, err := s.client.ListMasters()
|
||||||
|
if err != nil {
|
||||||
|
if s.cfg.DryRun {
|
||||||
|
existing = []MasterResponse{}
|
||||||
|
fmt.Printf(" [DRY-RUN] Unable to list existing masters, proceeding anyway\n")
|
||||||
|
} else {
|
||||||
|
return fmt.Errorf("list masters: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
existingMap := make(map[string]MasterResponse)
|
||||||
|
for _, m := range existing {
|
||||||
|
existingMap[m.Name] = m
|
||||||
|
}
|
||||||
|
|
||||||
|
seededMasters = make([]MasterResponse, 0, len(masters))
|
||||||
|
|
||||||
|
for _, master := range masters {
|
||||||
|
if existingMaster, exists := existingMap[master.Name]; exists {
|
||||||
|
if s.cfg.Reset && strings.HasSuffix(master.Name, "-demo") {
|
||||||
|
if err := s.client.DeleteMaster(existingMaster.ID); err != nil {
|
||||||
|
return fmt.Errorf("delete master %s: %w", master.Name, err)
|
||||||
|
}
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ✗ %s (deleted for reset)\n", master.Name)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||||
|
}
|
||||||
|
s.summary.Masters.Skipped++
|
||||||
|
seededMasters = append(seededMasters, existingMaster)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
created, err := s.client.CreateMaster(master)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
if s.cfg.Verbose {
|
||||||
|
fmt.Printf(" ○ %s (exists, skipped)\n", master.Name)
|
||||||
|
}
|
||||||
|
s.summary.Masters.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create master %s: %w", master.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s (created)\n", master.Name)
|
||||||
|
}
|
||||||
|
s.summary.Masters.Created++
|
||||||
|
if created != nil {
|
||||||
|
seededMasters = append(seededMasters, *created)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedKeys() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
|
||||||
|
seededKeys = make(map[uint][]KeyResponse)
|
||||||
|
|
||||||
|
for _, master := range seededMasters {
|
||||||
|
keys := gen.GenerateKeys(s.profile.KeysPerMaster)
|
||||||
|
|
||||||
|
masterKeys := make([]KeyResponse, 0, len(keys))
|
||||||
|
createdCount := 0
|
||||||
|
|
||||||
|
for _, key := range keys {
|
||||||
|
created, err := s.client.CreateKey(master.ID, key)
|
||||||
|
if err != nil {
|
||||||
|
if apiErr, ok := err.(*APIError); ok && apiErr.IsConflict() {
|
||||||
|
s.summary.Keys.Skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return fmt.Errorf("create key for master %s: %w", master.Name, err)
|
||||||
|
}
|
||||||
|
s.summary.Keys.Created++
|
||||||
|
createdCount++
|
||||||
|
if created != nil {
|
||||||
|
masterKeys = append(masterKeys, *created)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
seededKeys[master.ID] = masterKeys
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %s: %d keys created\n", master.Name, createdCount)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Seeder) seedUsageSamples() error {
|
||||||
|
gen := NewGenerator(s.rng, s.seederTag, s.cfg.Profile)
|
||||||
|
|
||||||
|
logs := gen.GenerateUsageSamples(seededMasters, seededKeys, seededProviderGroups, s.cfg.UsageDays)
|
||||||
|
|
||||||
|
if len(logs) == 0 {
|
||||||
|
fmt.Printf(" ○ No masters or groups to generate samples for\n")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch insert logs
|
||||||
|
batchSize := 100
|
||||||
|
for i := 0; i < len(logs); i += batchSize {
|
||||||
|
end := i + batchSize
|
||||||
|
if end > len(logs) {
|
||||||
|
end = len(logs)
|
||||||
|
}
|
||||||
|
batch := logs[i:end]
|
||||||
|
|
||||||
|
if err := s.client.CreateLogsBatch(batch); err != nil {
|
||||||
|
return fmt.Errorf("create logs batch: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
s.summary.UsageSamples = len(logs)
|
||||||
|
|
||||||
|
if s.cfg.Verbose || s.cfg.DryRun {
|
||||||
|
fmt.Printf(" ✓ %d days of data generated (%d log entries)\n", s.cfg.UsageDays, len(logs))
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
2
go.mod
2
go.mod
@@ -44,6 +44,7 @@ require (
|
|||||||
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
|
||||||
github.com/goccy/go-json v0.10.2 // indirect
|
github.com/goccy/go-json v0.10.2 // indirect
|
||||||
github.com/goccy/go-yaml v1.18.0 // indirect
|
github.com/goccy/go-yaml v1.18.0 // indirect
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||||
github.com/jackc/pgx/v5 v5.6.0 // indirect
|
github.com/jackc/pgx/v5 v5.6.0 // indirect
|
||||||
@@ -66,6 +67,7 @@ require (
|
|||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
|
||||||
github.com/spf13/afero v1.15.0 // indirect
|
github.com/spf13/afero v1.15.0 // indirect
|
||||||
github.com/spf13/cast v1.10.0 // indirect
|
github.com/spf13/cast v1.10.0 // indirect
|
||||||
|
github.com/spf13/cobra v1.10.2 // indirect
|
||||||
github.com/spf13/pflag v1.0.10 // indirect
|
github.com/spf13/pflag v1.0.10 // indirect
|
||||||
github.com/subosito/gotenv v1.6.0 // indirect
|
github.com/subosito/gotenv v1.6.0 // indirect
|
||||||
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
|
||||||
|
|||||||
7
go.sum
7
go.sum
@@ -15,6 +15,7 @@ github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
|
|||||||
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
|
||||||
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
|
||||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||||
|
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
@@ -79,6 +80,8 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
|
|||||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
|
||||||
|
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||||
@@ -132,6 +135,7 @@ github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7
|
|||||||
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
|
||||||
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
|
||||||
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
|
||||||
|
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||||
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
|
||||||
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik=
|
||||||
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw=
|
||||||
@@ -140,6 +144,9 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
|
|||||||
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
|
||||||
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
|
||||||
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
|
||||||
|
github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
|
||||||
|
github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
|
||||||
|
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
|
||||||
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
|
||||||
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
|
||||||
|
|||||||
Reference in New Issue
Block a user