diff --git a/cmd/logql-analyzer/main.go b/cmd/logql-analyzer/main.go index beed1226709d4..6b4ceb8a53ca6 100644 --- a/cmd/logql-analyzer/main.go +++ b/cmd/logql-analyzer/main.go @@ -11,7 +11,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/grafana/loki/v3/pkg/logqlanalyzer" - "github.com/grafana/loki/v3/pkg/sizing" util_log "github.com/grafana/loki/v3/pkg/util/log" ) @@ -48,12 +47,6 @@ func createServer(cfg server.Config, logger log.Logger) (*server.Server, error) s.HTTP.Use(logqlanalyzer.CorsMiddleware()) s.HTTP.Handle("/api/logql-analyze", &logqlanalyzer.LogQLAnalyzeHandler{}).Methods(http.MethodPost, http.MethodOptions) - sizingHandler := sizing.NewHandler(log.With(logger, "component", "sizing")) - - s.HTTP.Handle("/api/sizing/helm", http.HandlerFunc(sizingHandler.GenerateHelmValues)).Methods(http.MethodGet, http.MethodOptions) - s.HTTP.Handle("/api/sizing/nodes", http.HandlerFunc(sizingHandler.Nodes)).Methods(http.MethodGet, http.MethodOptions) - s.HTTP.Handle("/api/sizing/cluster", http.HandlerFunc(sizingHandler.Cluster)).Methods(http.MethodGet, http.MethodOptions) - s.HTTP.HandleFunc("/ready", func(w http.ResponseWriter, _ *http.Request) { http.Error(w, "ready", http.StatusOK) }).Methods(http.MethodGet) diff --git a/pkg/sizing/algorithm.go b/pkg/sizing/algorithm.go deleted file mode 100644 index dcd2d6bb89911..0000000000000 --- a/pkg/sizing/algorithm.go +++ /dev/null @@ -1,98 +0,0 @@ -package sizing - -import ( - "math" -) - -type ClusterSize struct { - TotalNodes int - TotalReadReplicas int - TotalWriteReplicas int - TotalCoresRequest float64 - TotalMemoryRequest int - - expectedMaxReadThroughputBytesSec float64 - expectedMaxIngestBytesDay float64 -} - -type QueryPerf string - -const ( - Basic QueryPerf = "basic" - Super QueryPerf = "super" -) - -func calculateClusterSize(nt NodeType, bytesDayIngest float64, qperf QueryPerf) ClusterSize { - - // 1 Petabyte per day is maximum. We use decimal prefix https://en.wikipedia.org/wiki/Binary_prefix - bytesDayIngest = math.Min(bytesDayIngest, 1e15) - bytesSecondIngest := bytesDayIngest / 86400 - numWriteReplicasNeeded := math.Ceil(bytesSecondIngest / nt.writePod.rateBytesSecond) - - // High availability requires at least 3 replicas. - numWriteReplicasNeeded = math.Max(3, numWriteReplicasNeeded) - - //Hack based on current 4-1 mem to cpu ratio and base machine w/ 4 cores and 1 write/read - writeReplicasPerNode := float64(nt.cores / 4) - fullyWritePackedNodes := math.Floor(numWriteReplicasNeeded / writeReplicasPerNode) - replicasOnLastNode := math.Mod(numWriteReplicasNeeded, writeReplicasPerNode) - - coresOnLastNode := 0.0 - if replicasOnLastNode > 0.0 { - coresOnLastNode = math.Max(float64(nt.cores)-replicasOnLastNode*nt.writePod.cpuRequest, 0.0) - } - - nodesNeededForWrites := math.Ceil(numWriteReplicasNeeded / writeReplicasPerNode) - - // Hack based on packing 1 read and 1 write per node - readReplicasPerNode := writeReplicasPerNode - readReplicasOnFullyPackedWriteNodes := readReplicasPerNode * fullyWritePackedNodes - readReplicasOnPartiallyPackedWriteNodes := math.Floor(coresOnLastNode / nt.readPod.cpuRequest) - - // Required read replicase without considering required query performance. - baselineReadReplicas := readReplicasOnFullyPackedWriteNodes + readReplicasOnPartiallyPackedWriteNodes - - scaleUp := 0.25 - additionalReadReplicas := 0.0 - if qperf != Basic { - additionalReadReplicas = baselineReadReplicas * scaleUp - } - - readReplicasPerEmptyNode := math.Floor(float64(nt.cores) / nt.readPod.cpuRequest) - additionalNodesNeededForReads := additionalReadReplicas / readReplicasPerEmptyNode - - actualNodesAddedForReads := calculateActualReadNodes(additionalNodesNeededForReads) - actualReadReplicasAdded := actualNodesAddedForReads * readReplicasPerEmptyNode - - totalReadReplicas := actualReadReplicasAdded + baselineReadReplicas - - // High availability requires at least 3 replicas. - totalReadReplicas = math.Max(3, totalReadReplicas) - - totalReadThroughputBytesSec := totalReadReplicas * nt.readPod.rateBytesSecond - - totalNodesNeeded := nodesNeededForWrites + actualNodesAddedForReads - totalCoresRequest := numWriteReplicasNeeded*nt.writePod.cpuRequest + totalReadReplicas*nt.readPod.cpuRequest - totalMemoryRequest := numWriteReplicasNeeded*float64(nt.writePod.memoryRequest) + totalReadReplicas*float64(nt.readPod.memoryRequest) - - return ClusterSize{ - TotalNodes: int(totalNodesNeeded), - TotalReadReplicas: int(totalReadReplicas), - TotalWriteReplicas: int(numWriteReplicasNeeded), - TotalCoresRequest: totalCoresRequest, - TotalMemoryRequest: int(totalMemoryRequest), - - expectedMaxReadThroughputBytesSec: totalReadThroughputBytesSec, - expectedMaxIngestBytesDay: (nt.writePod.rateBytesSecond * numWriteReplicasNeeded) * 86400, - } -} - -func calculateActualReadNodes(additionalNodesNeededForReads float64) float64 { - if additionalNodesNeededForReads == 0.0 { - return 0 - } - if 0.0 < additionalNodesNeededForReads && additionalNodesNeededForReads < 1.0 { - return 1 - } - return math.Floor(additionalNodesNeededForReads) -} diff --git a/pkg/sizing/algorithm_test.go b/pkg/sizing/algorithm_test.go deleted file mode 100644 index 4ef36ad274c71..0000000000000 --- a/pkg/sizing/algorithm_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package sizing - -import ( - "testing" - "testing/quick" - - "github.com/stretchr/testify/require" -) - -func Test_Algorithm(t *testing.T) { - f := func(ingest float64) bool { - if ingest < 0 { - ingest = -ingest - } - postiveReplicas := true - for _, cloud := range NodeTypesByProvider { - for _, node := range cloud { - size := calculateClusterSize(node, ingest, Basic) - postiveReplicas = size.TotalNodes > 0.0 && size.TotalReadReplicas > 0.0 && size.TotalWriteReplicas > 0.0 - require.Truef(t, postiveReplicas, "Cluster size was empty: ingest=%d cluster=%v node=%v", ingest, size, node) - require.InDelta(t, size.TotalReadReplicas, size.TotalWriteReplicas, 5.0, "Replicas have different sizes: ingest=%d node=%s", ingest, node.name) - - size = calculateClusterSize(node, ingest, Super) - postiveReplicas = size.TotalNodes > 0.0 && size.TotalReadReplicas > 0.0 && size.TotalWriteReplicas > 0.0 - require.Truef(t, postiveReplicas, "Cluster size was empty: ingest=%d cluster=%v node=%v", ingest, size, node) - } - } - - return postiveReplicas - } - - if err := quick.Check(f, nil); err != nil { - t.Error(err) - } - - // Sanity check for 1TB/Day - size := calculateClusterSize(NodeTypesByProvider["AWS"]["t2.xlarge"], 1e12, Basic) - require.Equalf(t, 4, size.TotalNodes, "given ingest=1PB/Day totla nodes must be big") -} - -func Test_CoresNodeInvariant(t *testing.T) { - for _, queryPerformance := range []QueryPerf{Basic, Super} { - for _, ingest := range []float64{30, 300, 1000, 2000} { - for _, cloud := range NodeTypesByProvider { - for _, node := range cloud { - size := calculateClusterSize(node, ingest, queryPerformance) - require.LessOrEqualf(t, size.TotalCoresRequest, float64(size.TotalNodes*node.cores), "given ingest=%d node=%s total cores must be less than available cores", ingest, node.name) - } - } - } - } -} - -func Test_MinimumReplicas(t *testing.T) { - for _, queryPerformance := range []QueryPerf{Basic, Super} { - for _, ingest := range []float64{1, 1000} { - for _, cloud := range NodeTypesByProvider { - for _, node := range cloud { - size := calculateClusterSize(node, ingest, queryPerformance) - require.GreaterOrEqual(t, size.TotalReadReplicas, 3) - require.GreaterOrEqual(t, size.TotalWriteReplicas, 3) - } - } - } - } -} diff --git a/pkg/sizing/helm.go b/pkg/sizing/helm.go deleted file mode 100644 index 8e988c8b55b2e..0000000000000 --- a/pkg/sizing/helm.go +++ /dev/null @@ -1,91 +0,0 @@ -package sizing - -type Pod struct { - Replicas int `json:"replicas"` - Rate int `json:"rate"` - CPU struct { - Request float64 `json:"request"` - Limit float64 `json:"limit"` - } `json:"cpu"` - Memory struct { - Request int `json:"request"` - Limit int `json:"limit"` - } `json:"memory"` -} - -type Loki struct { - AuthEnabled bool `json:"auth_enabled"` -} - -type Read struct { - Replicas int `json:"replicas"` - Resources Resources `json:"resources"` -} - -type Write struct { - Replicas int `json:"replicas"` - Resources Resources `json:"resources"` -} - -type Resources struct { - Requests struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - } `json:"requests"` - Limits struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - } `json:"limits"` -} - -type Values struct { - Loki Loki `json:"loki"` - Read Read `json:"read"` - Write Write `json:"write"` -} - -func constructHelmValues(cluster ClusterSize, nodeType NodeType) Values { - return Values{ - Loki: Loki{ - AuthEnabled: false, - }, - Read: Read{ - Replicas: cluster.TotalReadReplicas, - Resources: Resources{ - Requests: struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - }{ - CPU: nodeType.readPod.cpuRequest, - Memory: nodeType.readPod.memoryRequest, - }, - Limits: struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - }{ - CPU: nodeType.readPod.cpuLimit, - Memory: nodeType.readPod.memoryLimit, - }, - }, - }, - Write: Write{ - Replicas: cluster.TotalWriteReplicas, - Resources: Resources{ - Requests: struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - }{ - CPU: nodeType.writePod.cpuRequest, - Memory: nodeType.writePod.memoryRequest, - }, - Limits: struct { - CPU float64 `json:"cpu"` - Memory int `json:"memory"` - }{ - CPU: nodeType.writePod.cpuLimit, - Memory: nodeType.writePod.memoryLimit, - }, - }, - }, - } -} diff --git a/pkg/sizing/http.go b/pkg/sizing/http.go deleted file mode 100644 index f0b755af32ead..0000000000000 --- a/pkg/sizing/http.go +++ /dev/null @@ -1,120 +0,0 @@ -package sizing - -import ( - "encoding/json" - "fmt" - "net/http" - "strconv" - "strings" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "gopkg.in/yaml.v2" -) - -type Message struct { - NodeType NodeType - Ingest int - Retention int - QueryPerformance QueryPerf -} - -func decodeMesage(req *http.Request, msg *Message) error { - var err error - types := strings.Split(req.FormValue("node-type"), " - ") - nodeTypes, ok := NodeTypesByProvider[types[0]] - if !ok { - return fmt.Errorf("unknown cloud provider %s", types[0]) - } - msg.NodeType, ok = nodeTypes[types[1]] - if !ok { - return fmt.Errorf("unknown node type %s", types[1]) - } - - msg.Ingest, err = strconv.Atoi(req.FormValue("ingest")) - if err != nil { - return fmt.Errorf("cannot read ingest: %w", err) - } - - msg.Retention, err = strconv.Atoi(req.FormValue("retention")) - if err != nil { - return fmt.Errorf("cannot read retention: %w", err) - } - - msg.QueryPerformance = QueryPerf(strings.ToLower(req.FormValue("queryperf"))) - - return nil -} - -// Handler defines the REST API of the sizing tool. -type Handler struct { - logger log.Logger -} - -func NewHandler(logger log.Logger) *Handler { - return &Handler{logger: logger} -} - -func (h *Handler) GenerateHelmValues(w http.ResponseWriter, req *http.Request) { - - var msg Message - err := decodeMesage(req, &msg) - if err != nil { - level.Error(h.logger).Log("error", err) - h.respondError(w, err) - return - } - - w.Header().Set("Content-Type", "application/x-yaml; charset=utf-8") - - cluster := calculateClusterSize(msg.NodeType, float64(msg.Ingest), msg.QueryPerformance) - helm := constructHelmValues(cluster, msg.NodeType) - - enc := yaml.NewEncoder(w) - err = enc.Encode(helm) - if err != nil { - level.Error(h.logger).Log("msg", "could not encode Helm Chart values", "error", err) - } -} - -func (h *Handler) Nodes(w http.ResponseWriter, _ *http.Request) { - var nodes []string - for cloud, n := range NodeTypesByProvider { - for nodeType := range n { - nodes = append(nodes, fmt.Sprintf("%s - %s", cloud, nodeType)) - } - } - - w.Header().Set("Content-Type", "application/json") - err := json.NewEncoder(w).Encode(nodes) - if err != nil { - level.Error(h.logger).Log("msg", "could not encode node values", "error", err) - } -} - -func (h *Handler) respondError(w http.ResponseWriter, err error) { - w.WriteHeader(http.StatusBadRequest) - _, err = w.Write([]byte(fmt.Sprintf("error: %q", err))) - if err != nil { - level.Error(h.logger).Log("msg", "could not write error message", "error", err) - } -} - -func (h *Handler) Cluster(w http.ResponseWriter, req *http.Request) { - var msg Message - - err := decodeMesage(req, &msg) - if err != nil { - level.Error(h.logger).Log("error", err) - h.respondError(w, err) - return - } - - cluster := calculateClusterSize(msg.NodeType, float64(msg.Ingest), msg.QueryPerformance) - - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(cluster) - if err != nil { - level.Error(h.logger).Log("msg", "could not encode cluster size", "error", err) - } -} diff --git a/pkg/sizing/node.go b/pkg/sizing/node.go deleted file mode 100644 index 79899539ae972..0000000000000 --- a/pkg/sizing/node.go +++ /dev/null @@ -1,98 +0,0 @@ -package sizing - -type NodeType struct { - name string - cores int - memoryGB int - readPod NodePod - writePod NodePod -} - -type NodePod struct { - cpuRequest float64 - cpuLimit float64 // Or null - memoryRequest int - memoryLimit int - rateBytesSecond float64 -} - -var StandardWrite = NodePod{ - cpuRequest: 1, - cpuLimit: 2, - memoryRequest: 6, - memoryLimit: 12, - rateBytesSecond: 3 * 1024 * 1024, -} - -var StandardRead = NodePod{ - cpuRequest: 3, - cpuLimit: 3, // Undefined TODO: Is this a bug - memoryRequest: 6, - memoryLimit: 8, - rateBytesSecond: 768 * 1024 * 1024, -} - -var NodeTypesByProvider = map[string]map[string]NodeType{ - "AWS": { - "t2.xlarge": { - name: "t2.xlarge", - cores: 4, - memoryGB: 16, - readPod: StandardRead, - writePod: StandardWrite, - }, - "t2.2xlarge": { - name: "t2.2xlarge", - cores: 8, - memoryGB: 32, - readPod: StandardRead, - writePod: StandardWrite, - }, - }, - "GCP": { - "e2-standard-4": { - name: "e2-standard-4", - cores: 4, - memoryGB: 16, - readPod: StandardRead, - writePod: StandardWrite, - }, - "e2-standard-8": { - name: "e2-standard-8", - cores: 8, - memoryGB: 32, - readPod: StandardRead, - writePod: StandardWrite, - }, - "e2-standard-16": { - name: "e2-standard-16", - cores: 16, - memoryGB: 64, - readPod: StandardRead, - writePod: StandardWrite, - }, - }, - "OVHcloud": { - "b2-30": { - name: "b2-30", - cores: 8, - memoryGB: 30, - readPod: StandardRead, - writePod: StandardWrite, - }, - "b2-60": { - name: "b2-60", - cores: 16, - memoryGB: 60, - readPod: StandardRead, - writePod: StandardWrite, - }, - "b2-120": { - name: "b2-120", - cores: 32, - memoryGB: 120, - readPod: StandardRead, - writePod: StandardWrite, - }, - }, -}