From 9a3fdc7368c3382a6005103c9f2596f5a42f3941 Mon Sep 17 00:00:00 2001 From: Carmine Di Monaco Date: Fri, 22 Dec 2023 17:01:18 +0100 Subject: [PATCH] Charts Without Grafana (#2049) * Wip on TimeSeriesLineChart component * Time Series line chart without border on dot * Add support for multiple time series with different colors in TimeSeries chart * Tooltip text alignment in TimeSeriesLineChart * Npm run format and lint * TimeSeriesLineChart test * TimeSeriesLineChart stories with live updates example * Fix TimeSeriesLineChart zoom fetching * Add prometheus and node exporter as optional docker compose profile * Add ChartTimeSeries data structure to charts context * Add HostCpuChart data structure to charts context * Add HostDataFetcher Behaviour to charts context * Wip implementation of host charts using prometheus * Add example agentID labels to prometheus dev conf * Chart time series structure with utc datetime as timestamp * Add agentID to host chart query * Prometheus in docker compose start with predefined seed * Wip on chart context entrypoint * Add busy irqs to host chart cpu * Prometheus api, implementing host data fetcher * Add tests for prometheus chart integration * prometheus api chart testing * Removed dedicated profile to prometheus and and node-exporter * Refactored usage of deftype in charts context * Charts context entrypoint performs charts queries * Prometheus api handles no result case * Add tests for Charts module * Prometheus data fetching of hosts memory data * Add host memory chart to Charts context * Mix credo fix * Charts context check if host exists before querying for charts * Chart controller with host cpu chart action * Host memory chart action in ChartController * Change agentID label in prometheus config with a fixture agent * Move TimeSeriesChart component in common module * fix relabel config in prometheus dev config * Changed default step parameter for query range to 60 seconds * Charts modules now uses Datetime module for intervals * Chart controller with ISO8601 timestamps * TimeSeriesLineChart supports 6 series * TimeSeriesLineChart component supports plain date object * Wip on host chart component * Removed grafanaPublicURL reference from assets * remove grafana from docker compose * Remove grafana from elixir code * Removed grafana dashboard from priv * Remove grafana from docs and readme * Leftovers grafana removal from elixir code * Custom scale and legend in TimeSeriesLineChart * Fix TimeSeriesLineChart stories * Chart controller fix mispell * fix chart * Npm run lint * npm format * Skip prometheus integration test, ci not ready * mix credo * mix format * Fix HostDetails test, mock chart request * Remove example from timestamp field in ChartTimeSeries openapi schema * mix format * Disabled chart controller test for missing prometheus in ci * host details e2e stub chart request * prettier on e2e test * npm format * support for api charts mocking globally in e2e * HostChart guard for chartjs chartref when data is missing * Fix charts request stub * Host cpu chart values are normalized for the numbers of cpus * Fix HostDetails cpu chart scale * Add prometheus integration tests * Remove charts request stubbing in e2e tests * Host memory chart 3 hour interval * Review feedbacks * Addressing feedbacks --- README.md | 2 +- .../TimeSeriesLineChart.jsx | 209 + .../TimeSeriesLineChart.stories.jsx | 199 + .../TimeSeriesLineChart.test.jsx | 98 + assets/js/common/TimeSeriesLineChart/index.js | 3 + assets/js/pages/HostDetailsPage/HostChart.jsx | 94 + .../js/pages/HostDetailsPage/HostDetails.jsx | 43 +- .../HostDetailsPage/HostDetails.stories.jsx | 8 - .../HostDetailsPage/HostDetails.test.jsx | 9 + .../pages/HostDetailsPage/HostDetailsPage.jsx | 4 - assets/package-lock.json | 315 +- assets/package.json | 5 + config/config.exs | 7 +- config/runtime.exs | 6 - .../prometheus/prometheus_entrypoint.sh | 10 + .../prometheus/prometheus_snap.tar.xz | Bin 0 -> 970544 bytes docker-compose.yaml | 36 +- guides/development/environment_variables.md | 5 +- guides/development/hack_on_the_trento.md | 2 +- guides/monitoring/monitoring.md | 9 +- lib/mix/tasks/init_grafana_dashboards.ex | 24 - lib/trento/charts.ex | 90 + lib/trento/charts/chart_time_series.ex | 16 + lib/trento/charts/chart_time_series_sample.ex | 14 + lib/trento/charts/hosts/host_cpu_chart.ex | 27 + lib/trento/charts/hosts/host_data_fetcher.ex | 43 + lib/trento/charts/hosts/host_memory_chart.ex | 25 + lib/trento/hosts.ex | 9 + lib/trento/infrastructure/grafana/grafana.ex | 112 - .../prometheus/adapter/prometheus_api.ex | 124 + .../prometheus/chart_integration.ex | 23 + .../prometheus/prometheus_samples.ex | 13 + lib/trento/release.ex | 7 - lib/trento_web/controllers/page_controller.ex | 2 - .../controllers/v1/chart_controller.ex | 100 + lib/trento_web/openapi/v1/schema/chart.ex | 75 + lib/trento_web/router.ex | 3 + lib/trento_web/templates/page/index.html.heex | 1 - lib/trento_web/views/v1/chart_view.ex | 51 + mix.exs | 2 +- priv/data/grafana/node_exporter.json | 12963 ---------------- prometheus-dev-config.yml | 14 + test/trento/charts_test.exs | 139 + .../adapters/prometheus_api_test.exs | 211 + .../prometheus/chart_integration_test.exs | 41 + .../controllers/v1/chart_controller_test.exs | 112 + 46 files changed, 2088 insertions(+), 13217 deletions(-) create mode 100644 assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.jsx create mode 100644 assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.stories.jsx create mode 100644 assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.test.jsx create mode 100644 assets/js/common/TimeSeriesLineChart/index.js create mode 100644 assets/js/pages/HostDetailsPage/HostChart.jsx create mode 100755 container_fixtures/prometheus/prometheus_entrypoint.sh create mode 100644 container_fixtures/prometheus/prometheus_snap.tar.xz delete mode 100644 lib/mix/tasks/init_grafana_dashboards.ex create mode 100644 lib/trento/charts.ex create mode 100644 lib/trento/charts/chart_time_series.ex create mode 100644 lib/trento/charts/chart_time_series_sample.ex create mode 100644 lib/trento/charts/hosts/host_cpu_chart.ex create mode 100644 lib/trento/charts/hosts/host_data_fetcher.ex create mode 100644 lib/trento/charts/hosts/host_memory_chart.ex delete mode 100644 lib/trento/infrastructure/grafana/grafana.ex create mode 100644 lib/trento/infrastructure/prometheus/chart_integration.ex create mode 100644 lib/trento/infrastructure/prometheus/prometheus_samples.ex create mode 100644 lib/trento_web/controllers/v1/chart_controller.ex create mode 100644 lib/trento_web/openapi/v1/schema/chart.ex create mode 100644 lib/trento_web/views/v1/chart_view.ex delete mode 100644 priv/data/grafana/node_exporter.json create mode 100644 prometheus-dev-config.yml create mode 100644 test/trento/charts_test.exs create mode 100644 test/trento/infrastructure/prometheus/chart_integration_test.exs create mode 100644 test/trento_web/controllers/v1/chart_controller_test.exs diff --git a/README.md b/README.md index 5f047e1866..886337f69b 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ Here's a non-comprehensive list of the capabilities provided by the [Trento Web] ## Monitoring It is important in critical business systems to have access to relevant information about _how things are going_. -Currently, Trento provides basic integration with **Grafana** and **Prometheus**. +Currently, Trento provides basic integration with **Prometheus**. See [related documentation](./guides/monitoring/monitoring.md) for more information. diff --git a/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.jsx b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.jsx new file mode 100644 index 0000000000..b2721b8d57 --- /dev/null +++ b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.jsx @@ -0,0 +1,209 @@ +import React, { useEffect, useState } from 'react'; +import { + Chart as ChartJS, + Legend, + LinearScale, + LineElement, + LogarithmicScale, + PointElement, + TimeScale, + Tooltip, +} from 'chart.js'; +import ZoomPlugin from 'chartjs-plugin-zoom'; +import classNames from 'classnames'; +import 'chartjs-adapter-date-fns'; +import { Line } from 'react-chartjs-2'; + +const AVAILABLE_COLORS = [ + { + line: '#A5EED4', + point: '#58CF9B', + }, + { + line: '#FEF08A', + point: '#FACC15', + }, + { + line: '#FECACA', + point: '#F87171', + }, + { + line: '#BFDBFE', + point: '#60A5FA', + }, + { + line: '#E5E7EB', + point: '#9CA3AF', + }, + { + line: '#004DCF', + point: '#1273DE', + }, +]; + +ChartJS.register( + LinearScale, + PointElement, + LineElement, + TimeScale, + Legend, + LogarithmicScale, + Tooltip, + ZoomPlugin +); + +function TimeSeriesLineChart({ + chartRef, + chartWrapperClassNames, + className, + datasets, + end, + onIntervalChange, + start, + title, + yAxisMaxValue, + yAxisLabelFormatter = (value) => value, + yAxisScaleType = 'linear', +}) { + const onZoomChange = ({ + chart: { + scales: { x }, + }, + }) => onIntervalChange(x.min, x.max); + + const [chartDatasets, setChartDatasets] = useState([]); + const [zoomOptions, setZoomOptions] = useState({ + limits: { + x: { min: 'original', max: 'original', minRange: 60 * 1000 }, + }, + zoom: { + wheel: { + enabled: true, + }, + drag: { + enabled: true, + }, + pan: { + enabled: false, + }, + mode: 'x', + onZoomComplete: onZoomChange, + // Prevent zoom reset on legend change, https://github.com/chartjs/chartjs-plugin-zoom/issues/256#issuecomment-1826812558 + onZoomStart: (e) => + e.point.x > e.chart.chartArea.left && + e.point.x < e.chart.chartArea.right && + e.point.y > e.chart.chartArea.top && + e.point.y < e.chart.chartArea.bottom, + }, + }); + + useEffect(() => { + const newDatasets = datasets.map((d, i) => ({ + label: d.name, + data: d.timeFrames.map(({ time, value }) => ({ + x: time, + y: value, + })), + borderColor: AVAILABLE_COLORS[i].line, + pointBackgroundColor: AVAILABLE_COLORS[i].point, + pointBorderWidth: 0, + pointRadius: 1.8, + pointHoverRadius: 8, + })); + + setChartDatasets(newDatasets); + }, [datasets]); + + useEffect(() => { + setZoomOptions((currentOptions) => ({ + ...currentOptions, + limits: { + x: { + ...currentOptions.x, + min: start.getTime(), + max: end.getTime(), + }, + }, + })); + }, [start, end]); + + const scales = { + x: { + position: 'bottom', + min: start, + max: end, + type: 'time', + ticks: { + autoSkip: true, + autoSkipPadding: 50, + maxRotation: 0, + }, + time: { + displayFormats: { + hour: 'HH:mm', + minute: 'HH:mm', + second: 'HH:mm:ss', + }, + }, + }, + y: { + type: yAxisScaleType, + position: 'left', + max: yAxisMaxValue, + ticks: { + callback: yAxisLabelFormatter, + }, + }, + }; + + const options = { + scales, + responsive: true, + maintainAspectRatio: false, + plugins: { + tooltip: { + titleAlign: 'center', + bodyAlign: 'center', + footerAlign: 'center', + displayColors: 'false', + }, + legend: { + position: 'bottom', + }, + }, + }; + + if (datasets.length > 6) { + throw new Error( + 'TimeSeriesLineChart component supports a maximum of 6 datasets' + ); + } + + return ( +
+

{title}

+
+ +
+
+ ); +} + +export default TimeSeriesLineChart; diff --git a/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.stories.jsx b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.stories.jsx new file mode 100644 index 0000000000..fcdb185c2d --- /dev/null +++ b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.stories.jsx @@ -0,0 +1,199 @@ +/* eslint-disable no-console */ +import React, { useEffect, useRef, useState } from 'react'; +import { faker } from '@faker-js/faker'; +import { + addMinutes, + addSeconds, + eachMinuteOfInterval, + subHours, + subMinutes, +} from 'date-fns'; +import TimeSeriesLineChart from './TimeSeriesLineChart'; + +const now = new Date(); + +/** + * TimeSeriesLineChart is a specialized line chart. + * + * It should be used for timeseries representation. + * On the X we have time, on the Y the value associated with time. + * + * The input data are a series of object, containing a timestamp and a value associated to the timestamp. + * + * Timestamp are plain js Date object, values are numbers. + * + * The timeseries chart offers zoom and sub-selection of time intervals, a prop function is called with the selected range. + * + */ +export default { + title: 'Components/TimeSeriesLineChart', + component: TimeSeriesLineChart, + argTypes: { + title: { + description: 'Chart title', + control: { type: 'text' }, + table: { + type: { summary: 'string' }, + }, + }, + start: { + description: 'Start of the time interval, as Date object', + control: { type: 'date' }, + }, + end: { + description: 'End of the time interval, as Date object', + control: { type: 'date' }, + }, + datasets: { + description: + 'Array of datasets, a series of objects containing a value and a Date', + control: { type: 'array' }, + }, + chartWrapperClassNames: { + description: 'Classnames for the parent of chart canvas', + }, + className: { + description: 'Classname for component', + }, + onIntervalChange: { + description: + 'Callback called with the selected/zoomed time interval, check the console', + }, + }, +}; + +const defaultTimeframes = eachMinuteOfInterval( + { start: subHours(now, 5), end: now }, + { step: 2 } +); + +const buildDatasets = (timeFrames) => [ + { + name: 'Busy User', + timeFrames: timeFrames.map((t) => ({ + time: t, + value: faker.number.float({ min: 0, max: 100 }), + })), + }, + { + name: 'Busy System', + timeFrames: timeFrames.map((t) => ({ + time: t, + value: faker.number.float({ min: 20, max: 30 }), + })), + }, + { + name: 'I/O', + timeFrames: timeFrames.map((t) => ({ + time: t, + value: faker.number.float({ min: 0, max: 50 }), + })), + }, + { + name: 'Other Cpu Value', + timeFrames: timeFrames.map((t) => ({ + time: t, + value: faker.number.float({ min: 0, max: 20 }), + })), + }, + { + name: 'Another Value', + timeFrames: timeFrames.map((t) => ({ + time: t, + value: faker.number.float({ min: 10, max: 200 }), + })), + }, +]; + +export const Default = { + args: { + title: 'CPU', + start: subHours(now, 5), + end: now, + onIntervalChange: (start, end) => + // eslint-disable-next-line no-console + console.log(`Interval changed, start ${start} - end ${end}`), + datasets: buildDatasets(defaultTimeframes), + }, +}; + +function ChartUpdaterWrapper(props) { + const defaultStart = new Date(); + const chartRef = useRef(null); + + const initialTimeFrames = eachMinuteOfInterval({ + start: subMinutes(defaultStart, 5), + end: addSeconds(now, 10), + }); + + const [datasets, setDasatets] = useState(buildDatasets(initialTimeFrames)); + const [interval, setChartInterval] = useState({ + start: subMinutes(defaultStart, 5), + end: addMinutes(defaultStart, 1), + }); + + const handleIntervalChange = (start, end) => { + console.log(`Interval changed, start ${start} - end ${end}`); + }; + + useEffect(() => { + const chartJsInstance = chartRef.current; + + setInterval(() => { + const timeNow = new Date(); + const newInterval = { + start: subMinutes(timeNow, 5), + end: addMinutes(timeNow, 1), + }; + const newFetchInterval = eachMinuteOfInterval({ + start: timeNow, + end: addSeconds(timeNow, 10), + }); + + const newDatasets = datasets.map((d) => ({ + ...d, + timeFrames: [ + ...d.timeFrames, + ...newFetchInterval.map((t) => ({ + time: t, + value: faker.number.float({ min: 10, max: 200 }), + })), + ], + })); + + console.log('zoom level', chartJsInstance.getZoomLevel()); + if (chartJsInstance.getZoomLevel() < 3.5) { + console.log('Updating interval, chart is not zoomed'); + setChartInterval(newInterval); + setDasatets(newDatasets); + console.log('Data updated!'); + } else { + console.log( + 'Chart zoomed too much, skipping updating', + chartJsInstance.getZoomLevel() + ); + } + }, 20000); + }, []); + + return ( + + ); +} + +/** + * Data updates every 20s + */ +export const WithUpdates = { + args: { + title: 'CPU', + }, + render: (args) => , +}; diff --git a/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.test.jsx b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.test.jsx new file mode 100644 index 0000000000..a82ba39438 --- /dev/null +++ b/assets/js/common/TimeSeriesLineChart/TimeSeriesLineChart.test.jsx @@ -0,0 +1,98 @@ +import React from 'react'; + +import { render } from '@testing-library/react'; +import TimeSeriesLineChart from './TimeSeriesLineChart'; +import '@testing-library/jest-dom'; + +describe('TimeSeriesLineChart component', () => { + it('should raise an error if the datasets are more then 5', () => { + const datasets = [ + { + name: 'Data 1', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 2', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 3', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 4', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 5', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 6', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 7', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + ]; + + const timeNow = new Date(); + + expect(() => + render( +
+ {}} + /> +
+ ) + ).toThrow('TimeSeriesLineChart component supports a maximum of 6 datasets'); + }); + + it('should render with the appropriate props', () => { + const datasets = [ + { + name: 'Data 1', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 2', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 3', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 4', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + { + name: 'Data 5', + timeFrames: [{ time: new Date(), value: 0.0 }], + }, + ]; + + const timeNow = new Date(); + + const { container } = render( +
+ {}} + /> +
+ ); + + const canvas = container.querySelector('canvas'); + expect(canvas).toBeDefined(); + }); +}); diff --git a/assets/js/common/TimeSeriesLineChart/index.js b/assets/js/common/TimeSeriesLineChart/index.js new file mode 100644 index 0000000000..069a0ae0b8 --- /dev/null +++ b/assets/js/common/TimeSeriesLineChart/index.js @@ -0,0 +1,3 @@ +import TimeSeriesLineChart from './TimeSeriesLineChart'; + +export default TimeSeriesLineChart; diff --git a/assets/js/pages/HostDetailsPage/HostChart.jsx b/assets/js/pages/HostDetailsPage/HostChart.jsx new file mode 100644 index 0000000000..fb7531ea5c --- /dev/null +++ b/assets/js/pages/HostDetailsPage/HostChart.jsx @@ -0,0 +1,94 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { get } from '@lib/network'; +import { addMinutes, parseISO, subMinutes } from 'date-fns'; +import TimeSeriesLineChart from '@common/TimeSeriesLineChart/TimeSeriesLineChart'; + +function HostChart({ + hostId, + chartId, + chartTitle, + yAxisFormatter, + yAxisScaleType, + yAxisMaxValue, + startInterval = subMinutes(new Date(), 10), + endInterval = new Date(), + updateFrequency = 30000, +}) { + const [chartStartInterval, setChartStartInterval] = useState(startInterval); + const [chartEndInterval, setChartEndInterval] = useState(endInterval); + const [chartData, setChartData] = useState([]); + + const chartRef = useRef(null); + const startIntervalRef = useRef(startInterval); + const endIntervalRef = useRef(endInterval); + + useEffect(() => { + startIntervalRef.current = chartStartInterval; + }, [chartStartInterval]); + + useEffect(() => { + endIntervalRef.current = chartEndInterval; + }, [chartEndInterval]); + + const fetchApiData = async (start, end) => { + const { data: chartApiData } = await get( + `charts/hosts/${hostId}/${chartId}?from=${start.toISOString()}&to=${end.toISOString()}` + ); + + const updatedChartData = Object.keys(chartApiData).map((chartKey) => ({ + name: chartApiData[chartKey].label, + timeFrames: chartApiData[chartKey].series.map((frame) => ({ + time: parseISO(frame.timestamp), + value: frame.value, + })), + })); + + return updatedChartData; + }; + + useEffect(() => { + async function startDataFetching() { + setInterval(async () => { + const fetchInterval = { + start: addMinutes(startIntervalRef.current, 1), + end: addMinutes(endIntervalRef.current, 1), + }; + if (chartRef.current && chartRef.current.getZoomLevel() < 3.5) { + const updatedChartData = await fetchApiData( + fetchInterval.start, + fetchInterval.end + ); + + setChartData(updatedChartData); + setChartStartInterval(fetchInterval.start); + setChartEndInterval(fetchInterval.end); + } + }, updateFrequency); + + const initialData = await fetchApiData(startInterval, endInterval); + setChartData(initialData); + } + startDataFetching(); + }, []); + + if (chartData.length === 0) { + return
Data is loading
; + } + + return ( + {}} + /> + ); +} + +export default HostChart; diff --git a/assets/js/pages/HostDetailsPage/HostDetails.jsx b/assets/js/pages/HostDetailsPage/HostDetails.jsx index 11fdceeeb8..4c5d48f6da 100644 --- a/assets/js/pages/HostDetailsPage/HostDetails.jsx +++ b/assets/js/pages/HostDetailsPage/HostDetails.jsx @@ -12,6 +12,7 @@ import PageHeader from '@common/PageHeader'; import Table from '@common/Table'; import Tooltip from '@common/Tooltip'; import WarningBanner from '@common/Banners/WarningBanner'; +import { subHours } from 'date-fns'; import SuseLogo from '@static/suse_logo.svg'; @@ -23,19 +24,28 @@ import HostSummary from './HostSummary'; import ProviderDetails from './ProviderDetails'; import SaptuneSummary from './SaptuneSummary'; import StatusPill from './StatusPill'; +import HostChart from './HostChart'; import { subscriptionsTableConfiguration, sapInstancesTableConfiguration, } from './tableConfigs'; +function formatBytes(bytes, decimals = 2) { + if (bytes === 0) return '0 Bytes'; + const k = 1024; + const dm = decimals < 0 ? 0 : decimals; + const sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB']; + const i = Math.floor(Math.log(bytes) / Math.log(k)); + return `${parseFloat((bytes / k ** i).toFixed(dm))} ${sizes[i]}`; +} + function HostDetails({ agentVersion, cluster, deregisterable, deregistering, exportersStatus = {}, - grafanaPublicUrl, heartbeat, hostID, hostname, @@ -81,6 +91,8 @@ function HostDetails({ const lastExecutionLoading = get(lastExecution, 'loading'); const lastExecutionError = get(lastExecution, 'error'); + const timeNow = new Date(); + return ( <> -
-