diff --git a/docs/docset.yml b/docs/docset.yml new file mode 100644 index 00000000000..2b82d085fc1 --- /dev/null +++ b/docs/docset.yml @@ -0,0 +1,508 @@ +project: 'Integration developer guide' +exclude: + - ci_pipelines.md + - dashboard_guidelines.md + - definitions.md + - developer_tsdb_migration_guidelines.md + - developer_workflow_bug_fix_older_package_version.md + - developer_workflow_design_build_test_integration.md + - developer_workflow_fleet_ui.md + - documentation_guidelines.md + - ecs@mappings_migration_guide.md + - fine_tune_integration.md + - generic_guidelines.md + - how_to_test_new_indexing_features.md + - import_from_beats.md + - subobjects_adoption_guide.md + - testing_and_validation.md + - tips_for_building_integrations.md +cross_links: + - docs-content + - ecs + - elasticsearch +toc: + - toc: extend +subs: + ref: "https://www.elastic.co/guide/en/elasticsearch/reference/current" + ref-bare: "https://www.elastic.co/guide/en/elasticsearch/reference" + ref-8x: "https://www.elastic.co/guide/en/elasticsearch/reference/8.1" + ref-80: "https://www.elastic.co/guide/en/elasticsearch/reference/8.0" + ref-7x: "https://www.elastic.co/guide/en/elasticsearch/reference/7.17" + ref-70: "https://www.elastic.co/guide/en/elasticsearch/reference/7.0" + ref-60: "https://www.elastic.co/guide/en/elasticsearch/reference/6.0" + ref-64: "https://www.elastic.co/guide/en/elasticsearch/reference/6.4" + xpack-ref: "https://www.elastic.co/guide/en/x-pack/6.2" + logstash-ref: "https://www.elastic.co/guide/en/logstash/current" + kibana-ref: "https://www.elastic.co/guide/en/kibana/current" + kibana-ref-all: "https://www.elastic.co/guide/en/kibana" + beats-ref-root: "https://www.elastic.co/guide/en/beats" + beats-ref: "https://www.elastic.co/guide/en/beats/libbeat/current" + beats-ref-60: "https://www.elastic.co/guide/en/beats/libbeat/6.0" + beats-ref-63: "https://www.elastic.co/guide/en/beats/libbeat/6.3" + beats-devguide: "https://www.elastic.co/guide/en/beats/devguide/current" + auditbeat-ref: "https://www.elastic.co/guide/en/beats/auditbeat/current" + packetbeat-ref: "https://www.elastic.co/guide/en/beats/packetbeat/current" + metricbeat-ref: "https://www.elastic.co/guide/en/beats/metricbeat/current" + filebeat-ref: "https://www.elastic.co/guide/en/beats/filebeat/current" + functionbeat-ref: "https://www.elastic.co/guide/en/beats/functionbeat/current" + winlogbeat-ref: "https://www.elastic.co/guide/en/beats/winlogbeat/current" + heartbeat-ref: "https://www.elastic.co/guide/en/beats/heartbeat/current" + journalbeat-ref: "https://www.elastic.co/guide/en/beats/journalbeat/current" + ingest-guide: "https://www.elastic.co/guide/en/ingest/current" + fleet-guide: "https://www.elastic.co/guide/en/fleet/current" + apm-guide-ref: "https://www.elastic.co/guide/en/apm/guide/current" + apm-guide-7x: "https://www.elastic.co/guide/en/apm/guide/7.17" + apm-app-ref: "https://www.elastic.co/guide/en/kibana/current" + apm-agents-ref: "https://www.elastic.co/guide/en/apm/agent" + apm-android-ref: "https://www.elastic.co/guide/en/apm/agent/android/current" + apm-py-ref: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-py-ref-3x: "https://www.elastic.co/guide/en/apm/agent/python/3.x" + apm-node-ref-index: "https://www.elastic.co/guide/en/apm/agent/nodejs" + apm-node-ref: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-node-ref-1x: "https://www.elastic.co/guide/en/apm/agent/nodejs/1.x" + apm-rum-ref: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-dotnet-ref: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref: "https://www.elastic.co/guide/en/apm/agent/php/current" + apm-ios-ref: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-lambda-ref: "https://www.elastic.co/guide/en/apm/lambda/current" + apm-attacher-ref: "https://www.elastic.co/guide/en/apm/attacher/current" + docker-logging-ref: "https://www.elastic.co/guide/en/beats/loggingplugin/current" + esf-ref: "https://www.elastic.co/guide/en/esf/current" + kinesis-firehose-ref: "https://www.elastic.co/guide/en/kinesis/{{kinesis_version}}" + estc-welcome-current: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions/current" + estc-welcome-all: "https://www.elastic.co/guide/en/starting-with-the-elasticsearch-platform-and-its-solutions" + hadoop-ref: "https://www.elastic.co/guide/en/elasticsearch/hadoop/current" + stack-ref: "https://www.elastic.co/guide/en/elastic-stack/current" + stack-ref-67: "https://www.elastic.co/guide/en/elastic-stack/6.7" + stack-ref-68: "https://www.elastic.co/guide/en/elastic-stack/6.8" + stack-ref-70: "https://www.elastic.co/guide/en/elastic-stack/7.0" + stack-ref-80: "https://www.elastic.co/guide/en/elastic-stack/8.0" + stack-ov: "https://www.elastic.co/guide/en/elastic-stack-overview/current" + stack-gs: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + stack-gs-current: "https://www.elastic.co/guide/en/elastic-stack-get-started/current" + javaclient: "https://www.elastic.co/guide/en/elasticsearch/client/java-api/current" + java-api-client: "https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current" + java-rest: "https://www.elastic.co/guide/en/elasticsearch/client/java-rest/current" + jsclient: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + jsclient-current: "https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current" + es-ruby-client: "https://www.elastic.co/guide/en/elasticsearch/client/ruby-api/current" + es-dotnet-client: "https://www.elastic.co/guide/en/elasticsearch/client/net-api/current" + es-php-client: "https://www.elastic.co/guide/en/elasticsearch/client/php-api/current" + es-python-client: "https://www.elastic.co/guide/en/elasticsearch/client/python-api/current" + defguide: "https://www.elastic.co/guide/en/elasticsearch/guide/2.x" + painless: "https://www.elastic.co/guide/en/elasticsearch/painless/current" + plugins: "https://www.elastic.co/guide/en/elasticsearch/plugins/current" + plugins-8x: "https://www.elastic.co/guide/en/elasticsearch/plugins/8.1" + plugins-7x: "https://www.elastic.co/guide/en/elasticsearch/plugins/7.17" + plugins-6x: "https://www.elastic.co/guide/en/elasticsearch/plugins/6.8" + glossary: "https://www.elastic.co/guide/en/elastic-stack-glossary/current" + upgrade_guide: "https://www.elastic.co/products/upgrade_guide" + blog-ref: "https://www.elastic.co/blog/" + curator-ref: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + curator-ref-current: "https://www.elastic.co/guide/en/elasticsearch/client/curator/current" + metrics-ref: "https://www.elastic.co/guide/en/metrics/current" + metrics-guide: "https://www.elastic.co/guide/en/metrics/guide/current" + logs-ref: "https://www.elastic.co/guide/en/logs/current" + logs-guide: "https://www.elastic.co/guide/en/logs/guide/current" + uptime-guide: "https://www.elastic.co/guide/en/uptime/current" + observability-guide: "https://www.elastic.co/guide/en/observability/current" + observability-guide-all: "https://www.elastic.co/guide/en/observability" + siem-guide: "https://www.elastic.co/guide/en/siem/guide/current" + security-guide: "https://www.elastic.co/guide/en/security/current" + security-guide-all: "https://www.elastic.co/guide/en/security" + endpoint-guide: "https://www.elastic.co/guide/en/endpoint/current" + sql-odbc: "https://www.elastic.co/guide/en/elasticsearch/sql-odbc/current" + ecs-ref: "https://www.elastic.co/guide/en/ecs/current" + ecs-logging-ref: "https://www.elastic.co/guide/en/ecs-logging/overview/current" + ecs-logging-go-logrus-ref: "https://www.elastic.co/guide/en/ecs-logging/go-logrus/current" + ecs-logging-go-zap-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-go-zerolog-ref: "https://www.elastic.co/guide/en/ecs-logging/go-zap/current" + ecs-logging-java-ref: "https://www.elastic.co/guide/en/ecs-logging/java/current" + ecs-logging-dotnet-ref: "https://www.elastic.co/guide/en/ecs-logging/dotnet/current" + ecs-logging-nodejs-ref: "https://www.elastic.co/guide/en/ecs-logging/nodejs/current" + ecs-logging-php-ref: "https://www.elastic.co/guide/en/ecs-logging/php/current" + ecs-logging-python-ref: "https://www.elastic.co/guide/en/ecs-logging/python/current" + ecs-logging-ruby-ref: "https://www.elastic.co/guide/en/ecs-logging/ruby/current" + ml-docs: "https://www.elastic.co/guide/en/machine-learning/current" + eland-docs: "https://www.elastic.co/guide/en/elasticsearch/client/eland/current" + eql-ref: "https://eql.readthedocs.io/en/latest/query-guide" + extendtrial: "https://www.elastic.co/trialextension" + wikipedia: "https://en.wikipedia.org/wiki" + forum: "https://discuss.elastic.co/" + xpack-forum: "https://discuss.elastic.co/c/50-x-pack" + security-forum: "https://discuss.elastic.co/c/x-pack/shield" + watcher-forum: "https://discuss.elastic.co/c/x-pack/watcher" + monitoring-forum: "https://discuss.elastic.co/c/x-pack/marvel" + graph-forum: "https://discuss.elastic.co/c/x-pack/graph" + apm-forum: "https://discuss.elastic.co/c/apm" + enterprise-search-ref: "https://www.elastic.co/guide/en/enterprise-search/current" + app-search-ref: "https://www.elastic.co/guide/en/app-search/current" + workplace-search-ref: "https://www.elastic.co/guide/en/workplace-search/current" + enterprise-search-node-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/enterprise-search-node/current" + enterprise-search-php-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/php/current" + enterprise-search-python-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/python/current" + enterprise-search-ruby-ref: "https://www.elastic.co/guide/en/enterprise-search-clients/ruby/current" + elastic-maps-service: "https://maps.elastic.co" + integrations-docs: "https://docs.elastic.co/en/integrations" + integrations-devguide: "https://www.elastic.co/guide/en/integrations-developer/current" + time-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#time-units" + byte-units: "https://www.elastic.co/guide/en/elasticsearch/reference/current/api-conventions.html#byte-units" + apm-py-ref-v: "https://www.elastic.co/guide/en/apm/agent/python/current" + apm-node-ref-v: "https://www.elastic.co/guide/en/apm/agent/nodejs/current" + apm-rum-ref-v: "https://www.elastic.co/guide/en/apm/agent/rum-js/current" + apm-ruby-ref-v: "https://www.elastic.co/guide/en/apm/agent/ruby/current" + apm-java-ref-v: "https://www.elastic.co/guide/en/apm/agent/java/current" + apm-go-ref-v: "https://www.elastic.co/guide/en/apm/agent/go/current" + apm-ios-ref-v: "https://www.elastic.co/guide/en/apm/agent/swift/current" + apm-dotnet-ref-v: "https://www.elastic.co/guide/en/apm/agent/dotnet/current" + apm-php-ref-v: "https://www.elastic.co/guide/en/apm/agent/php/current" + ecloud: "Elastic Cloud" + esf: "Elastic Serverless Forwarder" + ess: "Elasticsearch Service" + ece: "Elastic Cloud Enterprise" + eck: "Elastic Cloud on Kubernetes" + serverless-full: "Elastic Cloud Serverless" + serverless-short: "Serverless" + es-serverless: "Elasticsearch Serverless" + es3: "Elasticsearch Serverless" + obs-serverless: "Elastic Observability Serverless" + sec-serverless: "Elastic Security Serverless" + serverless-docs: "https://docs.elastic.co/serverless" + cloud: "https://www.elastic.co/guide/en/cloud/current" + ess-utm-params: "?page=docs&placement=docs-body" + ess-baymax: "?page=docs&placement=docs-body" + ess-trial: "https://cloud.elastic.co/registration?page=docs&placement=docs-body" + ess-product: "https://www.elastic.co/cloud/elasticsearch-service?page=docs&placement=docs-body" + ess-console: "https://cloud.elastic.co?page=docs&placement=docs-body" + ess-console-name: "Elasticsearch Service Console" + ess-deployments: "https://cloud.elastic.co/deployments?page=docs&placement=docs-body" + ece-ref: "https://www.elastic.co/guide/en/cloud-enterprise/current" + eck-ref: "https://www.elastic.co/guide/en/cloud-on-k8s/current" + ess-leadin: "You can run Elasticsearch on your own hardware or use our hosted Elasticsearch Service that is available on AWS, GCP, and Azure. https://cloud.elastic.co/registration{ess-utm-params}[Try the Elasticsearch Service for free]." + ess-leadin-short: "Our hosted Elasticsearch Service is available on AWS, GCP, and Azure, and you can https://cloud.elastic.co/registration{ess-utm-params}[try it for free]." + ess-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elasticsearch Service\"]" + ece-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud_ece.svg[link=\"https://cloud.elastic.co/registration{ess-utm-params}\", title=\"Supported on Elastic Cloud Enterprise\"]" + cloud-only: "This feature is designed for indirect use by https://cloud.elastic.co/registration{ess-utm-params}[Elasticsearch Service], https://www.elastic.co/guide/en/cloud-enterprise/{ece-version-link}[Elastic Cloud Enterprise], and https://www.elastic.co/guide/en/cloud-on-k8s/current[Elastic Cloud on Kubernetes]. Direct use is not supported." + ess-setting-change: "image:https://doc-icons.s3.us-east-2.amazonaws.com/logo_cloud.svg[link=\"{ess-trial}\", title=\"Supported on {ess}\"] indicates a change to a supported https://www.elastic.co/guide/en/cloud/current/ec-add-user-settings.html[user setting] for Elasticsearch Service." + ess-skip-section: "If you use Elasticsearch Service, skip this section. Elasticsearch Service handles these changes for you." + api-cloud: "https://www.elastic.co + - api/doc/cloud" + api-ece: "https://www.elastic.co + - api/doc/cloud-enterprise" + api-kibana-serverless: "https://www.elastic.co + - api/doc/serverless" + es-feature-flag: "This feature is in development and not yet available for use. This documentation is provided for informational purposes only." + es-ref-dir: "'{{elasticsearch-root}} + - reference'" + apm-app: "APM app" + uptime-app: "Uptime app" + synthetics-app: "Synthetics app" + logs-app: "Logs app" + metrics-app: "Metrics app" + infrastructure-app: "Infrastructure app" + siem-app: "SIEM app" + security-app: "Elastic Security app" + ml-app: "Machine Learning" + dev-tools-app: "Dev Tools" + ingest-manager-app: "Ingest Manager" + stack-manage-app: "Stack Management" + stack-monitor-app: "Stack Monitoring" + alerts-ui: "Alerts and Actions" + rules-ui: "Rules" + rac-ui: "Rules and Connectors" + connectors-ui: "Connectors" + connectors-feature: "Actions and Connectors" + stack-rules-feature: "Stack Rules" + user-experience: "User Experience" + ems: "Elastic Maps Service" + ems-init: "EMS" + hosted-ems: "Elastic Maps Server" + ipm-app: "Index Pattern Management" + ingest-pipelines: "ingest pipelines" + ingest-pipelines-app: "Ingest Pipelines" + ingest-pipelines-cap: "Ingest pipelines" + ls-pipelines: "Logstash pipelines" + ls-pipelines-app: "Logstash Pipelines" + maint-windows: "maintenance windows" + maint-windows-app: "Maintenance Windows" + maint-windows-cap: "Maintenance windows" + custom-roles-app: "Custom Roles" + data-source: "data view" + data-sources: "data views" + data-source-caps: "Data View" + data-sources-caps: "Data Views" + data-source-cap: "Data view" + data-sources-cap: "Data views" + project-settings: "Project settings" + manage-app: "Management" + index-manage-app: "Index Management" + data-views-app: "Data Views" + rules-app: "Rules" + saved-objects-app: "Saved Objects" + tags-app: "Tags" + api-keys-app: "API keys" + transforms-app: "Transforms" + connectors-app: "Connectors" + files-app: "Files" + reports-app: "Reports" + maps-app: "Maps" + alerts-app: "Alerts" + crawler: "Enterprise Search web crawler" + ents: "Enterprise Search" + app-search-crawler: "App Search web crawler" + agent: "Elastic Agent" + agents: "Elastic Agents" + fleet: "Fleet" + fleet-server: "Fleet Server" + integrations-server: "Integrations Server" + ingest-manager: "Ingest Manager" + ingest-management: "ingest management" + package-manager: "Elastic Package Manager" + integrations: "Integrations" + package-registry: "Elastic Package Registry" + artifact-registry: "Elastic Artifact Registry" + aws: "AWS" + stack: "Elastic Stack" + xpack: "X-Pack" + es: "Elasticsearch" + kib: "Kibana" + esms: "Elastic Stack Monitoring Service" + esms-init: "ESMS" + ls: "Logstash" + beats: "Beats" + auditbeat: "Auditbeat" + filebeat: "Filebeat" + heartbeat: "Heartbeat" + metricbeat: "Metricbeat" + packetbeat: "Packetbeat" + winlogbeat: "Winlogbeat" + functionbeat: "Functionbeat" + journalbeat: "Journalbeat" + es-sql: "Elasticsearch SQL" + esql: "ES|QL" + elastic-agent: "Elastic Agent" + k8s: "Kubernetes" + log-driver-long: "Elastic Logging Plugin for Docker" + security: "X-Pack security" + security-features: "security features" + operator-feature: "operator privileges feature" + es-security-features: "Elasticsearch security features" + stack-security-features: "Elastic Stack security features" + endpoint-sec: "Endpoint Security" + endpoint-cloud-sec: "Endpoint and Cloud Security" + elastic-defend: "Elastic Defend" + elastic-sec: "Elastic Security" + elastic-endpoint: "Elastic Endpoint" + swimlane: "Swimlane" + sn: "ServiceNow" + sn-itsm: "ServiceNow ITSM" + sn-itom: "ServiceNow ITOM" + sn-sir: "ServiceNow SecOps" + jira: "Jira" + ibm-r: "IBM Resilient" + webhook: "Webhook" + webhook-cm: "Webhook - Case Management" + opsgenie: "Opsgenie" + bedrock: "Amazon Bedrock" + gemini: "Google Gemini" + hive: "TheHive" + monitoring: "X-Pack monitoring" + monitor-features: "monitoring features" + stack-monitor-features: "Elastic Stack monitoring features" + watcher: "Watcher" + alert-features: "alerting features" + reporting: "X-Pack reporting" + report-features: "reporting features" + graph: "X-Pack graph" + graph-features: "graph analytics features" + searchprofiler: "Search Profiler" + xpackml: "X-Pack machine learning" + ml: "machine learning" + ml-cap: "Machine learning" + ml-init: "ML" + ml-features: "machine learning features" + stack-ml-features: "Elastic Stack machine learning features" + ccr: "cross-cluster replication" + ccr-cap: "Cross-cluster replication" + ccr-init: "CCR" + ccs: "cross-cluster search" + ccs-cap: "Cross-cluster search" + ccs-init: "CCS" + ilm: "index lifecycle management" + ilm-cap: "Index lifecycle management" + ilm-init: "ILM" + dlm: "data lifecycle management" + dlm-cap: "Data lifecycle management" + dlm-init: "DLM" + search-snap: "searchable snapshot" + search-snaps: "searchable snapshots" + search-snaps-cap: "Searchable snapshots" + slm: "snapshot lifecycle management" + slm-cap: "Snapshot lifecycle management" + slm-init: "SLM" + rollup-features: "data rollup features" + ipm: "index pattern management" + ipm-cap: "Index pattern" + rollup: "rollup" + rollup-cap: "Rollup" + rollups: "rollups" + rollups-cap: "Rollups" + rollup-job: "rollup job" + rollup-jobs: "rollup jobs" + rollup-jobs-cap: "Rollup jobs" + dfeed: "datafeed" + dfeeds: "datafeeds" + dfeed-cap: "Datafeed" + dfeeds-cap: "Datafeeds" + ml-jobs: "machine learning jobs" + ml-jobs-cap: "Machine learning jobs" + anomaly-detect: "anomaly detection" + anomaly-detect-cap: "Anomaly detection" + anomaly-job: "anomaly detection job" + anomaly-jobs: "anomaly detection jobs" + anomaly-jobs-cap: "Anomaly detection jobs" + dataframe: "data frame" + dataframes: "data frames" + dataframe-cap: "Data frame" + dataframes-cap: "Data frames" + watcher-transform: "payload transform" + watcher-transforms: "payload transforms" + watcher-transform-cap: "Payload transform" + watcher-transforms-cap: "Payload transforms" + transform: "transform" + transforms: "transforms" + transform-cap: "Transform" + transforms-cap: "Transforms" + dataframe-transform: "transform" + dataframe-transform-cap: "Transform" + dataframe-transforms: "transforms" + dataframe-transforms-cap: "Transforms" + dfanalytics-cap: "Data frame analytics" + dfanalytics: "data frame analytics" + dataframe-analytics-config: "'{dataframe} analytics config'" + dfanalytics-job: "'{dataframe} analytics job'" + dfanalytics-jobs: "'{dataframe} analytics jobs'" + dfanalytics-jobs-cap: "'{dataframe-cap} analytics jobs'" + cdataframe: "continuous data frame" + cdataframes: "continuous data frames" + cdataframe-cap: "Continuous data frame" + cdataframes-cap: "Continuous data frames" + cdataframe-transform: "continuous transform" + cdataframe-transforms: "continuous transforms" + cdataframe-transforms-cap: "Continuous transforms" + ctransform: "continuous transform" + ctransform-cap: "Continuous transform" + ctransforms: "continuous transforms" + ctransforms-cap: "Continuous transforms" + oldetection: "outlier detection" + oldetection-cap: "Outlier detection" + olscore: "outlier score" + olscores: "outlier scores" + fiscore: "feature influence score" + evaluatedf-api: "evaluate {dataframe} analytics API" + evaluatedf-api-cap: "Evaluate {dataframe} analytics API" + binarysc: "binary soft classification" + binarysc-cap: "Binary soft classification" + regression: "regression" + regression-cap: "Regression" + reganalysis: "regression analysis" + reganalysis-cap: "Regression analysis" + depvar: "dependent variable" + feature-var: "feature variable" + feature-vars: "feature variables" + feature-vars-cap: "Feature variables" + classification: "classification" + classification-cap: "Classification" + classanalysis: "classification analysis" + classanalysis-cap: "Classification analysis" + infer-cap: "Inference" + infer: "inference" + lang-ident-cap: "Language identification" + lang-ident: "language identification" + data-viz: "Data Visualizer" + file-data-viz: "File Data Visualizer" + feat-imp: "feature importance" + feat-imp-cap: "Feature importance" + nlp: "natural language processing" + nlp-cap: "Natural language processing" + apm-agent: "APM agent" + apm-go-agent: "Elastic APM Go agent" + apm-go-agents: "Elastic APM Go agents" + apm-ios-agent: "Elastic APM iOS agent" + apm-ios-agents: "Elastic APM iOS agents" + apm-java-agent: "Elastic APM Java agent" + apm-java-agents: "Elastic APM Java agents" + apm-dotnet-agent: "Elastic APM .NET agent" + apm-dotnet-agents: "Elastic APM .NET agents" + apm-node-agent: "Elastic APM Node.js agent" + apm-node-agents: "Elastic APM Node.js agents" + apm-php-agent: "Elastic APM PHP agent" + apm-php-agents: "Elastic APM PHP agents" + apm-py-agent: "Elastic APM Python agent" + apm-py-agents: "Elastic APM Python agents" + apm-ruby-agent: "Elastic APM Ruby agent" + apm-ruby-agents: "Elastic APM Ruby agents" + apm-rum-agent: "Elastic APM Real User Monitoring (RUM) JavaScript agent" + apm-rum-agents: "Elastic APM RUM JavaScript agents" + apm-lambda-ext: "Elastic APM AWS Lambda extension" + project-monitors: "project monitors" + project-monitors-cap: "Project monitors" + private-location: "Private Location" + private-locations: "Private Locations" + pwd: "YOUR_PASSWORD" + esh: "ES-Hadoop" + default-dist: "default distribution" + oss-dist: "OSS-only distribution" + observability: "Observability" + api-request-title: "Request" + api-prereq-title: "Prerequisites" + api-description-title: "Description" + api-path-parms-title: "Path parameters" + api-query-parms-title: "Query parameters" + api-request-body-title: "Request body" + api-response-codes-title: "Response codes" + api-response-body-title: "Response body" + api-example-title: "Example" + api-examples-title: "Examples" + api-definitions-title: "Properties" + multi-arg: "†footnoteref:[multi-arg,This parameter accepts multiple arguments.]" + multi-arg-ref: "†footnoteref:[multi-arg]" + yes-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-yes.png[Yes,20,15]" + no-icon: "image:https://doc-icons.s3.us-east-2.amazonaws.com/icon-no.png[No,20,15]" + es-repo: "https://github.com/elastic/elasticsearch/" + es-issue: "https://github.com/elastic/elasticsearch/issues/" + es-pull: "https://github.com/elastic/elasticsearch/pull/" + es-commit: "https://github.com/elastic/elasticsearch/commit/" + kib-repo: "https://github.com/elastic/kibana/" + kib-issue: "https://github.com/elastic/kibana/issues/" + kibana-issue: "'{kib-repo}issues/'" + kib-pull: "https://github.com/elastic/kibana/pull/" + kibana-pull: "'{kib-repo}pull/'" + kib-commit: "https://github.com/elastic/kibana/commit/" + ml-repo: "https://github.com/elastic/ml-cpp/" + ml-issue: "https://github.com/elastic/ml-cpp/issues/" + ml-pull: "https://github.com/elastic/ml-cpp/pull/" + ml-commit: "https://github.com/elastic/ml-cpp/commit/" + apm-repo: "https://github.com/elastic/apm-server/" + apm-issue: "https://github.com/elastic/apm-server/issues/" + apm-pull: "https://github.com/elastic/apm-server/pull/" + kibana-blob: "https://github.com/elastic/kibana/blob/current/" + apm-get-started-ref: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-server-ref: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-v: "https://www.elastic.co/guide/en/apm/server/current" + apm-server-ref-m: "https://www.elastic.co/guide/en/apm/server/master" + apm-server-ref-62: "https://www.elastic.co/guide/en/apm/server/6.2" + apm-server-ref-64: "https://www.elastic.co/guide/en/apm/server/6.4" + apm-server-ref-70: "https://www.elastic.co/guide/en/apm/server/7.0" + apm-overview-ref-v: "https://www.elastic.co/guide/en/apm/get-started/current" + apm-overview-ref-70: "https://www.elastic.co/guide/en/apm/get-started/7.0" + apm-overview-ref-m: "https://www.elastic.co/guide/en/apm/get-started/master" + infra-guide: "https://www.elastic.co/guide/en/infrastructure/guide/current" + a-data-source: "a data view" + icon-bug: "pass:[]" + icon-checkInCircleFilled: "pass:[]" + icon-warningFilled: "pass:[]" diff --git a/docs/extend/_publish_an_integration.md b/docs/extend/_publish_an_integration.md new file mode 100644 index 00000000000..c247b678867 --- /dev/null +++ b/docs/extend/_publish_an_integration.md @@ -0,0 +1,37 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/_publish_an_integration.html +--- + +# Publish an integration [_publish_an_integration] + +When your integration is done, it’s time to open a PR to include it in the integrations repository. Before opening your PR, run: + +```bash +elastic-package check +``` + +The `check` command ensures the package is built correctly, formatted properly, and aligned with the spec. Passing the `check` command is required before adding your integration to the repository. + +When CI is happy, merge your PR into the integrations repository. + +CI will kick off a build job for the main branch, which can release your integration to the package-storage. It means that it will open a PR to the Package Storage/snapshot with the built integration if only the package version doesn’t already exist in the storage (hasn’t been released yet). + + +## Promote [_promote] + +Now that you’ve tested your integration with {{kib}}, it’s time to promote it to staging or production. Run: + +```bash +elastic-package promote +``` + +The tool will open 2 pull requests (promote and delete) to the package-storage: target and source branches. + +Please review both pull requests on your own, check if CI is happy and merge - first target, then source. Once any PR is merged, the CI will kick off a job to bake a new Docker image of package-storage (tracking). Ideally the "delete" PR should be merged once the CI job for "promote" is done, as the Docker image of previous stage depends on the later one. + +::::{tip} +When you are ready for your changes in the integration to be released, remember to bump up the package version. It is up to you, as the package developer, to decide how many changes you want to release in a single version. For example, you could implement a change in a PR and bump up the package version in the same PR. Or you could implement several changes across multiple pull requests and then bump up the package version in the last of these pull requests or in a separate follow up PR. +:::: + + diff --git a/docs/extend/add-data-stream.md b/docs/extend/add-data-stream.md new file mode 100644 index 00000000000..d75ff9381bd --- /dev/null +++ b/docs/extend/add-data-stream.md @@ -0,0 +1,43 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/add-a-data-stream.html +--- + +# Add a data stream [add-a-data-stream] + +A data stream is a logical sub-division of an integration package, dealing with a specific observable aspect of the service or product being observed. For example, the [Apache integration](https://github.com/elastic/integrations/tree/main/packages/apache) has three data streams, each represented by a separate folder of assets in the `data_stream` directory: + +```text +apache +└───data_stream +│ └───access +│ └───error +│ └───status +``` + +::::{admonition} +**Data streams** allow you to store time series data across multiple indices while giving you a single named resource for requests. + +A data stream defines multiple {{es}} assets, like index templates, ingest pipelines, and field definitions. These assets are loaded into {{es}} when a user installs an integration using the {{fleet}} UI in {{kib}}. + +A data stream also defines a policy template. Policy templates include variables that allow users to configure the data stream using the {{fleet}} UI in {{kib}}. Then, the {{agent}} interprets the resulting policy to collect relevant information from the product or service being observed. Policy templates can also define an integration’s supported [`deployment_modes`](/extend/define-deployment-modes.md#deployment_modes). + +See [data streams](docs-content://reference/ingestion-tools/fleet/data-streams.md) for more information. + +:::: + + +Bootstrap a new data stream using the TUI wizard. In the directory of your package, run: + +```bash +elastic-package create data-stream +``` + +Follow the prompts to name, title, and select your data stream type. Then, run this command each time you add a new data stream to your integration. + +Next, manually adjust the data stream: + +* define required variables +* define used fields +* define ingest pipeline definitions (if necessary) +* update the {{agent}}'s stream configuration diff --git a/docs/extend/add-mapping.md b/docs/extend/add-mapping.md new file mode 100644 index 00000000000..435a0a0511a --- /dev/null +++ b/docs/extend/add-mapping.md @@ -0,0 +1,127 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/add-a-mapping.html +--- + +# Edit field mappings [add-a-mapping] + +Ingest pipelines create fields in an {{es}} index, but don’t define the fields themselves. Instead, each field requires a defined data type or mapping. + +::::{admonition} +**Mapping** is the process of defining how a document, and the fields it contains, are stored and indexed. Each document is a collection of fields, each having its own data type. When mapping your data, create a mapping definition containing a list of fields pertinent to the document. A mapping definition also includes metadata fields, like the _source field, which customize how the associated metadata of a document is handled. + +To learn more, see [mapping](docs-content://manage-data/data-store/mapping.md). + +:::: + + +In the integration, the `fields` directory serves as the blueprint used to create component templates for the integration. The content from all files in this directory will be unified when the integration is built, so the mappings need to be unique per data stream dataset. + +Like ingest pipelines, mappings only apply to the data stream dataset, for our example the `apache.access` dataset. + ++ NOTE: The names of these files are conventions, any file name with a `.yml` extension will work. + +Integrations have had significant enhancements in how ECS fields are defined. Below is a guide on which approach to use, based on the version of Elastic your integration will support. + ++ . ECS mappings component template (>=8.13.0) Integrations **only** supporting version 8.13.0 and up, can use the [ecs@mappings](https://github.com/elastic/elasticsearch/blob/c2a3ec42632b0339387121efdef13f52c6c66848/x-pack/plugin/core/template-resources/src/main/resources/ecs%40mappings.json) component template installed by Fleet. This makes explicitly declaring ECS fields unnecessary; the `ecs@mappings` component template in Elasticsearch will automatically detect and configure them. However, should ECS fields be explicitly defined, they will overwrite the dynamic mapping provided by the `ecs@mappings` component template. They can also be imported with an `external` declaration, as seen in the example below. + ++ . Dynamic mappings imports (<8.13.0 & >=8.13.0) Integrations supporting the Elastic stack below version 8.13.0 can still dynamically import ECS field mappings by defining `import_mappings: true` in the ECS section of the `_dev/build/build.yml` file in the root of the package directory. This introduces a [dynamic mapping](https://github.com/elastic/elastic-package/blob/f439b96a74c27c5adfc3e7810ad584204bfaf85d/internal/builder/_static/ecs_mappings.yaml) with most of the ECS definitions. Using this method means that, just like the previous approach, ECS fields don’t need to be defined in your integration, they are dynamically integrated into the package at build time. Explicitly defined ECS fields can be used and will also overwrite this mechanism. + +An example of the aformentioned `build.yml` file for this method: + ++ + +```yaml +dependencies: + ecs: + reference: git@v8.6.0 + import_mappings: true +``` + ++ . Explicit ECS mappings As mentioned in the previous two approaches, ECS mappings can still be set explicitly and will overwrite the dynamic mappings. This can be done in two ways: - Using an `external: ecs` reference to import the definition of a specific field. - Literally defining the ECS field. + +The `external: ecs` definition instructs the `elastic-package` command line tool to refer to an external ECS reference to resolve specific fields. By default it looks at the [ECS reference](https://raw.githubusercontent.com/elastic/ecs/v8.6.0/generated/ecs/ecs_nested.yml) file hosted on Github. This external reference file is determined by a Git reference found in the `_dev/build/build.yml` file, in the root of the package directory. The `build.yml` file set up for external references: + ++ + +```yaml +dependencies: + ecs: + reference: git@v8.6.0 +``` + +Literal definition a ECS field: + +```yaml +- name: cloud.acount.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organ....' + example: 43434343 +``` + +1. Local ECS reference file (air-gapped setup) By changing the Git reference in in `_dev/build/build.yml` to the path of the downloaded [ECS reference](https://raw.githubusercontent.com/elastic/ecs/v8.6.0/generated/ecs/ecs_nested.yml) file, it is possible for the `elastic-package` command line tool to look for this file locally. Note that the path should be the full path to the reference file. Doing this, our `build.yml` file looks like: + + ``` + dependencies: + ecs: + reference: file:///home/user/integrations/packages/apache/ecs_nested.yml + ``` + + +The `access` data stream dataset of the Apache integration has four different field definitions: + ++ NOTE: The `apache` integration below has not yet been updated to use the dynamic ECS field definition and uses `external` references to define ECS fields in `ecs.yml`. + ++ + +```text +apache +└───data_stream +│ └───access +│ │ └───elasticsearch/ingest_pipeline +│ │ │ default.yml +│ │ └───fields +│ │ agent.yml +│ │ base-fields.yml +│ │ ecs.yml +│ │ fields.yml +│ └───error +│ │ └───elasticsearch/ingest_pipeline +│ │ │ default.yml +│ │ └───fields +│ │ agent.yml +│ │ base-fields.yml +│ │ ecs.yml +│ │ fields.yml +│ └───status +``` + +## agent.yml [_agent_yml] + +The `agent.yml` file defines fields used by default processors. Examples: `cloud.account.id`, `container.id`, `input.type` + + +## base-fields.yml [_base_fields_yml] + +In this file, the `data_stream` subfields `type`, `dataset` and `namespace` are defined as type `constant_keyword`, the values for these fields are added by the integration. The `event.module` and `event.dataset` fields are defined with a fixed value specific for this integration: - `event.module: apache` - `event.dataset: apache.access` Field `@timestamp` is defined here as type `date`. + + +## fields.yml [_fields_yml] + +Here we define fields that we need in our integration and are not found in the ECS. The example below defines field `apache.access.ssl.protocol` in the Apache integration. + ++ + +```yaml +- name: apache.access + type: group + fields: + - name: ssl.protocol + type: keyword + description: | + SSL protocol version. +``` + +Learn more about fields in the [general guidelines](/extend/general-guidelines.md#_document_all_fields). diff --git a/docs/extend/asset-testing.md b/docs/extend/asset-testing.md new file mode 100644 index 00000000000..999ea668bff --- /dev/null +++ b/docs/extend/asset-testing.md @@ -0,0 +1,64 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/asset-testing.html +--- + +# Asset testing [asset-testing] + +Elastic Packages define assets to be loaded into {{es}} and {{kib}}. Asset loading tests exercise install a package to ensure that its assets are loaded into {{es}} and {{kib}} as expected. + + +## Conceptual process [asset-testing-concepts] + +Conceptually, running an asset load test involves the following steps: + +1. Build the package. +2. Deploy {{es}}, {{kib}}, and the {{package-registry}} (all of which are part of the {{stack}}). This step takes time, so you should typically do it once as a prerequisite to running asset loading tests on multiple packages. +3. Install the package. +4. Use various {{kib}} and {{es}} APIs to confirm that the package assets were loaded into {{kib}} and {{es}} as expected. +5. Remove the package. + + +## Define an asset loading test [define-asset-test] + +As a package developer, there is no work required to define an asset loading test for your package. All the necessary information is contained in the package files. + + +## Run an asset loading test [running-asset-test] + +First, you must build your package. This step corresponds to step 1 in the [Conceptual process](#asset-testing-concepts) section. + +Navigate to the root folder of the package, or any sub-folder under it, and run the following command. + +```bash +elastic-package build +``` + +Next, deploy {{es}}, {{kib}}, and the {{package-registry}}. This step corresponds to step 2 in the [Conceptual process](#asset-testing-concepts) section. + +```bash +elastic-package stack up -d +``` + +To view a list of the available options for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, set the environment variables that are required for additional `elastic-package` commands. + +```bash +$(elastic-package stack shellinit) +``` + +Next, invoke the asset loading test runner. This step corresponds to steps 3 to 5 in the [Conceptual process](#asset-testing-concepts) section. + +Navigate to the root folder of the package, or any sub-folder under it, and run the following command. + +```bash +elastic-package test asset +``` + +Finally, when all the asset loading tests have completed, bring down the {{stack}}. This step corresponds to step 4 in the [Conceptual process](#asset-testing-concepts) section. + +```bash +elastic-package stack down +``` + diff --git a/docs/extend/build-create-package.md b/docs/extend/build-create-package.md new file mode 100644 index 00000000000..64fe6d9c550 --- /dev/null +++ b/docs/extend/build-create-package.md @@ -0,0 +1,23 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/build-create-package.html +--- + +# Create a new package [build-create-package] + +Rather than copying the source of an existing package, we recommend using the `elastic-package create` command to build a new package. Running this command ensures that your integration follows the latest recommendations for the package format. + +Use the `elastic-package` TUI wizard to bootstrap a new package: + +```bash +elastic-package create package +``` + +The wizard walks you through the creation of the package, including setting a package name, version, category, etc. When the wizard completes, you’ll have a basic package complete with a sample manifest, changelog, documentation, and screenshot. + +::::{note} +It may not do anything yet, but your integration can be built and loaded into your locally running package registry from this step forward. Jump to [Build](/extend/build-it.md) at any point in this documentation to take your integration for a test run. + +:::: + + diff --git a/docs/extend/build-it.md b/docs/extend/build-it.md new file mode 100644 index 00000000000..744336e26db --- /dev/null +++ b/docs/extend/build-it.md @@ -0,0 +1,25 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/build-it.html +--- + +# Build [build-it] + +To format, lint, and build your integration, in that order, run: + +```bash +elastic-package check +``` + +Problems and potential solutions will display in the console. Fix them and rerun the command. Alternatively, skip formatting and linting with the `build` command: + +```bash +elastic-package build +``` + +With the package built, run the following command from inside of the integration directory to recycle the package-registry docker container. This refreshes the {{fleet}} UI, allowing it to pick up the new integration in {{kib}}. + +```bash +elastic-package stack up --services package-registry +``` + diff --git a/docs/extend/build-new-integration.md b/docs/extend/build-new-integration.md new file mode 100644 index 00000000000..fec8cf7ab1c --- /dev/null +++ b/docs/extend/build-new-integration.md @@ -0,0 +1,38 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/build-a-new-integration.html +--- + +# Build an integration [build-a-new-integration] + +Ready to monitor, ingest, and visualize something? Let’s get started. + +* [Overview and prerequisites](/extend/build-overview.md) +* [Spin up the {{stack}}](/extend/build-spin-stack.md) +* [Create a new package](/extend/build-create-package.md) +* [Add a data stream](/extend/add-data-stream.md) +* [Define deployment modes](/extend/define-deployment-modes.md) +* [Edit ingest pipelines](/extend/edit-ingest-pipeline.md) +* [Edit field mappings](/extend/add-mapping.md) +* [Create and export dashboards](/extend/create-dashboards.md) +* [Testing and validation](/extend/testing-validation.md) +* [Finishing touches](/extend/finishing-touches.md) +* [Tips for building integrations](/extend/tips-for-building.md) + +::::{tip} +Familiar with the {{stack}} and just want a quick way to get started? See [*Quick start: Sample integration*](/extend/quick-start.md). +:::: + + + + + + + + + + + + + + diff --git a/docs/extend/build-overview.md b/docs/extend/build-overview.md new file mode 100644 index 00000000000..4ede5bc9b12 --- /dev/null +++ b/docs/extend/build-overview.md @@ -0,0 +1,14 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/build-overview.html +--- + +# Overview and prerequisites [build-overview] + +Before building an integration, you should have an understanding of the following: + +* {{stack}} concepts, like data streams, ingest pipelines, and mappings +* The [*Package specification*](/extend/package-spec.md) + +In addition, you must have [`elastic-package`](/extend/elastic-package.md) installed on your machine. Using `elastic-package` is recommended for integration maintainers as it provides crucial utilities and scripts for building out integrations. + diff --git a/docs/extend/build-spin-stack.md b/docs/extend/build-spin-stack.md new file mode 100644 index 00000000000..c5774f444dd --- /dev/null +++ b/docs/extend/build-spin-stack.md @@ -0,0 +1,31 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/build-spin-stack.html +--- + +# Spin up the Elastic Stack [build-spin-stack] + +The [`elastic-package`](/extend/elastic-package.md) tool provides a quick way to spin up the {{stack}}. The following command deploys {{es}}, {{kib}}, and the {{package-registry}}: + +```bash +elastic-package stack up -v -d +``` + +To view a list of the available options for this command, run: + +```bash +elastic-package stack up -h +``` + +When complete, go to [http://localhost:5601](http://localhost:5601) and log in with the username `elastic` and the password `changeme`. + +::::{tip} +Development time over? Tear down the {{stack}} with: + +```bash +elastic-package stack down +``` + +:::: + + diff --git a/docs/extend/changelog-spec.md b/docs/extend/changelog-spec.md new file mode 100644 index 00000000000..334d9440834 --- /dev/null +++ b/docs/extend/changelog-spec.md @@ -0,0 +1,59 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/changelog-spec.html +--- + +# changelog.yml [changelog-spec] + +The integration’s changelog. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +## +## Describes the specification for the package's CHANGELOG file +## +spec: + # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability + type: array + items: + type: object + additionalProperties: false + properties: + version: + description: Package version. + $ref: "./manifest.spec.yml#/definitions/version" + changes: + description: List of changes in package version. + type: array + items: + type: object + additionalProperties: false + properties: + description: + description: Description of change. + type: string + examples: + - "Fix broken template" + type: + description: Type of change. + type: string + enum: + - "breaking-change" + - "bugfix" + - "enhancement" + link: + description: Link to issue or PR describing change in detail. + type: string + examples: + - "https://github.com/elastic/integrations/pull/550" + required: + - description + - type + - link + required: + - version + - changes +``` diff --git a/docs/extend/create-dashboards.md b/docs/extend/create-dashboards.md new file mode 100644 index 00000000000..7462eef690a --- /dev/null +++ b/docs/extend/create-dashboards.md @@ -0,0 +1,123 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/create-dashboards.html +--- + +# Create and export dashboards [create-dashboards] + +Visualizing integration data in a meaningful way is an important aspect of an integration. + +When creating a new integration, it’s important to add dashboards. + +To get started, create a new dashboard, or customize an existing one. You can use `elastic-package` to boot up the service stack. Navigate to the package you want to create dashboards for, and run: + +```bash +elastic-package service +``` + +When you’re done making changes, you can use `elastic-package` to export the dashboards and their dependencies to the package source. + + +## Dashboard planning [_dashboard_planning] + +Many integrations cover more than one component of a target system. For example, the RabbitMQ module provides several metricsets covering connection, exchange, node, queue. It makes sense to break this information down into several interconnected dashboards. The default one is an overview of a target system, and the others provide deep-dives into the various parts of the target system. The content of the Overview dashboard should be cherry-picked from all datasets and individually compiled for every such integration. + + +### Metrics [_metrics] + +Always check the type of a metric and ensure that the correct transformation is applied where applicable. For example, in most cases for cumulative counters, it makes sense to use the rate function. + + +### Visualization type [_visualization_type] + +For new visualizations, we recommend using Lens first. If what you’re trying to achieve cannot be accomplished with the current capabilities of Lens, try TSVB. + + +### Filters [_filters] + +When building a dashboard, always consider adding a filter dropdown. Why? In most cases, the integrations monitor multiple instances of a target system, so we need to provide a way to switch between them. + +To build a filter dropdown, use the Controls visualization. Here’s an example of a host name dropdown that you can add to the System dashboard: + + +### Navigation [_navigation] + +If an integration has several dashboards, ensure that you can easily navigate all of them. To build dashboard navigation, use the Markdown visualization type. + +For example, the System dashboard provides the following navigation: + +Source: + +```text +[System Overview](#/dashboard/system-Metrics-system-overview-ecs) | [Host Overview](#/dashboard/system-79ffd6e0-faa0-11e6-947f-177f697178b8-ecs) | +[Containers overview](#/dashboard/system-CPU-slash-Memory-per-container-ecs) +``` + +While this can work, it doesn’t highlight the selected dashboard. Unfortunately the Markdown control is not optimized for navigation, which makes it cumbersome to build navigation with highlighted links because each link should be highlighted separately. This means that the navigation control you’re building has to be cloned as many times as there are dashboard to ensure proper link highlighting. E.g. + +```text +**[System Overview](#/dashboard/system-Metrics-system-overview-ecs)** | [Host Overview](#/dashboard/system-79ffd6e0-faa0-11e6-947f-177f697178b8-ecs) | +[Containers overview](#/dashboard/system-CPU-slash-Memory-per-container-ecs) + +[System Overview](#/dashboard/system-Metrics-system-overview-ecs) | **[Host Overview](#/dashboard/system-79ffd6e0-faa0-11e6-947f-177f697178b8-ecs)** | +[Containers overview](#/dashboard/system-CPU-slash-Memory-per-container-ecs) + +[System Overview](#/dashboard/system-Metrics-system-overview-ecs) | [Host Overview](#/dashboard/system-79ffd6e0-faa0-11e6-947f-177f697178b8-ecs) | +**[Containers overview](#/dashboard/system-CPU-slash-Memory-per-container-ecs)** +``` + + +### Target system name [_target_system_name] + +Currently we don’t make it a rule to show on a dashboard what system it’s designed to monitor. The only way to see it is through the dashboard name. + +When using multiple dashboards on bigger screens, it makes it hard to distinguish between the dashboards. You can improve this by using the Markdown control to display the target system the dashboard is used for. + + +### Naming [_naming] + +When building dashboards, use the following naming convention. + + +#### Visualizations [_visualizations] + +```text + [ ] +``` + +Examples: + +* Memory Usage Gauge [Metrics System] +* New groups [Logs System] + +Rename all visualizations added to a dashboard only to show the part. + + +#### Dashboards [_dashboards] + +```text +[ ] +``` + +Examples: + +* [Metrics System] Host overview +* [Metrics MongoDB] Overview + + +### Screenshots [_screenshots] + +Letter casing is important for screenshot descriptions. Descriptions are shown in the {{kib}} UI, so try and keep them clean and consistent. + +These descriptions are visualized in the {{kib}} UI. It would be better experience to have them clean and consistent. + +* Bad candidate: filebeat running on ec2 machine +* Good candidates: {{filebeat}} running on AWS EC2 machine + + +## Exporting [_exporting] + +```bash +elastic-package export +``` + diff --git a/docs/extend/dashboard-guidelines.md b/docs/extend/dashboard-guidelines.md new file mode 100644 index 00000000000..6bcd351beeb --- /dev/null +++ b/docs/extend/dashboard-guidelines.md @@ -0,0 +1,159 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/dashboard-guidelines.html +--- + +# Dashboard guidelines [dashboard-guidelines] + +A [Kibana dashboard](docs-content://explore-analyze/dashboards.md) is a set of one or more panels, also referred to as visualizations. Panels display data in charts, tables, maps, and more. Dashboards support several types of panels to display your data, and several options to create panels. + +The goal of each integration dashboard is to: + +* Provide a way to explore ingested data out of the box. +* Provide an overview of the monitored resources through installing the integration. + +Each integration package should contain one or more dashboards. + + +## Dashboard Best Practices [_dashboard_best_practices] + +Following are recommended best practices for designing Kibana dashboards. + + +### Build dashboards on stable versions [_build_dashboards_on_stable_versions] + +Avoid building dashboards on SNAPSHOT versions because as long as the release is not stable behavior changes might render your dashboard unusable. The only supported approach is to use a globally released version from the [official releases list](https://www.elastic.co/downloads/past-releases#kibana). + + +### Not too many visualizations per dashboard [_not_too_many_visualizations_per_dashboard] + +Include only necessary visualizations inside a dashboard, and, when possible, split them across separate dashboards. Linking can be done: + +* By using a Markdown visualization to improve performance +* Use [drilldowns](docs-content://explore-analyze/dashboards/drilldowns.md) to connect dashboards where they make sense. + + +### Out of date fields in dashboards [_out_of_date_fields_in_dashboards] + +The dashboards must be updated to reflect any changes to field names or types. If a pull request updates a field name or type, make sure it is correctly updated in any dashboard the field is being used in. + + +### Add visualizations by value, not by reference [_add_visualizations_by_value_not_by_reference] + +Kibana visualizations can be added into a dashboard by value or by reference. Historically, adding by value did not exist. Switching to value has the advantage that the dashboards are fully self contained and only need a single request to be installed. + +To achieve this: + +* Migrate existing dashboards from `by reference` to `by value`. +* Create new dashboards adding visualizations by value. + +A migration script is available to help with the migration: [flash1293/legacy_vis_analyzer](https://github.com/elastic/visualizations_integrations_tools) + + +### Choose the context of your Dashboard [_choose_the_context_of_your_dashboard] + +You should always try to understand as much as possible what kind of context your users need to interact with the dashboard. Keep the minimal context needed by answering the following questions: + +* Who is going to use this dashboard? +* How much time will the users have? +* What is the main goal of this dashboard and what are any secondary goals? +* What kind of charts can help users identify insights in the most immediate and clear way? + + +### Organisation and hierarchy matters in your dashboards [_organisation_and_hierarchy_matters_in_your_dashboards] + +Keep the following guidelines in mind when positioning your elements on dashboards: + +* Keep related visualizations close to each other. + + :::{image} ../images/grouping-in-visualizations.png + :alt: Closely grouped visualizations + ::: + +* Use Markdown to create blocks of related content. + + :::{image} ../images/markdown-grouping.png + :alt: Markdown grouping in visualizations + ::: + +* Reading Direction + + Most people are used to reading from top to bottom. Place at the top of your page the most important charts and the ones that could give a brief and immediate summary of the context. A good general guidelines is to increase the level of detail as you approach the bottom of the dashboard. This way, users interested in getting all the information can obtain it without requiring too much effort, and other users can gather what they need from only a quick glance at the topmost dashboards. + +* Central focal point + + Placing a big chart at the center of a dashboard, especially one with prominent visual shapes such as rectangles, helps to reinforce a natural visual focal point that lies in the center of the interface. + + :::{image} ../images/rows-in-visualizations.png + :alt: Central focal point in visualization + ::: + + + +### Use Margins [_use_margins] + +Kibana dashboards offer the possibility to apply margins between visualizations, and this is highly recommended. Margins create separation between charts, which is an important visual feature, and they help users to identify when two elements belong together. At the same time, the added space makes the interface appear more clean and elegant. + + +## Visualization Best Practices [_visualization_best_practices] + +Following are recommended best practices for designing Kibana vizualizations. + + +### Lens vs TSVB visualizations [_lens_vs_tsvb_visualizations] + +**Always use Lens**, when possible. It’s the best choice to be consistent and up to date. + +When possible, migrate dashboards from TSVB to Lens. If it’s not possible, please engage with the Kibana team to identify any gaps that prevent full TSVB to Lens dashboard migration. + + +### Visualizations should contain a filter [_visualizations_should_contain_a_filter] + +Kibana visualizations can define a filter to avoid performance issues when querying all metrics (`metrics-*`) or logs (`logs-*`) indices. + +It is recommended to set a filter in each visualization at least by the required `data_stream.dataset`. For more details, refer to the the [Elastic data stream naming scheme](https://www.elastic.co/blog/an-introduction-to-the-elastic-data-stream-naming-scheme). + +As much as possible, avoid using general filters, that is filters with `-*`. Combine multiple fields and values inside a filter with AND/OR operators. Although your filter might become more complex, it will avoid extra queries. + +Example: + +:::{image} ../images/filter-in-visualization.png +:alt: Filter in a visualization +::: + + +### Do not use library visualizations [_do_not_use_library_visualizations] + +Do not use the visualizations that appear in **Analytics > Visualize library**. Instead, define visualizations as part of the dashboard. This is the default when creating new panels by clicking **Add new visualization** on the dashboard. If some panels are already saved to the library, you can unlink them and delete them from the library + +There are some cases where library visualizations are preferable. It makes sense, for example, if a given visualization always has to be exactly the same on multiple dashboards or if its users frequently look at the visualization without looking at the whole dashboard. + + +## Use dashboard-native controls [_use_dashboard_native_controls] + +The **Input controls** visualization type is deprecated in favor of **Controls** embedded into the dashboard itself. The **Controls** dropdown in the Dashboard menu bar should be used. Refer to [Filter dashboard data with controls](docs-content://explore-analyze/dashboards/add-controls.md) for more information. + + +### Keep Consistent Color [_keep_consistent_color] + +Use color to distinguish categories, represent quantity/density, and highlight data. When using color in this way, be aware that too many colors in a single chart can create noise and hinder quick comprehension. + +[Elastic UI](https://elastic.github.io/eui/#/elastic-charts/creating-charts) provides guidance for correct color choice. Colors provided there for visualization have been tested for accessibility contrast. By using them, you are sure properly serve the largest possible audience. + +If your dashboard is made to identify specific behaviors, it might be interesting to consider a color setting that could help to point those out. Use a neutral color for generic elements and an accented color for the things that you want to highlight. + +:::{image} ../images/colors-in-visualizations.png +:alt: Colors in visualizations +::: + + +## Titles in Visualisations matter [_titles_in_visualisations_matter] + +Titles can have a strong visual impact on dashboards, especially when there are a lot of small charts. Two principles can generally be followed: + +* Remove unnecessary or repetitive titles when the information is already explained or written within the chart. +* When a title is needed, make it self explanatory and exhaustive. This way, you will be able to remove axis titles and other specifications leaving more space for the chart itself. + +:::{image} ../images/titles-in-visualizations.png +:alt: Titles in visualizations +::: diff --git a/docs/extend/data-stream-spec.md b/docs/extend/data-stream-spec.md new file mode 100644 index 00000000000..11ff3ec2e27 --- /dev/null +++ b/docs/extend/data-stream-spec.md @@ -0,0 +1,128 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/data-stream-spec.html +--- + +# data_stream [data-stream-spec] + +Data stream assets, including ingest pipelines, field definitions, metadata, and sample events. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +spec: + additionalContents: false + totalContentsLimit: 500 + contents: + - description: Folder containing a single data stream definition + type: folder + pattern: '^([a-z0-9]{2}|[a-z0-9][a-z0-9_]+[a-z0-9])$' + forbiddenPatterns: + # Avoid collision with ingest pipeline created by fleet, see https://github.com/elastic/package-spec/issues/699 + - '^integration$' + required: true + additionalContents: false + contents: + - description: A data stream's manifest file + type: file + contentMediaType: "application/x-yaml" + sizeLimit: 5MB + name: "manifest.yml" + required: true + $ref: "./manifest.spec.yml" + - description: Folder containing field definitions + type: folder + name: fields + required: true + $ref: "./fields/spec.yml" + - description: Folder containing agent-related definitions + type: folder + name: agent + required: false + additionalContents: false + $ref: "./agent/spec.yml" + - description: Folder containing Elasticsearch assets + type: folder + name: elasticsearch + additionalContents: false + contents: + - description: Folder containing Elasticsearch ILM Policy Definition + type: folder + name: ilm + additionalContents: false + contents: + - description: Supporting ILM policy definitions in YAML + type: file + pattern: '^.+\.yml$' + # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54) + contentMediaType: "application/x-yaml; require-document-dashes=true" + required: false + - description: Supporting ILM policy definitions in JSON + type: file + pattern: '^.+\.json$' + contentMediaType: "application/json" + required: false + - description: Folder containing Elasticsearch Ingest Node pipeline definitions + type: folder + name: ingest_pipeline + additionalContents: false + contents: + - description: Supporting ingest pipeline definitions in YAML + type: file + pattern: '^.+\.yml$' + # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54) + contentMediaType: "application/x-yaml; require-document-dashes=true" + required: false + $ref: "../../integration/elasticsearch/pipeline.spec.yml" + - description: Supporting ingest pipeline definitions in JSON + type: file + pattern: '^.+\.json$' + contentMediaType: "application/json" + required: false + $ref: "../../integration/elasticsearch/pipeline.spec.yml" + - description: Sample event file + type: file + name: "sample_event.json" + contentMediaType: "application/json" + required: false + - description: Folder containing testing related files and sub-folders + type: folder + name: "test" + required: false + - description: Folder containing development resources + type: folder + name: _dev + required: false + visibility: private + $ref: "./_dev/spec.yml" + - description: File containing routing rules definitions (technical preview) + type: file + contentMediaType: "application/x-yaml" + name: "routing_rules.yml" + required: false + $ref: "./routing_rules.spec.yml" + - description: File containing lifecycle configuration (technical preview) + type: file + contentMediaType: "application/x-yaml" + name: "lifecycle.yml" + required: false + $ref: "lifecycle.spec.yml" + +versions: + - before: 3.0.0 + patch: + - op: remove + path: "/contents/0/contents/3/contents/1/contents/0/$ref" # remove ingest pipeline validation as yaml + - op: remove + path: "/contents/0/contents/3/contents/1/contents/1/$ref" # remove ingest pipeline validation as json + - before: 2.10.0 + patch: + - op: remove + path: "/contents/0/contents/8" # remove lifecycle definition + - before: 2.9.0 + patch: + - op: remove + path: "/contents/0/contents/7" # remove routing_rules file definition +``` diff --git a/docs/extend/define-deployment-modes.md b/docs/extend/define-deployment-modes.md new file mode 100644 index 00000000000..8d3e2e4fa05 --- /dev/null +++ b/docs/extend/define-deployment-modes.md @@ -0,0 +1,89 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/define-deployment-modes.html +--- + +# Define deployment modes [define-deployment-modes] + +Some integrations can be deployed on fully managed agents. These integrations are known as "agentless" integrations. Define the deployment mode of an integration with the [`deployment_modes`](#deployment_modes) property and display/hide variables in different deployment modes with the [`hide_in_deployment_modes`](#hide_in_deployment_modes) property. + + +## `deployment_modes` [deployment_modes] + +Policy templates can indicate which deployment modes they support. Use the `deployment_modes` property in the policy template schema to define the supported deployment modes. Options are `default` and `agentless`. A policy template can support both modes. + +Example policy template declaration: + +```yaml +format_version: 3.2.0 +name: aws +title: AWS +version: 2.13.1 +... +policy_templates: + - name: billing + title: AWS Billing + description: Collect billing metrics with Elastic Agent + deployment_modes: <1> + default: + enabled: false <2> + agentless: + enabled: true <3> + data_streams: + - billing + ... +``` + +1. Defines the supported deployment modes +2. Disables agent deployment support +3. Enables agentless deployment support + + + +## `hide_in_deployment_modes` [hide_in_deployment_modes] + +Variables can be hidden in certain deployment modes. Use the `hide_in_deployment_modes` property to opt variables in or out of being displayed in default or agentless mode. This property works at any manifest level. + +Example variable declaration: + +```yaml +streams: + - input: filestream + vars: + - name: paths + type: text + title: Paths + multi: true + required: true + show_user: true + default: + - /var/log/my-package/*.log + - name: agentless_only + type: text + title: Agentless only variable + multi: false + required: false + show_user: true + hide_in_deployment_modes: <1> + - default + - name: hidden_in_agentless + type: text + title: Hidden in agentless variable + multi: false + required: false + show_user: true + hide_in_deployment_modes: <2> + - agentless +``` + +1. Disables visibility of the variable in agent deployment mode +2. Disables visibility of the variable in agentless deployment mode + + +For more information on variable property definitions, refer to [Define variable properties](/extend/finishing-touches.md#define-variable-properties). + + +## Agentless capabilities [agentless-capabilities] + +The capabilities feature protects agentless deployments from allowing undesired inputs to run. A static `capabilities.yml` file defines these allowed and disallowed inputs and is passed to deployed agents. To determine which capabilities are currently allowed on Agentless, refer to [`capabilities.yml`](https://github.com/elastic/agentless-controller/blob/main/controllers/config/capabilities.yml). + diff --git a/docs/extend/dev-spec.md b/docs/extend/dev-spec.md new file mode 100644 index 00000000000..033912d203e --- /dev/null +++ b/docs/extend/dev-spec.md @@ -0,0 +1,39 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/dev-spec.html +--- + +# _dev [dev-spec] + +Development resources. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +spec: + additionalContents: false + developmentFolder: true + contents: + - description: Folder containing resources related to package benchmarks. + type: folder + name: benchmark + required: false + $ref: "./benchmark/spec.yml" + - description: Folder containing resources related to building the package. + type: folder + name: build + required: false + $ref: "./build/spec.yml" + - description: Folder containing configuration related to deploying the package's service(s) required for testing scenarios. + type: folder + name: deploy + required: false + $ref: "./deploy/spec.yml" + - description: Folder containing configuration related test configuration. + type: folder + name: test + required: false + $ref: "./test/spec.yml" +``` diff --git a/docs/extend/developer-tsds-guidelines.md b/docs/extend/developer-tsds-guidelines.md new file mode 100644 index 00000000000..d956b2eeb31 --- /dev/null +++ b/docs/extend/developer-tsds-guidelines.md @@ -0,0 +1,214 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/developer-tsds-guidelines.html +--- + +# TSDS guidelines [developer-tsds-guidelines] + +This page describes how to enable TSDS functionality in your integration packages. Full details about TSDS can be found in [Time series data stream](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md) in the {{es}} documentation. + +In this document you can find: + +* [Background](#integrations-dev-tsds-background) +* [Steps for enabling TSDS for a metrics dataset](#integrations-dev-tsds-migrating) +* [Testing](#integrations-dev-tsds-testing) +* [Best practices](#integrations-dev-tsds-best-practices) +* [Troubleshooting](#integrations-dev-tsds-troubleshooting) + + +## Background [integrations-dev-tsds-background] + +A time series is a sequence of observations for a specific entity. TSDS enables the column-oriented functionality in elasticsearch by co-locating the data and optimizing the storage and aggregations to take advantage of such co-allocation. + +Integrations are one of the biggest sources of input data to Elasticsearch. Enabling TSDS on integration packages can be achieved by minimal changes made in the `fields.yml` and `manifest.yml` files of a package. + + +## Steps for enabling TSDS for a metrics dataset [integrations-dev-tsds-migrating] + +::::{important} +Datastreams having type `logs` are excluded from TSDS migration. +:::: + + + +## Step 1: Set the dimension fields [_step_1_set_the_dimension_fields] + +Each field belonging to the set of fields that uniquely identify a document is a dimension. For more details, refer to [Dimensions](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-dimension). + +To set a field as a dimension simply add `dimension: true` to its mapping: + +```yaml +- name: ApiId + type: keyword + dimension: true +``` + +::::{note} +A field having type [flattened](elasticsearch://docs/reference/elasticsearch/mapping-reference/flattened.md) cannot be selected as a dimension field. If the field that you are choosing as a dimension is too long or is of type flattened, consider hashing the value of this field and using the result as a dimension. [Fingerprint processor](elasticsearch://docs/reference/ingestion-tools/enrich-processor/fingerprint-processor.md) can be used for this purpose. + +You can find an example in [Oracle Integration TSDS Enablement Example](https://github.com/elastic/integrations/blob/8a57d6ba96d391afc33da20c80ec51280d22f009/packages/oracle/data_stream/performance/elasticsearch/ingest_pipeline/default.yml#LL127C4-L131C29) + +:::: + + +Important considerations: + +* There is a limit on how many dimension fields a datastream can have. By default, this value is `21`. You can adjust this restriction by altering the `index.mapping.dimension_fields.limit`: + + ```yaml + elasticsearch: + index_template: + settings: + index.mapping.dimension_fields.limit: 32 # Defaults to 21 + ``` + +* Dimension keys have a hard limit of 512b. Documents are rejected if this limit is reached. +* Dimension values have a hard limit of 1024b. Documents are rejected if this limit is reached. + + +### ECS fields [_ecs_fields] + +There are fields that are part of every package, and they are potential candidates for becoming dimension fields: + +* `host.name` +* `service.address` +* `agent.id` +* `container.id` + +For products that are capable of running both on-premise and in a public cloud environment (by being deployed on public cloud virtual machines), it is recommended to annotate the ECS fields listed below as dimension fields: + +* `host.name` +* `service.address` +* `container.id` +* `cloud.account.id` +* `cloud.provider` +* `cloud.region` +* `cloud.availability_zone` +* `agent.id` +* `cloud.instance.id` + +For products operating as managed services within cloud providers like AWS, Azure, and GCP, it is advised to label the fields listed below as dimension fields: + +* `cloud.account.id` +* `cloud.region` +* `cloud.availability_zone` +* `cloud.provider` +* `agent.id` + +Note that for some packages some of these fields do not hold any value, so make sure to only use the needed ones. + + +### Integration specific fields [_integration_specific_fields] + +The `files.yml` file has the field mappings specific to a datastream of an integration. Some of these fields might need to be set as a dimension if the set of dimension fields in ECS is not enough to create a unique [`_tsid`](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#tsid). + +Adding an inline comment prior to the dimension annotation is advised, detailing the rationale behind the choice of a particular field as a dimension field: + +```yaml +- name: wait_class + type: keyword + # Multiple events are generated based on the values of wait_class. Hence, it is a dimension + dimension: true + description: Every wait event belongs to a class of wait events. +``` + + +## Step 2: Set type for metric fields [_step_2_set_type_for_metric_fields] + +Metrics are fields that contain numeric measurements, as well as aggregations and/or down sampling values based off of those measurements. Annotate each metric with the correct metric type. The [currently supported values](docs-content://manage-data/data-store/data-streams/time-series-data-stream-tsds.md#time-series-metric) are `gauge`, `counter`, and `null`. + +Example of adding a metric type to a field: + +```yaml +- name: compactions_failed + type: double + metric_type: counter + description: | + Counter of TSM compactions by level that have failed due to error. +``` + +::::{note} +Some of the aggregation functions are not supported for certain `metric_type` values. In such a scenario, please revisit to see if the selection of `metric_type` you made is indeed correct for that field. If valid, please create an issue in [elastic/elasticsearch](https://github.com/elastic/elasticsearch) explaining the use case. +:::: + + + +## Step 3: Update Kibana version [_step_3_update_kibana_version] + +Modify the `kibana.version` to at least `8.8.0` in the `manifest.yml` file of the package: + +```yaml +conditions: + kibana.version: "^8.8.0" +``` + + +## Step 4: Enable `time_series` index mode [_step_4_enable_time_series_index_mode] + +Add the changes to the `manifest.yml` file of the datastream as shown to enable the timeseries index mode: + +```yaml +elasticsearch: + index_mode: "time_series" +``` + + +## Testing [integrations-dev-tsds-testing] + +* If the number of dimensions is insufficient, we will have loss of data. Consider testing this using the [TSDS migration test kit](https://github.com/elastic/TSDB-migration-test-kit). +* Verify the dashboard is rendering the data properly. If certain visualisation do not work, consider migrating to [Lens](docs-content://explore-analyze/visualize/lens.md). Remember that certain aggregation functions are not supported when a field has metric type `counter`, for example, `avg()`. Replace such aggregation functions with a supported aggregation type such as `max()` or `min()`. + + +## Best practices [integrations-dev-tsds-best-practices] + +* Use [Lens](docs-content://explore-analyze/visualize/lens.md) as the preferred visualisation type. +* Always assess the number of unique values the field that is selected to be a dimension would hold, especially if it is a numeric field. A field that holds millions of unique values may not be an ideal candidate for becoming a dimension field. +* If the dimension field value length is very long (max limit is 1024B), consider transforming the value to hash value representation. [Fingerprint processor](elasticsearch://docs/reference/ingestion-tools/enrich-processor/fingerprint-processor.md) can be used for this purpose. +* In the field mapping files above each dimension field, add in-line comments stating the reason for selecting the field as a dimension field. +* As part of TSDS migration testing, you may discover other errors which may be unrelated to TSDS migration. Keep the pull request for TSDS migration free from such changes. This helps in obtaining quick PR approval. + + +## Troubleshooting [integrations-dev-tsds-troubleshooting] + + +### Dropped documents [_dropped_documents] + +In the event that after enabling TSDS you notice that metrics data is being dropped from an index, the [TSDS test migration kit](https://github.com/elastic/TSDB-migration-test-kit) can be used as a helpful debugging tool. + + +### Conflicting field type [_conflicting_field_type] + +Fields having conflicting field type will not be considered as dimension. Resolve the field type ambiguity before defining a field as dimension field. + + +### Identification of write index [_identification_of_write_index] + +When mappings are modified for a datastream, index rollover happens and a new index is created under the datastream. Even if there exists a new index, the data continues to go to the old index until the timestamp matches `index.time_series.start_time` of the newly created index. + +An [enhancement request](https://github.com/elastic/kibana/issues/150549) for Kibana is created to indicate the write index. Until then, refer to the index.time_series.start_time of indices and compare with the current time to identify the write index. + +If you find this error (for reference, see [integrations issue #7345](https://github.com/elastic/integrations/issues/7345) and [elasticsearch PR #98518](https://github.com/elastic/elasticsearch/pull/98518)): + +```console +... (status=400): {"type":"illegal_argument_exception","reason":"the document timestamp [2023-08-07T00:00:00.000Z] is outside of ranges of currently writable indices [[2023-08-07T08:55:38.000Z,2023-08-07T12:55:38.000Z]]"}, dropping event! +``` + +Consider: + +1. Defining the `look_ahead` or `look_back_time` for each data stream. For example: + + ```yaml + elasticsearch: + index_mode: "time_series" + index_template: + settings: + index.look_ahead_time: "10h" + ``` + + ::::{note} + Updating the package with this does not cause an automatic rollover on the data stream. You have to do that manually. + :::: + +2. Updating the `timestamp` of the document being rejected. +3. Finding a fix to receive the document without a delay. + diff --git a/docs/extend/developer-workflow-fleet-UI.md b/docs/extend/developer-workflow-fleet-UI.md new file mode 100644 index 00000000000..c4b9d12f6f7 --- /dev/null +++ b/docs/extend/developer-workflow-fleet-UI.md @@ -0,0 +1,105 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-fleet-UI.html +--- + +# Development process for Fleet UI [developer-workflow-fleet-UI] + +See the Kibana docs for [how to set up your dev environment](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#setting-up-your-development-environment), [run Elasticsearch](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#running-elasticsearch), and [start Kibana](https://github.com/elastic/kibana/blob/main/CONTRIBUTING.md#running-kibana). + +One common development workflow is: + +1. Clone Kibana repo + + ```bash + git clone https://github.com/[YOUR_USERNAME]/kibana.git kibana + cd kibana + ``` + +2. Install Dependencies + + ```bash + nvm use + npm install -g yarn + ``` + +3. Bootstrap Kibana + + ```bash + yarn kbn bootstrap + ``` + +4. Start Elasticsearch in one shell + + ```bash + yarn es snapshot -E xpack.security.authc.api_key.enabled=true + ``` + +5. Start Kibana in another shell + + ```bash + yarn start --xpack.fleet.enabled=true --no-base-path + ``` + +6. Download fleet-server package from [https://www.elastic.co/downloads/past-releases/#elastic-agent](https://www.elastic.co/downloads/past-releases/#elastic-agent) +7. Untar fleet server tarball and `cd` to the directory +8. Install fleet-server (See also the alternative solution) + + ```bash + sudo ./elastic-agent install -f \ + --fleet-server-es=http://elastic:changeme@localhost:9200 \ + --fleet-server-policy= + ``` + + The `default policy id` can be retrieved by fleet ui instructions in Kibana before any fleet server is installed. Fleet Server will start in `+https://users_machine_ip:8220+` + +9. Update Fleet settings on the top right corner of Fleet UI to set the correct Fleet Server hosts (ip from previous step). +10. After that user can enroll as many agents as they want +11. Any code update in Kibana fleet plugin should be picked up automatically and either cause the server to restart, or be served to the browser on the next page refresh. + + +## Alternative solution for fleet server [_alternative_solution_for_fleet_server] + +Instead of download fleet server package and running it as a local process you can run Fleet Server Locally in a Container. + +It can be useful to run Fleet Server in a container on your local machine in order to free up your actual "bare metal" machine to run Elastic Agent for testing purposes. Otherwise, you’ll only be able to a single instance of Elastic Agent dedicated to Fleet Server on your local machine, and this can make testing integrations and policies difficult. + +*The following is adapted from the Fleet Server [README](https://github.com/elastic/fleet-server#running-elastic-agent-with-fleet-server-in-container)* + +1. Add the following configuration to your `config/kibana.yml` + + ```yaml + server.host: 0.0.0.0 + ``` + +2. Append the following option to the command you use to start Elasticsearch + + ```yaml + -E http.host=0.0.0.0 + ``` + + This command should look something like this: + + ```bash + yarn es snapshot --license trial -E xpack.security.authc.api_key.enabled=true -E path.data=/tmp/es-data -E http.host=0.0.0.0 + ``` + +3. Run the Fleet Server Docker container. Make sure you include a `BASE-PATH` value if your local Kibana instance is using one. `YOUR-IP` should correspond to the IP address used by your Docker network to represent the host. For Windows and Mac machines, this should be `192.168.65.2`. If you’re not sure what this IP should be, run the following to look it up: + + ```bash + docker run -it --rm alpine nslookup host.docker.internal + ``` + + To run the Fleet Server Docker container: + + ```bash + docker run -e KIBANA_HOST=http://{YOUR-IP}:5601/{BASE-PATH} -e KIBANA_USERNAME=elastic -e KIBANA_PASSWORD=changeme -e ELASTICSEARCH_HOST=http://{YOUR-IP}:9200 -e ELASTICSEARCH_USERNAME=elastic -e ELASTICSEARCH_PASSWORD=changeme -e KIBANA_FLEET_SETUP=1 -e FLEET_SERVER_ENABLE=1 -e FLEET_SERVER_INSECURE_HTTP=1 -p 8220:8220 docker.elastic.co/elastic-agent/elastic-agent:{VERSION} + ``` + + Ensure you provide the `-p 8220:8220` port mapping to map the Fleet Server container’s port `8220` to your local machine’s port `8220` in order for Fleet to communicate with Fleet Server. + + For the latest version, use `8.0.0-SNAPSHOT`. Otherwise, you can explore the available versions at [https://www.docker.elastic.co/r/beats/elastic-agent](https://www.docker.elastic.co/r/beats/elastic-agent). + + Once the Fleet Server container is running, you should be able to treat it as if it were a local process running on `+http://localhost:8220+` when configuring Fleet via the UI. You can then run `elastic-agent` on your local machine directly for testing purposes. + + diff --git a/docs/extend/developer-workflow-import-beat.md b/docs/extend/developer-workflow-import-beat.md new file mode 100644 index 00000000000..47bea4b5077 --- /dev/null +++ b/docs/extend/developer-workflow-import-beat.md @@ -0,0 +1,172 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-import-beat.html +--- + +# Import integration from Beats modules [developer-workflow-import-beat] + +The import procedure heavily uses on the *import-beats* script. If you are interested how does it work internally, feel free to review the script’s [README](https://github.com/elastic/integrations/tree/main/dev/import-beats/README.md). + +1. Create an issue in the [integrations](https://github.com/elastic/integrations) to track ongoing progress with the integration (especially manual changes). + + Focus on the one particular product (e.g. MySQL, ActiveMQ) you would like to integrate with. Use this issue to mention every manual change that has been applied. It will help in adjusting the `import-beats` script and reviewing the integration. + +2. Prepare the developer environment: + + 1. Clone/refresh the following repositories: + + * [https://github.com/elastic/beats](https://github.com/elastic/beats) + * [https://github.com/elastic/ecs](https://github.com/elastic/ecs) + * [https://github.com/elastic/eui](https://github.com/elastic/eui) + * [https://github.com/elastic/kibana](https://github.com/elastic/kibana) + + Make sure you don’t have any manual changes applied as they will reflect on the integration. + + 2. Clone/refresh the Elastic Integrations to always use the latest version of the script: + + * [https://github.com/elastic/integrations](https://github.com/elastic/integrations) + + 3. Make sure you’ve the `mage` tool installed: + + ```bash + $ go get -u -d github.com/magefile/mage + ``` + +3. Use the `elastic-package stack up -v -d` command to boot up required dependencies: + + 1. Elasticseach instance: + + * Kibana’s dependency + + 2. Kibana instance: + + * used to migrate dashboards, if not available, you can skip the generation (`SKIP_KIBANA=true`) + + *Hint*. There is the `elastic-package` cheat sheet available [here](https://github.com/elastic/integrations/blob/main/testing/environments/README.md). + +4. Create a new branch for the integration in `integrations` repository (diverge from main). +5. Run the command: `mage ImportBeats` to start the import process (note that the import script assumes the projects checked out in step 2 are at `+../{{project-name}}+`). + + The outcome of running the `import-beats` script is directory with refreshed and updated integrations. + + It will take a while to finish, but the console output should be updated frequently to track the progress. The command should terminate with an exit code of 0. If it doesn’t, please open an issue. + + Generated packages are stored by default in the `packages` directory. Generally, the import process updates all of the integrations, so don’t be surprised if you notice updates to multiple integrations, including the one you’re currently working on (e.g. `packages/foobarbaz`). You can either commit these changes or leave them for later. + + If you want to select a subgroup of packages, set the environment variable `PACKAGES` (comma-delimited list): + + ```bash + $ PACKAGES=aws,cisco mage ImportBeats + ``` + + + +## Fine tune the integration [_fine_tune_the_integration] + +Most of migration work has been done by the `import-beats` script, but there’re tasks that require developer’s interaction. + +It may happen that your integration misses a screenshot or an icon, it’s a good moment to add missing resources to Beats/Kibana repositories and re-import the integration (idempotent). + + +### Checklist [_checklist] + +The order of action items on the checklist is advised to prevent the contributor from repeating some actions (fixing what’s been already fixed, as the script has overridden part of it). + +1. Add icon if missing. + + The integration icons are presented in different places in Kibana, hence it’s better to define custom icons to make the UI easier to navigate. + + As the `import-beats` script looks for icons in Kibana and EUI repositories, add an icon to the first one the same way as for tutorial resources (Kibana directory: `src/legacy/core_plugins/kibana/public/home/tutorial_resources/logos/`). + +2. Add screenshot if missing. + + The Kibana Integration Manager shows screenshots related with the integration. Screenshots present Kibana dashboards visualizing the metric/log data. + + The `import-beats` script finds references to screenshots mentioned in `_meta/docs.asciidoc` and copies image files from the Beats directories: + + * `metricbeat/docs/images` + * `filebeat/docs/images` + +3. Improve/correct spelling product names. + + The correct spelling of product names simply makes better impression. The `import-beats` scripts uses the `fields.yml` file as the source of the correct spelling (`title` property), e.g. Mysql - MySQL, Nginx - NGINX, Aws - AWS. + + Keep in mind that this step requires reimporting package contents. + +4. Write README template file for the integration. + + The README template is used to render the final README file including exported fields. The template should be placed in the `package//_dev/build/docs/README.md`. If the directory doesn’t exist, please create it. + + Review the MySQL docs template to see how to use template functions (e.g. `{{fields "data-stream-name"}}`). If the same data stream name is used in both metrics and logs, please add `-metrics` and `-logs` in the template. For example, `elb` is a data stream for log and also a data stream for metrics. In README.md template, `{{fields "elb_logs"}}` and `{{fields "elb_metrics"}}` are used to separate them. + +5. Review fields file and exported fields in docs. + + The goal of this action item is to verify if produced artifacts are correct. + + The fields files (package-fields.yml, fields.yml and ecs.yml) in the package were created from original fields.yml files (that may contain ECS schema fields) and fields.epr.yml (defining some other fields used in the ingest pipeline). It may happen that original sources have a typo, bad description or misses a field definition. The sum of fields in all present files should contain only fields that are really used, e.g. not all existing ECS fields. + + It may happen that the ingest pipeline uses fields abstracted from ECS, but not mentioned in `fields.yml`. Integrations should contain these fields and also have them documented. + + The fields for an integration package are divided into the following three files: + + * ecs.yml: ECS compliant fields that are used by this particular data stream. + * package-fields.yml: Package level fields that are used by this particular data stream, which does not exist under `.`. + * fields.yml: Dataset level fields that are specific to this particular data stream, and non ECS compliant. + + See the PR [https://github.com/elastic/beats/pull/17895](https://github.com/elastic/beats/pull/17895) to understand how to add them to Beats (e.g. `event.code`, `event.provider`) using the `fields.epr.yml` file. + +6. Metricbeat: add missing configuration options. + + The `import-beats` script extracts configuration options from Metricbeat module’s `_meta` directory. It analyzes the configuration files and selects options based on enabled metricsets (not commented). If you notice that some configuration options are missing in your package’s manifest files, simply create the `config.epr.yml` file with all required options. + + Sample PR: [https://github.com/elastic/beats/pull/17323](https://github.com/elastic/beats/pull/17323) + +7. Review *titles* and *descriptions* in manifest files. + + Titles and descriptions are fields visualized in the Kibana UI. Most users will use them to see how to configure the integration with their installation of a product or to how to use advanced configuration options. + +8. Compact configuration options (vars). + + Currently, all configuration options are set by the `import-beats` script on the stream level (path: `data stream//manifest.yml`). + + It may happen that some of them in different data streams are simply duplicates or concern the same setting, which will be always equal (e.g. MySQL username, password). Keep in mind that two data streams may have the same configuration option, but different values (e.g. `period`, `paths`), hence can’t be compacted. + + To sum up, compacting takes down from the user the necessity to setup the same configuration option few times (one per data stream). + +9. Define all variable properties. + + The variable properties customize visualization of configuration options in the Kibana UI. Make sure they’re defined in all manifest files. + + ```yaml + vars: + - name: paths + required: true + show_user: true + title: Access log paths + description: Paths to the nginx access log file. + type: text + multi: true + default: + - /var/log/nginx/access.log* + ``` + + * **required** - option is required + * **show_user** - don’t hide the configuration option (collapsed menu) + * **title** - human readable variable name + * **description** - variable description (may contain some details) + * **type** - field type (according to the reference: text, password, bool, integer) + * **multi** - the field has mutliple values. + +10. Review stream configuration. + + Due to changed templating engine from a standard Golang one to [handlebars](https://handlebarsjs.com/), it may be hard to automatically convert the Filebeat input configuration (nested variables, many representations, conditions, loops). Please review the output stream configuration and identify potential bugs. + +11. Update docs template with sample events. + + The events collected by the agent slightly differ from the original, Metricbeat and Filebeat, ones. Adjust the event content manually basing on already migrated integrations (e.g. [MySQL integration](https://github.com/elastic/integrations/blob/main/packages/mysql/_dev/build/docs/README.md)) or copy them once managed to run whole setup with real agent. + +12. Kibana: use `stream.data stream` field instead of `event.data stream`. + + Using `stream.data stream` instead of `event.data stream` also makes queries a lot more efficient as this is a `constant_keyword`. Make sure that dashboards in your package don’t use the `event.data stream` field. If so, simply replace them with the more efficient one. + + diff --git a/docs/extend/developer-workflow-support-old-package.md b/docs/extend/developer-workflow-support-old-package.md new file mode 100644 index 00000000000..0355befeaba --- /dev/null +++ b/docs/extend/developer-workflow-support-old-package.md @@ -0,0 +1,122 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflow-support-old-package.html +--- + +# Release a bug fix for supporting older package version [developer-workflow-support-old-package] + +In some cases, when we drop the support for an older version of the stack and later on find out needing to add a bug fix to the some old package version, we have to make some manual changes to release the bug fix to users. For example: in this [PR](https://github.com/elastic/integrations/pull/3688) (AWS package version 1.23.4), support for Kibana version 7.x was dropped and bumped the AWS package version from 1.19.5 to 1.20.0. But we found a bug in the EC2 dashboard that needs to be fixed with Kibana version 7.x. So instead of adding a new AWS package version 1.23.5, we need to fix it between 1.19.5 and 1.20.0. + +Follow these detailed steps to release a fix for a given package version: + +1. **Find git commit (package version) that needs to be fixed** + + In the example above, the commit to be fixed is the one right before this [PR](https://github.com/elastic/integrations/pull/3688) updating package `aws`: + + * Using the web: + + * Look for the merge commit of the PR + + * [https://github.com/elastic/integrations/commit/aa63e1f6a61d2a017e1f88af2735db129cc68e0c](https://github.com/elastic/integrations/commit/aa63e1f6a61d2a017e1f88af2735db129cc68e0c) + * It can be found as one of the last messages in the PR ![merged commit](../images/merge_commit_message.png "") + * And then show the previous commits for that changeset inside the package folder (e.g. `packages/aws`): + * [https://github.com/elastic/integrations/commits/aa63e1f6a61d2a017e1f88af2735db129cc68e0c/packages/aws/](https://github.com/elastic/integrations/commits/aa63e1f6a61d2a017e1f88af2735db129cc68e0c/packages/aws/) ![commits from package](../images/browse_package_commits.png "") + + * Using the command line: + + ```bash + cd packages/ + git log --grep "#" . + git log -n 1 ^ . + + # following the example + $ cd packages/aws + $ git log --grep "#3688" + commit aa63e1f6a61d2a017e1f88af2735db129cc68e0c + Author: Joe Reuter + Date: Mon Aug 8 17:14:55 2022 +0200 + + Inline all aws dashboards (#3688) + + * inline all aws dashboards + + * format + + * apply the right format + + * inline again + + * format + $ git log -n 1 aa63e1f6a61d2a017e1f88af2735db129cc68e0c^ . + commit 8cb321075afb9b77ea965e1373a03a603d9c9796 + Author: Mario Castro + Date: Thu Aug 4 16:52:06 2022 +0200 + + Move lightweight manifest to integration for EBS data stream (#3856) + ``` + +2. Run the **integrations-backport** pipeline [https://buildkite.com/elastic/integrations-backport](https://buildkite.com/elastic/integrations-backport) for creating the backport branch. ![buildkite buid](../images/build.png "") + + **Please, pay attention!**, if you just run the pipeline it’ll wait for your inputs, nothing will happen without that. + + :::{image} ../images/backport_input_step.png + :alt: waiting input step + ::: + + Pipeline’s inputs: + + * **DRY_RUN** (default: "true"), If DRY_RUN is defined as "true" it will check: + + * if the package is published, + * if the entered commit exists, + * if the backport branch exists. Also, it will create the local branch, update the branch with `.buildkite` and `.ci` folders, and remove other packages except the defined one (if set as input). This local branch will not be pushed to the upstream repository in this mode. + + + If DRY_RUN is defined as "false", in addition to written above it will create a commit and push the local branch to the upstream repository [https://github.com/elastic/integrations.git](https://github.com/elastic/integrations.git). In this case, the name of the branch will be `+backport-${PACKAGE_NAME}-${TRIMMED_PACKAGE_VERSION}+`, for example, `backport-aws-1.19`. + + * **BASE_COMMIT** (default: "") - enter the commit from the previous step (8cb321075afb9b77ea965e1373a03a603d9c9796) + * **PACKAGE_NAME** (default: "") - enter the package name, for example aws + * **PACKAGE_VERSION** (default: "") - enter the package version, for example: 1.19.7, 1.0.0-beta1 + * **REMOVE_OTHER_PACKAGES** (default: "false") If **REMOVE_OTHER_PACKAGES** is defined as "true" all packages from the **packages** folder, except the defined package, will be removed from the created branch. + +3. **Create a PR for the bug fix** + + Create a new branch in your own remote (it is advised **not using** a branch name starting with `backport-`), and apply bugfixes there. Remember to update the version in the package manifest (update patch version like `1.19.`) and add a new changelog entry for this patch version. + + Once ready, open a PR selecting as a base branch the one created above: `backport--.` (e.g. `backport-aws-1.19`). + + Once this PR is merged, this new version of the package is going to be published automatically following the usual CI/CD jobs. + + If it is needed to release a new fix for that version, there is no need to create a new branch. Just create a new PR to merge a new branch onto the same backport branch created previously. + +4. **Update changelog in main** + + Once PR has been merged in the corresponding backport branch (e.g. `backport-aws-1.9`) and the package has been published, a new Pull Request should be created manually to update the changelog in the main branch to include the new version published in the backport branch. Take into account to add the changelog entry following the version order. + + In order to keep track, this new PR should have a reference (relates) to the backport PR too in its description. + +5. **Known issues and their solutions:** + + 1. Missing shellinit command: + + * Example of the error: [https://buildkite.com/elastic/integrations/builds/7634#018c87f4-7b0c-4d6f-8ddd-b779a9a7a019/507-512](https://buildkite.com/elastic/integrations/builds/7634#018c87f4-7b0c-4d6f-8ddd-b779a9a7a019/507-512) + + `Error: could not create kibana client: undefined environment variable: ELASTIC_PACKAGE_KIBANA_HOST. If you have started the Elastic stack using the elastic-package tool, please load stack environment variables using 'eval "$(elastic-package stack shellinit)"' or set their values manually` + + * **Solution**: add elastic-package stack shellinit command in `.buildkite/scripts/common.sh`. + + * `eval "$(elastic-package stack shellinit)"` + + Example: [https://github.com/elastic/integrations/blob/0226f93e0b1493d963a297e2072f79431f6cc443/.buildkite/scripts/common.sh#L828](https://github.com/elastic/integrations/blob/0226f93e0b1493d963a297e2072f79431f6cc443/.buildkite/scripts/common.sh#L828) + + 2. Not found license file: + + * Example of the error: [https://buildkite.com/elastic/integrations/builds/7644#018c883c-546f-4d32-ab4a-71e919ddebf8/270-309](https://buildkite.com/elastic/integrations/builds/7644#018c883c-546f-4d32-ab4a-71e919ddebf8/270-309) + + `Error: checking package failed: building package failed: copying license text file: failure while looking for license "licenses/Elastic-2.0.txt" in repository: failed to find repository license: stat /opt/buildkite-agent/builds/bk-agent-prod-gcp-1703092724145948143/elastic/integrations/licenses/Elastic-2.0.txt: no such file or directory` + + * **Solution**: Remove line defining `ELASTIC_PACKAGE_REPOSITORY_LICENSE` environment variable. + + * Example: [https://github.com/elastic/integrations/blob/0daff27f0e0195a483771a50d60ab28ca2830f75/.buildkite/pipeline.yml#L17](https://github.com/elastic/integrations/blob/0daff27f0e0195a483771a50d60ab28ca2830f75/.buildkite/pipeline.yml#L17) + + diff --git a/docs/extend/developer-workflows.md b/docs/extend/developer-workflows.md new file mode 100644 index 00000000000..23e63221d31 --- /dev/null +++ b/docs/extend/developer-workflows.md @@ -0,0 +1,14 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/developer-workflows.html +--- + +# Developer workflows [developer-workflows] + +* [Development process for Fleet UI](/extend/developer-workflow-fleet-UI.md) +* [Release a bug fix for supporting older package version](/extend/developer-workflow-support-old-package.md) +* [Import integration from Beats modules](/extend/developer-workflow-import-beat.md) + + + + diff --git a/docs/extend/docs-spec.md b/docs/extend/docs-spec.md new file mode 100644 index 00000000000..0c9db45c45c --- /dev/null +++ b/docs/extend/docs-spec.md @@ -0,0 +1,28 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/docs-spec.html +--- + +# docs [docs-spec] + +The built integration README file. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +spec: + additionalContents: false + contents: + - description: Main README file + type: file + contentMediaType: "text/markdown" + name: "README.md" + required: true + - description: Other README files (can be used by policy templates) + type: file + contentMediaType: "text/markdown" + pattern: '^.+.md' + required: false +``` diff --git a/docs/extend/documentation-guidelines.md b/docs/extend/documentation-guidelines.md new file mode 100644 index 00000000000..ea5d9cd7537 --- /dev/null +++ b/docs/extend/documentation-guidelines.md @@ -0,0 +1,276 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/documentation-guidelines.html +--- + +# Documentation guidelines [documentation-guidelines] + +The goal of each integration’s documentation is to: + +* Help the reader understand the benefits the integration offers and how Elastic can help with their use case. Inform the reader of any requirements, including system compatibility, supported versions of third-party products, permissions needed, and more. +* Provide a comprehensive list of collected fields and the data and metric types for each. The reader can reference this information while evaluating the integration, interpreting collected data, or troubleshooting issues. +* Set the reader up for a successful installation and setup by connecting them with any other resources they’ll need. +* Each integration document should contain several sections, and you should use consistent headings to make it easier for a single user to evaluate and use multiple integrations. + + * [Overview](#idg-docs-guidelines-overview) + * [Datastreams](#idg-docs-guidelines-datastreams) + * [Requirements](#idg-docs-guidelines-requirements) + * [Setup](#idg-docs-guidelines-setup) + * [Troubleshooting (optional)](#idg-docs-guidelines-troubleshooting) + * [Reference](#idg-docs-guidelines-reference) + + +Some considerations when these documentation files are written at `_dev/build/docs/*.md`: + +* These files follow the Markdown syntax and leverage the use of [documentation templates](https://github.com/elastic/elastic-package/blob/main/docs/howto/add_package_readme.md). +* There are some available functions or placeholders (`fields`, `event`, `url`) that can be used to help you write documentation. For more detail, refer to [placeholders](https://github.com/elastic/elastic-package/blob/main/docs/howto/add_package_readme.md#placeholders). +* Regarding the `url` placeholder, this placeholder should be used to add links to the [Elastic documentation guides](https://www.elastic.co/guide/index.html) in your documentation: + + * The file containing all of the defined links is in the root of the directory: [`links_table.yml`](https://github.com/elastic/elastic-package/blob/main/scripts/links_table.yml) + * If needed, more links to Elastic documentation guides can be added into that file. + * Example usage: + + * In the documentation files (`_dev/build/docs/*.md`), `{{ url "getting-started-observability" "Elastic guide" }}` generates a link to the Observability Getting Started guide. + + + +### Overview [idg-docs-guidelines-overview] + +The overview section explains what the integration is, defines the third-party product that is providing data, establishes its relationship to the larger ecosystem of Elastic products, and helps the reader understand how it can be used to solve a tangible problem. + +The overview should answer the following questions: + +* What is the integration? +* What is the third-party product that is providing data? +* What can you do with it? + + * General description + * Basic example + + + +#### Template [_template] + +Use this template language as a starting point, replacing `` with details about the integration: + +```text +The integration allows you to monitor . is . + +Use the integration to . Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference when troubleshooting an issue. + +For example, if you wanted to you could . Then you can by . +``` + + +#### Example [_example] + +```text +The AWS CloudFront integration allows you to monitor your AWS CloudFront usage. AWS CloudFront is a content delivery network (CDN) service. + +Use the AWS CloudFront integration to collect and parse logs related to content delivery. Then visualize that data in Kibana, create alerts to notify you if something goes wrong, and reference logs when troubleshooting an issue. + +For example, you could use the data from this integration to know when there are more than some number of failed requests for a single piece of content in a given time period. You could also use the data to troubleshoot the underlying issue by looking at additional context in the logs like the number of unique users (by IP address) who experienced the issue, the source of the request, and more. +``` + + +### Datastreams [idg-docs-guidelines-datastreams] + +The data streams section provides a high-level overview of the kind of data that is collected by the integration. This is helpful since it can be difficult to quickly derive an understanding from just the reference sections (since they’re so long). + +The data streams section should include: + +* A list of the types of data streams collected by the integration +* A summary of each type of data stream included and a link to the relevant reference section: + + * Logs + * Metrics + +* Notes (optional) + + +#### Template [_template_2] + +Use this template language as a starting point, replacing `` with details about the integration: + +```text +## Data streams + +The integration collects two types of data streams: logs and metrics. + +**Logs** help you keep a record of events happening in . +Log data streams collected by the integration include and more. See more details in the [Metrics]<#metrics-reference>. + + + + +``` + + +#### Example [_example_2] + +```text +The System integration collects two types of data: logs and metrics. + +Logs help you keep a record of events that happen on your machine. Log data streams collected by the System integration include application, system, and security events on machines running Windows or auth and syslog events on machines running macOS or Linux. See more details in the Logs reference. + +Metrics give you insight into the state of the machine. Metric data streams collected by the System integration include CPU usage, load statistics, memory usage, information on network behavior, and more. See more details in the Metrics reference. + +You can enable and disable individual data streams. If all data streams are disabled and the System integration is still enabled, Fleet uses the default data streams. +``` + + +### Requirements [idg-docs-guidelines-requirements] + +The requirements section helps readers to confirm that the integration will work with their systems. + +* Elastic prerequisites (for example, a self-managed or Cloud deployment) +* System compatibility +* Supported versions of third-party products +* Permissions needed +* Anything else that could block a user from successfully using the integration + + +#### Template [_template_3] + +Use this template language as a starting point, including any other requirements for the integration: + +```text +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + +``` + + +#### Example [_example_3] + +```text +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + +Each data stream collects different kinds of metric data, which may require dedicated permissions to be fetched and may vary across operating systems. Details on the permissions needed for each data stream are available in the Metrics reference. +``` + +For a much more detailed example, refer to the [AWS integration requirements](https://github.com/elastic/integrations/blob/main/packages/aws/_dev/build/docs/README.md#requirements). + + +### Setup [idg-docs-guidelines-setup] + +The setup section points the reader to the Observability [Getting started guide](docs-content://solutions/observability/get-started.md) for generic, step-by-step instructions. + +This section should also include any additional setup instructions beyond what’s included in the guide, which may include instructions to update the configuration of a third-party service. For example, for the Cisco ASA integration, users need to configure their Cisco device following the [steps found in the Cisco documentation](https://documentation.meraki.com/General_Administration/Monitoring_and_Reporting/Syslog_Server_Overview_and_Configuration#Configuring_a_Syslog_Server). + +::::{note} +When possible, use links to point to third-party documentation for configuring non-Elastic products since workflows may change without notice. +:::: + + + +#### Template [_template_4] + +Use this template language as a starting point, including any other setup instructions for the integration: + +```text +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +{{ url "getting-started-observability" "Getting started" }} guide. + + +``` + + +#### Example [_example_4] + +```text +Before sending logs to Elastic from your Cisco device, you must configure your device according to <>. + +After you've configured your device, you can set up the Elastic integration. For step-by-step instructions on how to set up an integration, see the <> guide. +``` + + +### Troubleshooting (optional) [idg-docs-guidelines-troubleshooting] + +The troubleshooting section is optional. It should contain information about special cases and exceptions that aren’t necessary for getting started or won’t be applicable to all users. + + +#### Template [_template_5] + +There is no standard format for the troubleshooting section. + + +#### Example [_example_5] + +```text +>Note that certain data streams may access `/proc` to gather process information, +>and the resulting `ptrace_may_access()` call by the kernel to check for +>permissions can be blocked by +>[AppArmor and other LSM software](https://gitlab.com/apparmor/apparmor/wikis/TechnicalDoc_Proc_and_ptrace), even though the System module doesn't use `ptrace` directly. +> +>In addition, when running inside a container the proc filesystem directory of the host +>should be set using `system.hostfs` setting to `/hostfs`. +``` + + +### Reference [idg-docs-guidelines-reference] + +Readers might use the reference section while evaluating the integration, interpreting collected data, or troubleshooting issues. + +There can be any number of reference sections (for example, `## Metrics reference`, `## Logs reference`). Each reference section can contain one or more subsections, such as one for each individual data stream (for example, `### Access Logs` and `### Error logs`). + +Each reference section should contain detailed information about: + +* A list of the log or metric types we support within the integration and a link to the relevant third-party documentation. +* (Optional) An example event in JSON format. +* Exported fields for logs, metrics, and events with actual types (for example, `counters`, `gauges`, `histograms` vs. `longs` and `doubles`). Fields should be generated using the instructions in [Fine-tune the integration](https://github.com/elastic/integrations/blob/main/docs/fine_tune_integration.md). +* ML Modules jobs. + + +#### Template [_template_6] + +```text + +## reference + + +## + +The `` data stream provides events from of the following types: . + + + + + + +### Exported fields + + +``` + + +#### Example [_example_6] + +```text +>## Logs reference +> +>### PAN-OS +> +>The `panos` data stream provides events from Palo Alto Networks device of the following types: [GlobalProtect](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/globalprotect-log-fields), [HIP Match](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/hip-match-log-fields), [Threat](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/threat-log-fields), [Traffic](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/traffic-log-fields) and [User-ID](https://docs.paloaltonetworks.com/pan-os/10-2/pan-os-admin/monitoring/use-syslog-for-monitoring/syslog-field-descriptions/user-id-log-fields). +> +>#### Example +> +>An example event for `panos` looks as following: +> +>(code block) +> +>#### Exported fields +> +>(table of fields) +``` + diff --git a/docs/extend/edit-ingest-pipeline.md b/docs/extend/edit-ingest-pipeline.md new file mode 100644 index 00000000000..d6050e8c68c --- /dev/null +++ b/docs/extend/edit-ingest-pipeline.md @@ -0,0 +1,55 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/edit-ingest-pipeline.html +--- + +# Edit ingest pipelines [edit-ingest-pipeline] + +In most instances, before you ingest data into the {{stack}}, the data needs to be manipulated. For example, you should parse your logs into structured data before ingestion. To do so, integrations use **ingest pipelines**. + +::::{admonition} +**Ingest pipelines** let you perform common transformations on your data before indexing. For example, you can use pipelines to remove fields, extract values from text, and enrich your data. + +A pipeline consists of a series of configurable tasks called processors. Each processor runs sequentially, making specific changes to incoming documents. After the processors have run, {{es}} adds the transformed documents to your data stream or index. + +Learn more in the [ingest pipeline reference](docs-content://manage-data/ingest/transform-enrich/ingest-pipelines.md). + +:::: + + +Ingest pipelines are defined in the `elasticsearch/ingest_pipeline` directory. They only apply to the parent data stream within which they live. For our example, this would be the `apache.access` dataset. + +For example, the [Apache integration](https://github.com/elastic/integrations/tree/main/packages/apache): + +```text +apache +└───data_stream +│ └───access +│ │ └───elasticsearch/ingest_pipeline +│ │ default.yml <1> +│ └───error +│ └───status +``` + +1. The ingest pipeline definition for the access logs data stream of the Apache integration + + +An ingest pipeline definition requires a description and an array of processors. Here’s a snippet of the access logs ingest pipeline: + +```yaml +description: "Pipeline for parsing Apache HTTP Server access logs." +processors: +- set: + field: event.ingested + value: '{{_ingest.timestamp}}' +- rename: + field: message + target_field: event.original +- remove: + field: apache.access.time + ignore_failure: true +``` + +Open each `elasticsearch/ingest_pipeline/default.yml` file created for each data stream. Edit each ingest pipeline to match your needs. + +The [processor reference](elasticsearch://docs/reference/ingestion-tools/enrich-processor/index.md) provides a list of all available processors and their configurations. diff --git a/docs/extend/elastic-package.md b/docs/extend/elastic-package.md new file mode 100644 index 00000000000..48722944d63 --- /dev/null +++ b/docs/extend/elastic-package.md @@ -0,0 +1,291 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/elastic-package.html +--- + +# elastic-package reference [elastic-package] + +`elastic-package` is a command line tool, written in Go, used for developing Elastic packages. It can help you lint, format, test, build, and promote your packages. + + +## Get started [elastic-package-start] + +1. Download and build the latest main of elastic-package binary: + + ```bash + git clone https://github.com/elastic/elastic-package.git + make build + ``` + + ::::{tip} + Make sure that you’ve correctly set up the [`$GOPATH` and `$PATH`](https://golang.org/doc/gopath_code.md#GOPATH) environment variables. `elastic-package` must be accessible from your `$PATH`. + :::: + +2. Change into the directory of the package under development: + + ```bash + cd my-package + ``` + +3. Run the `help` command to see available commands + + ```bash + elastic-package help + ``` + + + +## Command reference [elastic-package-command-reference] + +The following `elastic-package` commands are available. For more details on a specific command, run `elastic-package help `. + +Some commands have a *global context*, meaning that you can execute them from anywhere. Other commands have a *package context*; these must be executed from somewhere under a package root folder, and the command will only operate on the contents of that package. + + +### `elastic-package help` [_elastic_package_help] + +*Context: global* + +Use this command to list all commands available under `elastic-package` and a brief description of what each command does. + + +### `elastic-package build` [_elastic_package_build] + +*Context: package* + +Use this command to build a package. Currently, it supports only the "integration" package type. + +Built packages are stored in the "build/" folder located at the root folder of the local Git repository checkout that contains your package folder. The command will also render the README file in your package folder if a corresponding template file present in `_dev/build/docs/README.md`. All `_dev` directories under your package will be omitted. + +Built packages are served up by the {{package-registry}} running locally (see "elastic-package stack"). Therefore, if you want a local package to be served up by the local {{package-registry}}, make sure to build that package first using "elastic-package build". + +You can also publish built packages to the global package registry service. + + +### `elastic-package check` [_elastic_package_check] + +*Context: package* + +Use this command to verify if the package is correct in terms of formatting, validation and building. + +It will execute the format, lint, and build commands all at once, in that order. + + +### `elastic-package clean` [_elastic_package_clean] + +*Context: package* + +Use this command to clean resources used for building the package. + +The command will remove built package files (in build/), files needed for managing the development stack (in `~/.elastic-package/stack/development`) and stack service logs (in `~/.elastic-package/tmp/service_logs`). + + +### `elastic-package create` [_elastic_package_create] + +*Context: global* + +Use this command to create a new package or add more data streams. + +The command can help bootstrap the first draft of a package using an embedded package template. Then, you can use it to extend the package with more data streams. + +For details on creating a new package, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/create_new_package.md). + + +### `elastic-package export` [_elastic_package_export] + +*Context: package* + +Use this command to export assets relevant for the package, e.g. {{kib}} dashboards. + + +### `elastic-package format` [_elastic_package_format] + +*Context: package* + +Use this command to format the package files. + +The formatter supports JSON and YAML format and skips "ingest_pipeline" directories as it’s hard to correctly format Handlebars template files. As a result, formatted files are overwritten. + + +### `elastic-package install` [elastic-package-install] + +*Context: package* + +Use this command to upload and install a package in {{kib}}. + +Starting with Kibana version `8.7.0`, packages do not need to be exposed in the Package Registry to be installed. Instead, they can be upload as zip files built using the `elastic-package build` command. + +1. Ensure you’ve validated your package. Before building, validate the package by running the `elastic-package check` command. +2. Use either the `--zip` parameter to install a specific zip file or the `install` command to build the package and upload the built zip file to Kibana. + + +#### Install with `--zip` [_install_with_zip] + +Install a zipped package. This method relies on Package Registry. + +```shell +elastic-package stack up -d +elastic-package install --zip /home/user/Coding/work/integrations/build/packages/elastic_package_registry-0.0.6.zip -v +``` + + +#### Install with `elastic-package install` [_install_with_elastic_package_install] + +Build and upload a zipped package without relying on Package Registry. + +```shell +elastic-package stack up -v -d +elastic-package install -v +``` + + +#### Customization [_customization] + +Package installation can be customized to be installed in other Kibana instances with the following variables: + +* `ELASTIC_PACKAGE_KIBANA_HOST` +* `ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME` +* `ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD` +* `ELASTIC_PACKAGE_CA_CERT` + +For example: + +```bash +export ELASTIC_PACKAGE_KIBANA_HOST="https://test-installation.kibana.test:9243" +export ELASTIC_PACKAGE_ELASTICSEARCH_USERNAME="elastic" +export ELASTIC_PACKAGE_ELASTICSEARCH_PASSWORD="xxx" +# if it is a public instance, this variable should not be needed +export ELASTIC_PACKAGE_CA_CERT="" + +elastic-package install --zip elastic_package_registry-0.0.6.zip -v +``` + + +#### Older versions [_older_versions] + +For versions of Kibana `<8.7.0`, the package must be exposed via the Package Registry. In case of development, this means that the package should be built previously and then the Elastic stack must be started. Or, at least, the `package-registry` service needs to be restarted in the Elastic stack: + +```bash +elastic-package build -v +elastic-package stack up -v -d # elastic-package stack up -v -d --services package-registry +elastic-package install -v +``` + +To install the package in {{kib}}, the command uses {{kib}} API. The package must be exposed via the {{package-registry}}. + + +### `elastic-package lint` [_elastic_package_lint] + +*Context: package* + +Use this command to validate the contents of a package using the package specification (see: [https://github.com/elastic/package-spec](https://github.com/elastic/package-spec)). + +The command ensures that the package aligns with the package spec and that the README file is up-to-date with its template (if present). + + +### `elastic-package profiles` [_elastic_package_profiles] + +*Context: global* + +Use this command to add, remove, and manage multiple config profiles. + +Individual user profiles appear in ~/.elastic-package/stack and contain all the config files needed by the "stack" subcommand. Once a new profile is created, it can be specified with the -p flag, or the ELASTIC_PACKAGE_PROFILE environment variable. User profiles are not overwritten on an upgrade of elastic-stack and can be freely modified to allow for different stack configs. + + +### `elastic-package promote` [_elastic_package_promote] + +*Context: global* + +Use this command to move packages between the {{package-registry}} snapshot, staging, and production stages. + +This command is intended primarily for use by administrators. + +It allows for selecting packages for promotion and opens new pull requests to review changes. However, please be aware that the tool checks out an in-memory Git repository and switches over branches (snapshot, staging and production), so it may take longer to promote a larger number of packages. + + +### `elastic-package publish` [_elastic_package_publish] + +*Context: package* + +Use this command to publish a new package revision. + +The command checks if the package has already been published (whether it’s present in the snapshot/staging/production branch or open as pull request). If the package revision hasn’t been published, it will open a new pull request. + + +### `elastic-package service` [_elastic_package_service] + +*Context: package* + +Use this command to boot up the service stack that can be observed with the package. + +The command manages the lifecycle of the service stack defined for the package (`_dev/deploy`) for package development and testing purposes. + + +### `elastic-package stack` [_elastic_package_stack] + +*Context: global* + +Use this command to spin up a Docker-based {{stack}} consisting of {{es}}, {{kib}}, and the {{package-registry}}. By default, the latest released version of the {{stack}} is spun up, but it is possible to specify a different version, including SNAPSHOT versions. + +For details on connecting the service with the {{stack}}, see the [service command](https://github.com/elastic/elastic-package/blob/main/README.md#elastic-package-service). + + +### `elastic-package status [package]` [_elastic_package_status_package] + +*Context: package* + +Use this command to display the current deployment status of a package. + +If a package name is specified, then information about that package is returned. Otherwise, this command checks if the current directory is a package directory and reports its status. + + +### `elastic-package test` [_elastic_package_test] + +*Context: package* + +Use this command to run tests on a package. Currently, the following types of tests are available: + + +#### Asset Loading Tests [_asset_loading_tests] + +These tests ensure that all the {{es}} and {{kib}} assets defined by your package get loaded up as expected. + +For details on running asset loading tests for a package, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/asset_testing.md). + + +#### Pipeline Tests [_pipeline_tests] + +These tests allow you to exercise any Ingest Node Pipelines defined by your packages. + +For details on how configuring a pipeline test for a package, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/pipeline_testing.md). + + +#### Static Tests [_static_tests] + +These tests allow you to verify if all static resources of the package are valid, e.g. if all fields of the sample_event.json are documented. + +For details on running static tests for a package, see the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/static_testing.md). + + +#### System Tests [_system_tests] + +These tests allow you to test a package ability for ingesting data end-to-end. + +For details on configuring and running system tests, review the [HOWTO guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/system_testing.md). + + +### `elastic-package uninstall` [_elastic_package_uninstall] + +*Context: package* + +Use this command to uninstall the package in {{kib}}. + +To uninstall the package in {{kib}}, the command uses the {{kib}} API. The package must be exposed via the {{package-registry}}. + + +### `elastic-package version` [_elastic_package_version] + +*Context: global* + +Use this command to print the version of elastic-package that you have installed. This command is especially useful when reporting bugs. + diff --git a/docs/extend/finishing-touches.md b/docs/extend/finishing-touches.md new file mode 100644 index 00000000000..8419f942583 --- /dev/null +++ b/docs/extend/finishing-touches.md @@ -0,0 +1,82 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/finishing-touches.html +--- + +# Finishing touches [finishing-touches] + +## Words [_words] + +Tips for manifest files: + +* Descriptions of configuration options should be as short as possible. + + Remember to keep only the meaningful information about the configuration option. + + * Good candidates: references to the product configuration, accepted string values, explanation. + * Bad candidates: Collect metrics from A, B, C, D,…​ X, Y, Z datasets. + + +* Descriptions should be human readable. + + Try to rephrase sentences like: Collect foo_Bar3 metrics, into Collect Foo Bar metrics. + +* Descriptions should be easy to understand. + + Simplify sentences, don’t provide information about the input if not required. + + * Bad candidate: Collect application logs (log input) + * Good candidates: Collect application logs, Collect standard logs for the application + + + +## Add an icon [_add_an_icon] + +The integration icons are displayed in different places in {{kib}}, hence it’s better to define custom icons to make the UI easier to navigate. + + +## Add screenshots [_add_screenshots] + +The {{kib}} Integration Manager shows screenshots related to the integration. Screenshots include {{kib}} dashboards visualizing the metric and log data. + + +## Create a README file [_create_a_readme_file] + +The README template is used to render the final README file, including exported fields. The template should be placed in the `package//_dev/build/docs/README.md`. If the directory doesn’t exist, please create it. + +To see how to use template functions, for example {{fields "data-stream-name"}}, review the MySQL docs template. If the same data stream name is used in both metrics and logs, please add -metrics and -logs in the template. For example, ELB is a data stream for log and also a data stream for metrics. In README.md template, {{fields "elb_logs"}} and {{fields "elb_metrics"}} are used to separate them. + + +## Review artifacts [_review_artifacts] + + + +## Define variable properties [define-variable-properties] + +The variable properties customize visualization of configuration options in the {{kib}} UI. Make sure they’re defined in all manifest files. + +```yaml +vars: + - name: paths + required: true <1> + show_user: true <2> + title: Access log paths <3> + description: Paths to the apache access log file. <4> + type: text <5> + multi: true <6> + hide_in_deployment_modes: <7> + - agentless + default: + - /var/log/httpd/access.log* +``` + +1. option is required +2. don’t hide the configuration option (collapsed menu) +3. human readable variable name +4. variable description (may contain some details) +5. field type (according to the reference: text, password, bool, integer) +6. the field has multiple values +7. hides the variable in agentless mode (see [`hide_in_deployment_modes`](/extend/define-deployment-modes.md#hide_in_deployment_modes) for more information) + + + diff --git a/docs/extend/general-guidelines.md b/docs/extend/general-guidelines.md new file mode 100644 index 00000000000..47e06f032ed --- /dev/null +++ b/docs/extend/general-guidelines.md @@ -0,0 +1,184 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/general-guidelines.html +--- + +# General guidelines [general-guidelines] + +::::{important} +The following guidelines capture general aspects of the integrations that can be improved and should not be treated as a mandatory list of requirements every package should adhere to. Some guidelines that are applicable to one integration can be completely irrelevant to another. Treat them as best effort. +:::: + + +While the guidelines focus on metrics, they are equally applicable to logs. + + +## Data types [_data_types] + +Given that all packages are basic, developers should use Basic types (for example `histogram`. `wildcard`, etc.) when applicable. Of course, for ECS (see below) we should use the type specified by ECS. + + +## ECS compliance [_ecs_compliance] + +An integration package should be compliant with the most recent version of ECS. This implies an increased amount of relevant ECS fields populated by an integration. + +Starting with ECS 1.6, ECS is going to start using Basic types for some fields. Integration fields should be upgraded to the new types as part of the process. + + +## Document all fields [_document_all_fields] + +All fields produced by an integration must be mapped by `fields.yml`. This guarantees that their index mapping is correct, and Kibana has enough information to deal with all fields. + + +### Field limits [_field_limits] + +By default, data streams will have a `total_fields.limit` setting of 1000. Besides defined custom fields, this also includes dynamically generated ECS fields. If your data stream is expected to eventually house more than 1000 fields, set an explicit limit in the `manifest.yml` of the data stream: + +```yaml +elasticsearch: + index_template: + settings: + index: + mapping: + total_fields: + limit: 5000 +``` + +::::{note} +For backwards compatibility, the limit is automatically bumped to 10000 fields if there are more than 500 fields explicitly defined for a data stream, however newly created integrations should not rely on this behavior but instead assume a fixed limit of 1000 fields. +:::: + + + +### Specify metric types and units [_specify_metric_types_and_units] + +As part of the field definition, there are two settings that add metadata which will help Kibana graphing it: + +* `unit` applies to all data types, defines the units of the field. Examples of units are `byte` and `ms`. When using `percent` for percentages, the convention is to use 1 for 100%. You can find the full list of supported units in the [package spec](https://github.com/elastic/package-spec/blob/ff8286d0c40ad76bb082e9c8ea78f4551c2519c1/spec/integration/data_stream/fields/fields.spec.yml#L103). +* `metric_type` applies to metric events only, to be added to metric fields. It defines their metric type. It can be of type `gauge` or `counter`. Counters are used for metrics that always increase over time, such as number of page visits. Gauges are used for amounts that can increase or decrease over time, such as the amount of memory being used. + +The Elasticsearch documentation details the [expected values for these two fields](elasticsearch://docs/reference/elasticsearch/mapping-reference/mapping-field-meta.md). + +Other applications, including Kibana, can use the information provided by this metadata when accessing these fields. The `unit` is used when formatting the values of the field, and the `metric_type` can be used to provide better defaults when quering the data. + + +### Specify dimensions [_specify_dimensions] + +A set of fields of a data stream can be defined as dimensions. A set of dimensions with the same values identify a single time series. + +It is important to choose the set of fields carefully. They should be the minimal set of dimensions required to properly identify any time series included in the data stream. Too few dimensions can mix data of multiple time series into a single one, while too many dimensions can impact performance. + +A field can be configured as a dimension by setting `dimension: true` in its definition. + +Only fields of certain data types can be defined as dimensions. These data types include keywords, IPs and numeric types. + +Some guidelines to take into account when chosing dimensions: + +* They can affect ingestion performance, it is recommended to have as few dimensions as possible. When selecting dimensions, try to avoid redundant ones, such as unique identifiers and names that refer to the same object. +* Also be careful with having too few dimensions. There can be only one document with the same timestamp for a given set of dimensions. This can lead to data loss if different objects produce the same dimensions. +* Changing dimensions can be a breaking change. A different set of dimensions produces a different time series, even if they select the same data. + +Declaring dimensions is a requisite to use TSDB indexes. These indexes are optimized for time series use cases, bringing disk storage savings and additional queries and aggregations. + +TSDB indexes can be enabled in data streams by setting `elasticsearch.index_mode: time_series` in their manifests. + + +## Logs and Metrics UI compatibility [_logs_and_metrics_ui_compatibility] + +When applicable an integrataion package should provide the relevant fields for the Logs and Metrics Apps. This is especially relevant for integrations that are focused on compute-resources (VMs, containers, etc.). + +* Keep the [Logs app fields](docs-content://reference/observability/fields-and-object-schemas/logs-app-fields.md) reference up to date. +* Keep the [Infrastructure app fields](docs-content://reference/observability/fields-and-object-schemas/metrics-app-fields.md) reference up to date. + + +## Subtracting metrics [_subtracting_metrics] + +An integration package should collect a reasonable amount of metrics for any target system. In some cases this may mean removing some metrics that Filebeat and Metricbeat are collecting today. Collecting too many metrics has implications on metric storage as well as relevance of the data provided to the user. + +Potential candidates to remove: + +* low-level garbage collector metrics +* internal metrics showing code flow (for example, `Got100Continue`, `Wait100Continue`) +* redundant metrics (for example, metric collection for MQ topics doesn’t require collection of summary metrics) + + +## Relevant metrics [_relevant_metrics] + +This is probably the most important and hardest one of the guidelinesto satisfy, as it requires knowledge of every target system. Identifying relevant metrics should be considered case by case. + +There are no well defined guidelines for this exercise. It can be as simple as finding everything in one place (for example the [RabbitMQ documentation](https://www.rabbitmq.com/monitoring.md)) or as difficult as reviewing multiple sources including documentation, blog posts, and other integrations, and consolidating the discovered information in one place for revision. A recommendation is to only collect the metrics that are needed for dashboards and visualizations in general. + + +## Keep the original message field [_keep_the_original_message_field] + +Log integrations should keep the original message field (recommended name: `event.original`) so that it shows up in the Logs UI. It will also be useful when users want to reindex the data after changing a pipeline. In addition, the message field can be used as source for the some future Runtime fields. + +The original field should be user-configurable with the Kibana UI for better cost and storage management, and also consistency with other integrations. + + +## Document storage efficiency [_document_storage_efficiency] + +Every integration should strive to store collected data as efficiently as possible, which implies optimizing the way each integration generates documents. + + +## Default datasets [_default_datasets] + +When applicable, an integration package should provide a default dataset that aggregates a subset of the most relevant metrics across other data streams. Think of these as the metrics that are visualized on overview dashboards or are used for alerting. A guideline for creating a separate default dataset could be when the number of datasets in a package is more than three. + + +## Updated versions [_updated_versions] + +An integration package should support the most relevant versions of a target system. Some of our integrations support older versions of a target service/system, which were relevant at the time of implementation. Over time they can become outdated and require a revision, which can be as simple as testing the integration against the latest version and updating the compatibility section in the documentation, or it can mean refactoring the code to work with the latest version. For example, the Ceph module has recently been updated to support the latest version which had an entirely different way of collecting metrics. In order to accommodate both older and new versions in the module, metricsets were created in the module specifically for newer versions and it was noted in the documentation which metricsets to use. + + +## Updated configuration defaults [_updated_configuration_defaults] + +An integration package should provide meaningful defaults, such as collection intervals (periods), enabled metricsets, and any other integration specific configuration parameters. In the majority of cases users opt to use defaults. Hence, providing the relevant default values is crucial for the integration to be useful. In addition, integrations should strive to provide a one-click experience by providing the defaults that can cover 80% of use cases. + + +## Updated docs [_updated_docs] + +Integration packages should provide consistent and comprehensive documentation. For more details, refer to the [documentation guidelines](/extend/documentation-guidelines.md). + + +## Updated integration content [_updated_integration_content] + +Integration packages should provide out-of-the-box dashboards. For more details, refer to the [dashboard guidelines](/extend/dashboard-guidelines.md). + + +## Content for elastic.co/integrations [_content_for_elastic_cointegrations] + +Each integration will be listed on the public website `elastic.co/integrations` and the package registry will serve as the source of truth. As a result, documentation and screenshots should be high quality to showcase the integration. Please ensure to use `svg` for the logo and `png` for all other images. Any additional branding material should be reviewed carefully, for example: + +* logo format and quality +* permission to use logos and trademarks + + +## Curated user experiences [_curated_user_experiences] + +It’s advised to set integration policies in Fleet. Every integration and agent should be visible in Fleet and users should be able to add the integration directly from the integration list. This leads to better cohesion since it provides a consistent experience across integrations, allow users to add several integrations at once, and avoids sending them back and forth between multiple apps. It also allows users to discover new integrations in the list. + +Elastic products will also have the option to provide a curated UI for settings that are difficult to put in Fleet. It’s up to the product to decide how much flexibility they want to provide in changing the configuration directly from Fleet. This will depend on the use case and if it makes sense. Some level of configuration is recommended though. + + +## Asset tagging and metadata [_asset_tagging_and_metadata] + +When assets are installed through Fleet some metadata is added by default. + +For Elasticsearch assets such as index templates and ingest pipelines, a `_meta` property is added to the asset as follows: + +```json +{ + "managed_by": "fleet", + "managed": true, + "package": { + "name": "" + } +} +``` + +For Kibana assets, [tags](docs-content://explore-analyze/find-and-organize/tags.md) are generated in addition to the `_meta` property: + +* One tag with a `name` matching the package’s `title` property +* The `managed` tag, which Kibana uses to recognize "system" assets, or those that are installed by Kibana itself instead of generated by an end user + diff --git a/docs/extend/index.md b/docs/extend/index.md new file mode 100644 index 00000000000..125e50becd8 --- /dev/null +++ b/docs/extend/index.md @@ -0,0 +1,24 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/index.html +--- + +# Create an Integration + +This section provides the guidance you'll need to be able to create, manage, and optimize integrations with Elastic products. + +## Getting Started + +Begin by understanding what is an [integration](./what-is-an-integration.md). + +## Building Integrations + +Dive deep into the technical aspects of building integrations with Elastic products. Our [Building Integrations](./build-new-integration.md) guide covers everything from architecture and design principles to coding best practices and sample projects. + +## Testing and Validation + +Ensure your integrations work seamlessly by following our [Testing and Validation](./testing-validation.md) guidelines. Learn about different testing methodologies, tools, and techniques to validate your integration's performance and reliability. + +## Packaging and Deployment + +Once your integration is ready, our [Packaging and Deployment](./package-spec.md) guide will help you package your integration and deploy it efficiently. This section includes instructions on creating distributable packages, setting up deployment environments, and more. diff --git a/docs/extend/integration-definitions.md b/docs/extend/integration-definitions.md new file mode 100644 index 00000000000..02c219ba9b8 --- /dev/null +++ b/docs/extend/integration-definitions.md @@ -0,0 +1,51 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/integration-definitions.html +--- + +# Definitions [integration-definitions] + + +## Package [_package] + +An Elastic Package, or simply package for short, contains the dashboards, visualisations, and configurations to monitor the logs and metrics of a particular technology or group of related services, such as “MySQL”, or “System”. + +The package consists of: + +* Name +* Zero or more dashboards and visualisations and Canvas workpads +* Zero or more ML job definitions +* Zero or more data stream index templates + +The package is versioned. + + +## Integration [_integration] + +An integration is a specific type of a package defining data streams used to observe a product using logs, metrics, and traces. + + +## Data stream [_data_stream] + +A data stream is logical sub-division of an Integration package, dealing with a specific type of observable aspect of the service or product being observed. For example, the `mysql` package defines a data stream for collecting metrics and another data stream for collecting server logs. + +A data stream defines all the assets needed to create an Elasticsearch data stream, for example: index templates and ingest pipelines. These assets are loaded into Elasticsearch when a user installs a package via the Fleet UI in Kibana. + +A data stream also defines a policy template. Policy templates include variables that allow users to configure the data stream via the Fleet UI in Kibana. The resulting policy is interpreted by the Elastic Agent to collect relevant information from the product or service being observed. + +Data streams are defined inside the `data_stream` folder located under the package’s root directory. Each data stream is defined in it’s own sub-folder. + +The data stream consists of: + +* Field definitions (`fields.yml` files) +* Zero or more ingest pipelines +* An Elastic Agent policy template + + +## Development Extensions: `_dev` directories [_development_extensions_dev_directories] + +The `_dev` directory is part of the [package-spec](https://github.com/elastic/package-spec), and contains development resources. These development resources cover any types of files or folders needed only at development time. This includes resources needed for testing, but also includes any templates that might be used for generating documentation. In the future it could include other files or folders needed just at development time. It can be defined on the following levels: + +1. The package-level `_dev` folder contains files needed to set up the testing environment for that package. This environment setup is specified by files and folders in the `_dev/deploy` folder. For example, the `apache` package [specifies](https://github.com/elastic/integrations/tree/main/packages/apache/_dev/deploy) how to spin up an Apache Docker container for testing. +2. The data stream-level `_dev` folder contains test configuration files for various types of tests. For example, see the [`_dev/test folder`](https://github.com/elastic/integrations/tree/main/packages/apache/data_stream/error/_dev/test) under the `apache/error` data stream. The integrations have also [asset](https://github.com/elastic/elastic-package/blob/main/docs/howto/asset_testing.md) and [static](https://github.com/elastic/elastic-package/blob/main/docs/howto/static_testing.md) tests. They don’t require config files, but configs can be used to mark them as optional. + diff --git a/docs/extend/integrations-guidelines.md b/docs/extend/integrations-guidelines.md new file mode 100644 index 00000000000..6a1633d4e00 --- /dev/null +++ b/docs/extend/integrations-guidelines.md @@ -0,0 +1,16 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/integrations-guidelines.html +--- + +# Integrations guidelines [integrations-guidelines] + +Refer to the following pages for some tips and recommendations for building integrations. + +* [General guidelines](/extend/general-guidelines.md) +* [Dashboard guidelines](/extend/dashboard-guidelines.md) +* [Documentation guidelines](/extend/documentation-guidelines.md) + + + + diff --git a/docs/extend/integrations-tsds-synthetic-source.md b/docs/extend/integrations-tsds-synthetic-source.md new file mode 100644 index 00000000000..a937f5badc1 --- /dev/null +++ b/docs/extend/integrations-tsds-synthetic-source.md @@ -0,0 +1,14 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/integrations-tsds-synthetic-source.html +--- + +# Working with new indexing features [integrations-tsds-synthetic-source] + +These pages include details for incorporating new indexing features into your integrations, such as time series data stream (TSDS), `doc-value-only` fields, and synthetic source. + +* [TSDS guidelines](/extend/developer-tsds-guidelines.md) +* [How to test new indexing features](/extend/testing-new-indexing-features.md) + + + diff --git a/docs/extend/kibana-spec.md b/docs/extend/kibana-spec.md new file mode 100644 index 00000000000..88097727d3c --- /dev/null +++ b/docs/extend/kibana-spec.md @@ -0,0 +1,162 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/kibana-spec.html +--- + +# kibana [kibana-spec] + +The integration’s {{kib}} assets, like dashboards, visualizations, {{ml}} modules, etc. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +spec: + additionalContents: false + contents: + - description: Folder containing Kibana dashboard assets + type: folder + name: dashboard + required: false + contents: + - description: A dashboard asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: Folder containing Kibana visualization assets + type: folder + name: visualization + required: false + contents: + - description: A visualization asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: Folder containing Kibana saved search assets + type: folder + name: search + required: false + contents: + - description: A saved search asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: Folder containing Kibana map assets + type: folder + name: map + required: false + contents: + - description: A map asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: Folder containing Kibana lens assets + type: folder + name: lens + required: false + contents: + - description: A lens asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden + - description: Folder containing Kibana index pattern assets + type: folder + name: "index_pattern" + required: false + contents: + - description: An index pattern asset file + type: file + contentMediaType: "application/json" + pattern: '^.+\.json$' + - description: Folder containing rules + type: folder + name: "security_rule" + required: false + contents: + - description: An individual rule file for the detection engine + type: file + contentMediaType: "application/json" + pattern: '^.+\.json$' + - description: Folder containing CSP rule templates + type: folder + name: "csp_rule_template" + required: false + contents: + - description: An individual CSP rule template file for the cloud security posture management solution + type: file + contentMediaType: "application/json" + pattern: '^.+\.json$' + - description: Folder containing ML module assets + type: folder + name: ml_module + required: false + contents: + - description: An ML module asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + - description: Folder containing Kibana tags + type: folder + name: tag + required: false + contents: + - description: A dashboard tag file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + - description: Folder containing Osquery pack assets + type: folder + name: osquery_pack_asset + required: false + contents: + - description: An osquery pack asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + - description: Folder containing Osquery saved queries + type: folder + name: osquery_saved_query + required: false + contents: + - description: An osquery saved query file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + - description: File containing saved object tag definitions for assets + type: file + contentMediaType: "application/x-yaml" + name: "tags.yml" + required: false + $ref: "./tags.spec.yml" + - description: Folder containing Kibana SLO assets + type: folder + name: slo + required: false + contents: + - description: An SLO asset file + type: file + contentMediaType: "application/json" + pattern: '^{PACKAGE_NAME}-.+\.json$' + forbiddenPatterns: + - '^.+-(ecs|ECS)\.json$' # ECS suffix is forbidden +versions: + - before: 3.4.0 + patch: + - op: remove + path: "/contents/13" # remove SLO definitions + - before: 2.10.0 + patch: + - op: remove + path: "/contents/12" # remove tags definition +``` diff --git a/docs/extend/manifest-spec.md b/docs/extend/manifest-spec.md new file mode 100644 index 00000000000..f510cd98ba2 --- /dev/null +++ b/docs/extend/manifest-spec.md @@ -0,0 +1,675 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/manifest-spec.html +--- + +# manifest.yml [manifest-spec] + +Integration metadata, like version, name, license level, description, category, icon and screenshot mappings, and policy template definitions. + +**required** + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +## +## Describes the specification for the integration package's main manifest.yml file +## +spec: + # Everything under here follows JSON schema (https://json-schema.org/), written as YAML for readability + type: object + additionalProperties: false + definitions: + agent: + description: Declarations related to Agent configurations or requirements. + type: object + additionalProperties: false + properties: + privileges: + type: object + additionalProperties: false + properties: + root: + description: Set to true if collection requires root privileges in the agent. + type: boolean + categories: + description: Categories to which this package belongs. + type: array + items: + type: string + enum: + - advanced_analytics_ueba + - analytics_engine + - application_observability + - app_search + - auditd + - authentication + - aws + - azure + - big_data + - cdn_security + - cloud + - cloudsecurity_cdr + - config_management + - connector + - connector_client + - connector_package + - containers + - content_source + - crawler + - credential_management + - crm + - custom + - custom_logs + - database_security + - datastore + - dns_security + - edr_xdr + - elasticsearch_sdk + - elastic_stack + - email_security + - enterprise_search + - firewall_security + - google_cloud + - iam + - ids_ips + - infrastructure + - java_observability + - kubernetes + - language_client + - languages + - load_balancer + - message_queue + - monitoring + - native_search + - network + - network_security + - notification + - observability + - os_system + - process_manager + - productivity + - productivity_security + - proxy_security + - sdk_search + - security + - stream_processing + - support + - threat_intel + - ticketing + - version_control + - virtualization + - vpn_security + - vulnerability_management + - web + - web_application_firewall + - websphere + - workplace_search + examples: + - web + conditions: + description: Conditions under which this package can be installed. + type: object + additionalProperties: false + properties: + elastic: + description: Elastic conditions + type: object + additionalProperties: false + properties: + subscription: + description: The subscription required for this package. + type: string + enum: + - basic + - gold + - platinum + - enterprise + default: basic + examples: + - basic + capabilities: + description: |- + Stack features that are required by the package to work properly. + The package should not be used in deployments without the indicated features. + Packages that don't indicate any capability condition can be used on any deployment. + type: array + uniqueItems: true + items: + type: string + enum: + - apm + - enterprise_search + - observability + - security + - serverless_search + - uptime + kibana: + description: Kibana conditions + type: object + additionalProperties: false + properties: + version: + type: string + description: Kibana versions compatible with this package. + examples: + - ">=7.9.0" + description: + description: > + A longer description of the package. It should describe, at least all the kinds of + data that is collected and with what collectors, following the structure + "Collect X from Y with X". + type: string + examples: + - Collect logs and metrics from Apache HTTP Servers with Elastic Agent. + - Collect logs and metrics from Amazon Web Services with Elastic Agent. + deployment_modes: + description: > + Options related to the deployment modes. The deployment mode refers to the mode used to + deploy the Elastic Agents running this policy. + type: object + additionalProperties: false + properties: + default: + description: > + Options specific to the default deployment mode, where agents are normally managed + by users, explicitly enrolled to Fleet and visible in UIs. + type: object + properties: + enabled: + description: > + Indicates if the default deployment mode is available for this template policy. + It is enabled by default. + type: boolean + default: true + agentless: + description: > + Options specific to the Agentless deployment mode. This mode is used in offerings + where the Elastic Agents running these policies are fully managed for the user. + type: object + additionalProperties: false + properties: + enabled: + description: > + Indicates if the agentless deployment mode is available for this template policy. + It is disabled by default. + type: boolean + default: false + is_default: + description: > + On policy templates that support multiple deployment modes, this setting can be set to + true to use agentless mode by default. + type: boolean + default: false + organization: + description: > + The responsible organization of the integration. This is used to tag the agentless agent deployments + for monitoring. + type: string + examples: + - "security" + division: + description: > + The division responsible for the integration. This is used to tag the agentless agent deployments + for monitoring. + type: string + examples: + - "cloud-security" + team: + description: > + The team responsible for the integration. This is used to tag the agentless + agent deployments for monitoring. + type: string + examples: + - "cloud-security-posture-management" + resources: + description: > + The computing resources specifications for the Agentless deployment. + type: object + additionalProperties: false + properties: + requests: + description: > + The computing resources that the Agentless deployment will be initially allocated. + type: object + additionalProperties: false + properties: + memory: + description: > + The amount of memory that the Agentless deployment will be initially allocated. + type: string + examples: + - "1G" + - "1.5G" + cpu: + description: > + The amount of CPUs that the Agentless deployment will be initially allocated. + type: string + examples: + - "1" + - "1.5" + - "1500m" + allOf: + - if: + properties: + enabled: + const: true + then: + required: + - organization + - division + - team + configuration_links: + description: List of links related to inputs and policy templates. + type: array + minItems: 1 + items: + type: object + additionalProperties: false + properties: + title: + description: Link title + type: string + url: + description: Link url. Format is `http://...` or `https://...` for external links, `kbn:/app/...` for links internal to Kibana. + type: string + pattern: '^(http(s)?://|kbn:/)' + type: + description: Type of link. `next_steps` for links to locations that can be relevant right after configuring the policy. `action` for actions that can be performed while the policy is in use. + type: string + enum: + - action + - next_step + content: + description: Link description + type: string + required: + - title + - url + - type + icons: + description: List of icons for by this package. + type: array + items: + type: object + additionalProperties: false + properties: + src: + description: Relative path to the icon's image file. + type: string + format: relative-path + examples: + - "/img/logo_apache.svg" + title: + description: Title of icon. + type: string + examples: + - "Apache Logo" + size: + description: Size of the icon. + type: string + examples: + - "32x32" + type: + description: MIME type of the icon image file. + type: string + examples: + - "image/svg+xml" + dark_mode: + description: Is this icon to be shown in dark mode? + type: boolean + default: false + required: + - src + screenshots: + description: List of screenshots of Kibana assets created by this package. + type: array + items: + type: object + additionalProperties: false + properties: + src: + description: Relative path to the screenshot's image file. + type: string + format: relative-path + examples: + - "/img/apache_httpd_server_status.png" + title: + description: Title of screenshot. + type: string + examples: + - "Apache HTTPD Server Status" + size: + description: Size of the screenshot. + type: string + examples: + - "1215x1199" + type: + description: MIME type of the screenshot image file. + type: string + examples: + - "image/png" + required: + - src + - title + source: + description: Information about the source of the package. + type: object + additionalProperties: false + properties: + license: + description: Identifier of the license of the package, as specified in https://spdx.org/licenses/. + type: string + enum: + - "Apache-2.0" + - "Elastic-2.0" + examples: + - "Elastic-2.0" + title: + description: > + Title of the package. It should be the usual title given to the product, service or + kind of source being managed by this package. + type: string + examples: + - Apache HTTP Server + - MySQL + - AWS + version: + description: Version of the package, following semantic versioning. It can include pre-release labels. + type: string + pattern: '^([0-9]+)\.([0-9]+)\.([0-9]+)(?:-([0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+[0-9A-Za-z-]+)?$' + examples: + - "1.0.0" + - "1.0.0-beta1" + - "1.0.0-SNAPSHOT" + - "1.0.0-next" + owner: + type: object + additionalProperties: false + properties: + github: + description: Github team name of the package maintainer. + type: string + pattern: '^(([a-zA-Z0-9-_]+)|([a-zA-Z0-9-_]+\/[a-zA-Z0-9-_]+))$' + examples: + - "elastic" + - "apm-agent-java" + - "ux_infra_team" + type: + description: > + Describes who owns the package and the level of support that is + provided. The 'elastic' value indicates that the package is built + and maintained by Elastic. The 'partner' value indicates that the + package is built and maintained by a partner vendor and may include + involvement from Elastic. The 'community' value indicates the package + is built and maintained by non-Elastic community members. + type: string + default: community + enum: + - elastic + - partner + - community + examples: + - community + required: + - github + - type + properties: + format_version: + description: The version of the package specification format used by this package. + $ref: "#/definitions/version" + name: + description: The name of the package. + type: string + pattern: '^[a-z0-9_]+$' + examples: + - apache + title: + $ref: "#/definitions/title" + description: + $ref: "#/definitions/description" + version: + description: The version of the package. + $ref: "#/definitions/version" + source: + $ref: "#/definitions/source" + type: + description: The type of package. + type: string + enum: + - integration + examples: + - integration + categories: + $ref: "#/definitions/categories" + conditions: + $ref: "#/definitions/conditions" + # requires a conditional JSON schema to update the value depending + # on the policy_templates length + policy_templates_behavior: + description: > + Expected behavior when there are more than one policy template defined. + When set to `combined_policy`, a single policy template is available that + combines all the defined templates. When set to `individual_policies`, all + policies are individually available, but there is no combined policy. + The default value is `all`, where the combined policy template is available + along with the individual policies. + type: string + policy_templates: + description: List of policy templates offered by this package. + type: array + items: + type: object + additionalProperties: false + properties: + name: + description: Name of policy template. + type: string + examples: + - apache + title: + description: Title of policy template. + type: string + examples: + - Apache logs and metrics + categories: + $ref: "#/definitions/categories" + description: + description: Longer description of policy template. + type: string + examples: + - Collect logs and metrics from Apache instances + data_streams: + description: List of data streams compatible with the policy template. + type: array + items: + type: string + description: Data stream name + format: data-stream-name + examples: + - ec2_logs + - spamfirewall + - access + deployment_modes: + $ref: "#/definitions/deployment_modes" + configuration_links: + $ref: "#/definitions/configuration_links" + inputs: + description: List of inputs supported by policy template. + type: array + items: + type: object + additionalProperties: false + properties: + type: + description: Type of input. + type: string + title: + description: Title of input. + type: string + examples: + - Collect logs from Apache instances + description: + description: Longer description of input. + type: string + examples: + - Collecting Apache access and error logs + template_path: + description: Path of the config template for the input. + type: string + examples: + - ./agent/input/template.yml.hbs + input_group: + description: Name of the input group + type: string + enum: + - logs + - metrics + multi: + description: Can input be defined multiple times + type: boolean + default: false + required_vars: + $ref: "./data_stream/manifest.spec.yml#/definitions/required_vars" + vars: + $ref: "./data_stream/manifest.spec.yml#/definitions/vars" + required: + - type + - title + - description + multiple: + type: boolean + icons: + $ref: "#/definitions/icons" + screenshots: + $ref: "#/definitions/screenshots" + vars: + $ref: "./data_stream/manifest.spec.yml#/definitions/vars" + required: + - name + - title + - description + icons: + $ref: "#/definitions/icons" + screenshots: + $ref: "#/definitions/screenshots" + vars: + $ref: "./data_stream/manifest.spec.yml#/definitions/vars" + owner: + $ref: "#/definitions/owner" + agent: + $ref: "#/definitions/agent" + elasticsearch: + description: Elasticsearch requirements + type: object + additionalProperties: false + properties: + privileges: + description: Elasticsearch privilege requirements + type: object + additionalProperties: false + properties: + cluster: + # Available cluster privileges are available at https://www.elastic.co/guide/en/elasticsearch/reference/7.16/security-privileges.html#privileges-list-cluster + description: Elasticsearch cluster privilege requirements + type: array + items: + type: string + required: + - format_version + - name + - title + - description + - version + - type + - owner + allOf: + - if: + properties: + policy_templates: + maxItems: 1 + then: + properties: + policy_templates_behavior: + enum: + - all + default: all + else: + properties: + policy_templates_behavior: + enum: + - combined_policy + - individual_policies + - all + default: all + +# JSON patches for newer versions should be placed on top +versions: + - before: 3.3.2 + patch: + - op: remove + path: "/properties/policy_templates/items/properties/inputs/items/properties/required_vars" + - op: remove + path: "/definitions/deployment_modes/properties/agentless/properties/is_default" + - op: remove + path: "/definitions/deployment_modes/properties/agentless/properties/resources" + - before: 3.3.1 + patch: + - op: remove + path: "/properties/policy_templates/items/properties/configuration_links" + - before: 3.2.0 + patch: + - op: remove + path: "/definitions/deployment_modes/properties/default" + - before: 3.1.4 + patch: + - op: remove + path: "/properties/policy_templates/items/properties/deployment_modes" + - before: 3.0.0 + patch: + - op: replace + path: "/definitions/owner/required" + value: + - github + - before: 2.12.0 + patch: + - op: remove + path: "/properties/agent" + - before: 2.11.0 + patch: + - op: replace + path: "/definitions/owner/properties/type/default" + value: elastic + - before: 2.10.0 + patch: + - op: remove + path: "/definitions/conditions/properties/elastic/properties/capabilities" + - before: 2.3.0 + patch: + - op: add + path: "/properties/release" + value: + description: The stability of the package (deprecated, use prerelease tags in the version). + deprecated: true # See https://github.com/elastic/package-spec/issues/225 + type: string + enum: + - experimental + - beta + - ga + default: ga + examples: + - experimental + - before: 2.0.0 + patch: + - op: add + path: "/properties/license" + value: + description: The license under which the package is being released (deprecated, use subscription instead). + deprecated: true # See https://github.com/elastic/package-spec/issues/298. + type: string + enum: + - basic + default: basic + examples: + - basic +``` diff --git a/docs/extend/package-spec.md b/docs/extend/package-spec.md new file mode 100644 index 00000000000..0396c0e5346 --- /dev/null +++ b/docs/extend/package-spec.md @@ -0,0 +1,168 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/package-spec.html +--- + +# Package specification [package-spec] + +Integrations are a type of package and therefore must adhere to the Elastic package specification. The package specification describes: + +* The folder structure of a package and the expected files within these folders +* The structure of expected files' contents + + +### Asset organization [asset-organization] + +In general, assets within a package are organized by `/`. For example, ingest pipelines are stored in the `elasticsearch/ingest-pipeline` folder. This logic applies to all {{es}}, {{kib}}, and Agent assets. + +Top-level assets are picked up as JSON documents and pushed to the corresponding {{es}} and {{kib}} APIs. + + +#### Data streams [data-streams] + +There is a specific folder called `data_stream`. Each data stream should have its folder of assets within this folder, and the names of these data streams must follow the data stream naming scheme. + +The contents of these folders follow the `/` structure. During installation, {{fleet}} enforces data stream naming rules. All assets in this folder belong directly or indirectly to data streams. + +In most scenarios, only data stream assets are needed. However, there are exceptions where global assets are required to get more flexibility. For example, an {{ilm-init}} policy that applies to all data streams. + + +### Supported assets [supported-assets] + +The following assets are typically found in an Elastic package: + +* {es} + + * Ingest Pipeline + * Index Template + * Transform + * Index template settings + +* {kib} + + * Dashboards + * Visualization + * {data-sources-cap} + * {{ml-init}} Modules + * Map + * Search + * Security rules + +* Other + + * fields.yml + + + +### Directory structure [directory-structure] + +```text +apache +│ changelog.yml +│ manifest.yml +└───_dev +└───data_stream +└───docs +└───img +└───kibana +``` + + +### Spec [directory-spec] + +Included from the package-spec repository. This will update when the spec is updated. + +```yaml +## +## Entrypoint of "integration packages" specification. +## +## Describes the folders and files that make up a package. +## +spec: + additionalContents: true + totalContentsLimit: 65535 + totalSizeLimit: 250MB + sizeLimit: 150MB + configurationSizeLimit: 5MB + relativePathSizeLimit: 3MB + fieldsPerDataStreamLimit: 2048 + contents: + - description: The main package manifest file + type: file + contentMediaType: "application/x-yaml" + sizeLimit: 5MB + name: "manifest.yml" + required: true + $ref: "./manifest.spec.yml" + - description: The package's CHANGELOG file + type: file + contentMediaType: "application/x-yaml" + name: "changelog.yml" + required: true + $ref: "./changelog.spec.yml" + - description: The package's NOTICE file + type: file + contentMediaType: "text/plain" + name: "NOTICE.txt" + required: false + - description: The package's license file + type: file + contentMediaType: "text/plain" + name: "LICENSE.txt" + required: false + - description: Folder containing data stream definitions + type: folder + name: data_stream + required: false + $ref: "./data_stream/spec.yml" + - description: Folder containing documentation for the package + type: folder + name: docs + required: true + $ref: "./docs/spec.yml" + - description: Folder containing agent-related definitions + type: folder + name: agent + required: false + $ref: "./agent/spec.yml" + - description: Folder containing Kibana assets used by the package + type: folder + name: kibana + required: false + $ref: "./kibana/spec.yml" + - description: Folder containing development resources + type: folder + name: _dev + required: false + visibility: private + $ref: "./_dev/spec.yml" + - description: Folder containing Elasticsearch assets used by the package + type: folder + name: elasticsearch + required: false + $ref: "./elasticsearch/spec.yml" + - description: Configuration file to process the results returned from the package validation. This file is just for package validation and it should be ignored when installing or using the package. + type: file + contentMediaType: "application/x-yaml" + name: "validation.yml" + required: false + $ref: "./validation.spec.yml" + - description: Folder containing images for the package + type: folder + name: img + required: false + $ref: "./img/spec.yml" + +versions: + - before: 3.2.2 + patch: + - op: remove + path: "/contents/11" # Definition for img folder. +``` + + + + + + + diff --git a/docs/extend/pipeline-testing.md b/docs/extend/pipeline-testing.md new file mode 100644 index 00000000000..fe284b91d56 --- /dev/null +++ b/docs/extend/pipeline-testing.md @@ -0,0 +1,193 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/pipeline-testing.html +--- + +# Pipeline testing [pipeline-testing] + +Elastic Packages comprise of data streams. A pipeline test exercises {{es}} Ingest Node pipelines defined for a package’s data stream. + + +## Conceptual process [pipeline-concepts] + +Conceptually, running a pipeline test involves the following steps: + +1. Deploy the {{es}} instance (part of the {{stack}}). This step takes time, so you should typically do it once as a prerequisite to running pipeline tests on multiple data streams. +2. Upload ingest pipelines to be tested. +3. Use the [Simulate API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-simulate) to process logs/metrics with the ingest pipeline. +4. Compare generated results with expected ones. + + +## Limitations [pipeline-limitations] + +At the moment, pipeline tests have limitations. The main ones are: * As you’re only testing the ingest pipeline, you can prepare mocked documents with imaginary fields, different from ones collected in {{beats}}. Also, the other way round, you can skip most of the example fields and use tiny documents with a minimal set of fields just to satisfy the pipeline validation. * There might be integrations that transform data mainly using {{beats}} processors instead of ingest pipelines. In such cases, ingest pipelines are rather plain. + + +## Defining a pipeline test [pipeline-defining-test] + +Packages have a specific folder structure (only relevant parts shown). + +```bash +/ + data_stream/ + / + manifest.yml + manifest.yml +``` + +To define a pipeline test we must define configuration at each dataset’s level: + +```bash +/ + data_stream/ + / + _dev/ + test/ + pipeline/ + (test case definitions, both raw files and input events, optional configuration) + manifest.yml + manifest.yml +``` + + +### Test case definitions [pipeline-test-case] + +There are two types of test case definitions - **raw files** and **input events**. + + +#### Raw files [pipeline-raw-files] + +The raw files simplify preparing test cases using real application `.log` files. A sample log (e.g. `test-access-sample.log`) file may look like the following one for Nginx: + +```bash +127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] "GET /test1 HTTP/1.1" 404 571 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36" +127.0.0.1 - - [07/Dec/2016:11:04:58 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +127.0.0.1 - - [07/Dec/2016:11:04:59 +0100] "GET / HTTP/1.1" 304 0 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0" +``` + + +#### Input events [pipeline-input-events] + +The input events contain mocked JSON events that are ready to be passed to the ingest pipeline as-is. Such events can be helpful in situations in which an input event can’t be serialized to a standard log file, e.g. Redis input. A sample file with input events (e.g. `test-access-event.json`) looks as follows: + +```json +{ + "events": [ + { + "@timestamp": "2016-10-25T12:49:34.000Z", + "message": "127.0.0.1 - - [07/Dec/2016:11:04:37 +0100] \"GET /test1 HTTP/1.1\" 404 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.98 Safari/537.36\"\n" + }, + { + "@timestamp": "2016-10-25T12:49:34.000Z", + "message": "127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /taga HTTP/1.1\" 404 169 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"\n" + } + ] +} +``` + + +#### Test configuration [pipeline-test-config] + +Before sending log events to the ingest pipeline, a data transformation process is applied. The process can be customized using an optional configuration stored as a YAML file with the suffix `-config.yml` (e.g. `test-access-sample.log-config.yml`): + +```yaml +multiline: + first_line_pattern: "^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}" +fields: + "@timestamp": "2020-04-28T11:07:58.223Z" + ecs: + version: "1.5.0" + event.category: + - "web" +dynamic_fields: + url.original: "^/.*$" +numeric_keyword_fields: + - network.iana_number +``` + +The `multiline` section [raw files only](#pipeline-raw-files) configures the log file reader to detect multiline log entries using the `first_line_pattern`. Use this property if you may split your logs into multiple lines, e.g. Java stack traces. + +The `fields` section allows for customizing extra fields to be added to every read log entry (e.g. `@timestamp`, `ecs`). Use this property to extend your logs with data that can’t be extracted from log content, but it’s fine to have the same field values for every record (e.g. timezone, hostname). + +The `dynamic_fields` section allows for marking fields as dynamic (every time they have different non-static values), so that pattern matching instead of strict value check is applied. + +The `numeric_keyword_fields` section identifies fields whose values are numbers but are expected to be stored in {{es}} as `keyword` fields. + + +#### Expected results [pipeline-expected-results] + +Once the Simulate API processes the input data, the pipeline test runner will compare them with expected results. Test results are stored as JSON files with the suffix `-expected.json`. A sample test results file is shown below. + +```json +{ + "expected": [ + { + "@timestamp": "2016-12-07T10:04:37.000Z", + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + ... + }, + { + "@timestamp": "2016-12-07T10:05:07.000Z", + "nginx": { + "access": { + "remote_ip_list": [ + "127.0.0.1" + ] + } + }, + ... + } + ] +} +``` + +It’s possible to generate the expected test results from the output of the Simulate API. To do so, use the `--generate` switch: + +```bash +elastic-package test pipeline --generate +``` + + +## Running a pipeline test [pipeline-running-test] + +Once the configurations are defined as described in the previous section, you are ready to run pipeline tests for a package’s data streams. + +First, you must deploy the {{es}} instance. This corresponds to step 1 as described in the [Conceptual-process](#pipeline-concepts) section. + +```bash +elastic-package stack up -d --services=elasticsearch +``` + +For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, you must set environment variables needed for further `elastic-package` commands. + +```bash +$(elastic-package stack shellinit) +``` + +Next, you must invoke the pipeline tests runner. This corresponds to steps 2 through 4 as described in the [Conceptual-process](#pipeline-concepts) section. + +If you want to run pipeline tests for **all data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test pipeline +``` + +If you want to run pipeline tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test pipeline --data-streams [,,...] +``` + +Finally, when you are done running all pipeline tests, bring down the {{stack}}. This corresponds to step 4 as described in the [Conceptual-process](#pipeline-concepts) section. + +```bash +elastic-package stack down +``` diff --git a/docs/extend/quick-start.md b/docs/extend/quick-start.md new file mode 100644 index 00000000000..827e19b1fc6 --- /dev/null +++ b/docs/extend/quick-start.md @@ -0,0 +1,435 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/quick-start.html +--- + +# Quick start: Sample integration [quick-start] + +::::{note} +This quick start is designed for users familiar with the {{stack}}. If you’re new to Elastic, [*Build an integration*](/extend/build-new-integration.md) provides an in-depth look at creating a new integration. +:::: + + +This is a quick guide on how you can build your own integration package and upload it to Kibana. + +Follow these steps to create an integration package named `sample` and then add a `logs` dataset. The same procedure can be used for a `metrics` dataset, however for your first integration package `logs` is a bit simpler because a custom input is not required. + +* [Prerequisites](#qs-prereqs) +* [Step 1: Create the package](#qs-create-package) +* [Step 2: Upload the package to Kibana](#qs-test-upload) +* [Step 3: Create a dataset](#qs-create-dataset) +* [Step 4: Add processing](#qs-add-processing) +* [Step 5: Release a new version](#qs-release-new-version) +* [Step 6: Ingest data](#qs-ingest-data) +* [What’s next?](#qs-whats-next) + + +## Prerequisites [qs-prereqs] + +You’ll need to have a few requirements in place to run this tutorial: + +* [elastic-package](https://github.com/elastic/elastic-package) installed on your machine. This is a command line tool, written in Go, used for developing Elastic packages. It can help you lint, format, test, build, and promote your packages. Setup instructions can be found in the elastic-package repository readme. +* A [GitHub repository](https://github.com/) where you can upload your integration package. +* [Docker](https://www.docker.com/) set up and running on your machine. + + +## Step 1: Create the package [qs-create-package] + +1. To start, from inside a Git repository, run the `elastic-package create package` command. This will launch a wizard that will prompt you for some details, and will then build an empty package with all the necessary parts: + + ```console + elastic-package create package + ``` + +2. Respond to prompts as follows: + + * Package type: `integration` + * Package name: `sample` + * Version: `0.0.1` + * License: `Elastic-2.0` + * Package title: `My sample package` + * Description: `My first integrations package to collect logs` + * Categories: `custom` + * Kibana version constraint: `^8.12.2` + * Required Elastic subscription: `basic` + * Github owner: `` + * Owner type: `elastic` + +3. After entering the details, the command should return a confirmation that your package has been created. +4. Change into the new `sample` package directory. + + ```console + cd sample + ``` + +5. Validate that the new integration package was created correctly. + + 1. Check the linting rules for the package + + ```console + elastic-package lint + ``` + + 2. Format the package to fix linting + + ```console + elastic-package format + ``` + + 3. Build a `.zip` file out of the package assets + + ```console + elastic-package build + ``` + + 4. If you prefer, you can also run the three previous commands as a single batch: + + ```console + elastic-package check + ``` + + + +## Step 2: Upload the package to Kibana [qs-test-upload] + +1. To test that your package can be installed into Kibana, a cluster needs to spin up. For this step you to have a running Docker setup. Run the following command: + + ```console + elastic-package stack up --version=8.12.2 -v + ``` + + This spins up a cluster with the version 8.12.2 of the {{stack}}. The cluster can be accessed in your browser at [https://localhost:5601](https://localhost:5601) with username `elastic` and password `changeme`. + + ::::{note} + * If you want to update to the latest {{stack}} version, run `elastic-package stack update --version=8.12.2 -v`. + * You can also install the package directly into an existing cluster for testing. Steps and customization options for the `install` command are described in this [How To guide](https://github.com/elastic/elastic-package/blob/main/docs/howto/install_package.md) in the `elastic-package` repository. + + :::: + +2. After the cluster has finished setting up, open a second terminal window and run the following command to install your package: + + ```console + elastic-package install + ``` + +3. After the command runs, check that your new package appears in Kibana under **Management > Integrations > Installed integrations**. + + :::{image} ../images/package-installed.png + :alt: Kibana installed integrations tab with a card for my sample package + ::: + + + +## Step 3: Create a dataset [qs-create-dataset] + +You’ve now built an integration package, but it does not contain any assets. For the goal of starting to collect logs, you need to create a dataset, and for it the Elasticsearch mappings and ingest pipelines. If you want to be able to collect data through a managed {{agent}}, you also need to add an agent policy template. + +1. Create a new dataset: + + ```console + elastic-package create data-stream + ``` + +2. When prompted, provide the following details: + + * Data stream name: log + * Data stream title: My log lines + * Type: logs + + The command creates the required data in the `/data_stream/log` directory. If you pick `log` as data stream name, the dataset is called `sample.log` and the final data stream created will be `logs-sample.log-default` as an example. + +3. To not have to worry about mappings, you can pull in all [Elastic Common Schema (ECS) fields][Elastic Common Schema (ECS)](ecs://docs/reference/index.md)). To do this, create the file `_dev/build/build.yml` under the root directory and add the following content: + + ```yaml + dependencies: + ecs: + reference: git@v8.6.0 + import_mappings: true + ``` + +4. It’s always a good idea to re-check to make sure that your package still builds and works as expected. + + ```console + elastic-package check + ``` + +5. Re-install your package. + + ```console + elastic-package install + ``` + + This reinstalls the package and create mapping templates for `logs-sample.log-*`. You can also add your own mappings under `data_stream/log/fields/fields.yml` if needed. + + + +## Step 4: Add processing [qs-add-processing] + +You can now already ship log files to `logs-sample.log-default` but no processing will happen. So, let’s create a sample log file. + +1. Create a file `test-sample.log` with the following contents, and save it anywhere on your local machine. + + ```console + 2024-04-21T13:44:56.657+0100 INFO Hello world + 2024-04-21T13:45:56.657+0100 INFO This is a nice day + 2024-04-21T13:46:56.657+0100 INFO I am working on stuff + ``` + + Each line of the log file will be shipped by {{agent}} as a document with the message field containing the log line. You will set up the dissect processor to take the log line apart into `@timestamp`, `log.level`, and `message`. + +2. Next, test your ingest pipeline. In {{kib}} navigate to **Management > Dev Tools** and run the [Simulate pipeline API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-ingest-simulate): + + ```console + POST /_ingest/pipeline/_simulate + { + "pipeline" : + { + "description": "logs-sample.log", + "processors": [ + { + "dissect" : { + "field" : "message", + "pattern" : "%{@timestamp} %{log.level} %{message}" + } + } + ] + }, + "docs": [ + { + "_index": "index", + "_id": "id", + "_source": { + "message": "2023-02-21T13:46:56.657+0100 INFO I am working on stuff" + } + } + ] + } + ``` + + This returns: + + ```console + { + "docs": [ + { + "doc": { + "_index": "index", + "_version": "-3", + "_id": "id", + "_source": { + "@timestamp": "2023-02-21T13:46:56.657+0100", + "message": "I am working on stuff", + "log": { + "level": "INFO" + } + }, + "_ingest": { + "timestamp": "2024-04-30T17:51:22.16442471Z" + } + } + } + ] + } + ``` + +3. Now that you’ve confirmed that the ingest pipeline is working, add it to your dataset by modifying `data_stream/log/elasticsearch/ingest_pipline/default.yml` to: + + ```console + description: Pipeline for processing sample logs + processors: + - dissect: + field: message + pattern: "%{@timestamp} %{log.level} %{message}" + on_failure: + - set: + field: error.message + value: '{{ _ingest.on_failure_message }}' + ``` + +4. Now run `elastic-package check` again and then re-upload the package with `elastic-package install`. This installs your new ingest pipeline. +5. Do a quick test run to test the new pipeline. In the **Dev tools** console, run: + + ```console + POST logs-sample.log-default/_doc + { + "message": "2023-02-21T13:46:56.657+0100 INFO I am working on stuff" + } + ``` + + The response is: + + ```console + { + "_index": ".ds-logs-sample.log-default-2024.04.30-000001", + "_id": "BsUtMI8BQEniT9Md_TYh", + "_version": 1, + "result": "created", + "_shards": { + "total": 2, + "successful": 1, + "failed": 0 + }, + "_seq_no": 0, + "_primary_term": 1 + } + ``` + +6. Now run: + + ```console + GET logs-sample.log-default/_search + ``` + + The response is: + + ```console + { + "took": 1, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": { + "value": 1, + "relation": "eq" + }, + "max_score": 1, + "hits": [ + { + "_index": ".ds-logs-sample.log-default-2024.04.30-000001", + "_id": "BsUtMI8BQEniT9Md_TYh", + "_score": 1, + "_source": { + "@timestamp": "2023-02-21T13:46:56.657+0100", + "message": "I am working on stuff", + "event": { + "agent_id_status": "missing", + "ingested": "2024-04-30T18:04:31Z" + }, + "log": { + "level": "INFO" + } + } + } + ] + } + } + ``` + + +Now that you can see the dissected message documented, you’re ready to ingest data. + + +## Step 5: Release a new version [qs-release-new-version] + +1. Since your initial `0.0.1` version of the package, many modifications have been made. To build a new package version, open the `sample/manifest.yml` file and change the package version to `0.2.0`: + + ```console + format_version: 3.1.3 + name: sample + title: "My sample package" + version: 0.2.0 + ``` + +2. You also need to add an entry to your `sample/changelog.yml` file. Make sure to add the new entry at the top of the file: + + ```console + - version: "0.2.0" + changes: + - description: Added sample log processing pipeline + type: enhancement + link: http://fake-link + ``` + + ::::{note} + You can also update the changelog file automatically using the [`elastic-package changelog`](https://github.com/elastic/elastic-package?tab=readme-ov-file#elastic-package-changelog) command. + :::: + +3. Run `elastic-package check` again and then the `elastic-package install` command. + + The `0.1.0` version of the package is updated to version `0.2.0`. Only one version of a package can be installed at a time, but, following these steps, different versions of a package can be rolled out over time. + + +When developing integrations the following versioning guidelines should be used: + +* Patch release (x.y.**Z**): For backward-compatible bug fixes +* Minor release (x.**Y**.z): For backward-compatible new features +* Major release (**X**.y.z): For changes that break backward compatibility + + +## Step 6: Ingest data [qs-ingest-data] + +There are two different ways that you can ingest data, using either standalone {{agent}} or {{agent}} managed by {{fleet}}. For this example, you can use standalone {{agent}} since that won’t require any additional changes to the integration package. + +::::{note} +To run these steps using {{fleet}}-managed {{agent}}, you just need to update the files `data_stream/log/agent/stream/stream.yml.hbs` and `data_stream/log/manifest.yml` to provide the correct configuration, which you can find in the {{fleet}} UI. +:::: + + +1. [Download the {{agent}} install package](https://www.elastic.co/downloads/elastic-agent) to your machine. +2. Download the {{agent}} package, extract it, and change into the package directory. You can find the steps for each available platform in [Install standalone {{agents}}](docs-content://reference/ingestion-tools/fleet/install-standalone-elastic-agent.md). + + You can also download a package directly from the [{{agent}} download site](https://www.elastic.co/downloads/elastic-agent). + +3. In the {{agent}} package directory, open the `elastic-agent.yml` configuration file for editing. +4. Replace the contents of `elastic-agent.yml` with the following: + + ```console + inputs: + - type: logfile + streams: + - data_stream: + # This must be aligned with the dataset name given + dataset: test-sample.log + paths: + # Path to your log file + - //test-sample.log + + outputs: + default: + type: elasticsearch + hosts: ["https://127.0.0.1:9200"] + username: "elastic" + password: "changeme" + ssl.verification_mode: none + ``` + + Where: + + * `dataset` is set to match the `test-sample.log` file that you created. + * is the full path the `test-sample.log` file that you created. + +5. Run {{agent}}: + + ```console + sudo ./elastic-agent -e + ``` + + This will pick up the log file, ship it to {{es}}, and process it with the ingest pipeline. + +6. Confirm that your log file is being ingested as expected: + + 1. In {{kib}}, open **Discover**. + 2. In the search field, enter `log.file.path.text : *`. The search should return a couple of log entries. + 3. Hover over an entry and click `Enter` to view the cell contents. + + :::{image} ../images/datastream-log-message.png + :alt: Data stream showing log message: "this is a nice day" + ::: + + + +## What’s next? [qs-whats-next] + +You now have your own integration package that you can update with new features and ship to an {{stack}} or share with others. + +In the integrations [Contributing Guide](https://github.com/elastic/integrations/blob/main/CONTRIBUTING.md) you can find instructions for adding additional assets to your integrations, such as {{kib}} dashboards. + +Let others know about your new integration: + +* Promote your Integration with Elastic in the [Elastic Community](https://www.elastic.co/community/). +* Register on [Elastic’s Partner Portal](https://partners.elastic.co/English/register_email.aspx) as a Technology Partner. diff --git a/docs/extend/static-testing.md b/docs/extend/static-testing.md new file mode 100644 index 00000000000..617b367f255 --- /dev/null +++ b/docs/extend/static-testing.md @@ -0,0 +1,30 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/static-testing.html +--- + +# Static testing [static-testing] + +Static tests allow you to verify if all static resources of the package are valid, e.g. are all fields of the `sample_event.json` documented. They don’t require any additional configuration (unless you would like to skip them). + + +## Coverage [static-coverage] + +Static tests cover the following resources: + +1. Sample event for a data stream - verification if the file uses only documented fields. + + +## Running static tests [static-running] + +Static tests don’t require the {{stack}} to be up and running. Simply navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test static +``` + +If you want to run pipeline tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test static --data-streams [,,...] +``` diff --git a/docs/extend/system-testing.md b/docs/extend/system-testing.md new file mode 100644 index 00000000000..2f55ac9f319 --- /dev/null +++ b/docs/extend/system-testing.md @@ -0,0 +1,244 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/system-testing.html +--- + +# System testing [system-testing] + +Elastic Packages comprise of data streams. A system test exercises the end-to-end flow of data for a package’s data stream — from ingesting data from the package’s integration service all the way to indexing it into an {{es}} data stream. + + +## Conceptual process [system-concepts] + +Conceptually, running a system test involves the following steps: + +1. Deploy the {{stack}}, including {{es}}, {{kib}}, and the {{agent}}. This step takes time. so you should typically do it once as a prerequisite to running system tests on multiple data streams. +2. Enroll the {{agent}} with {{fleet}} (running in the {{kib}} instance). This step also can be done once, as a prerequisite. +3. Depending on the Elastic Package whose data stream is being tested, deploy an instance of the package’s integration service. +4. Create a test policy that configures a single data stream for a single package. +5. Assign the test policy to the enrolled Agent. +6. Wait a reasonable amount of time for the Agent to collect data from the integration service and index it into the correct {{es}} data stream. +7. Query the first 500 documents based on `@timestamp` for validation. +8. Validate mappings are defined for the fields contained in the indexed documents. +9. Validate that the JSON data types contained `_source` are compatible with mappings declared for the field. +10. Delete test artifacts and tear down the instance of the package’s integration service. +11. Once all desired data streams have been system tested, tear down the {{stack}}. + + +## Limitations [system-test-limitations] + +At the moment, system tests have limitations. The salient ones are: * There isn’t a way to assert that the indexed data matches data from a file (e.g. golden file testing). + + +## Defining a system test [system-test-definition] + +Packages have a specific folder structure (only relevant parts shown). + +```bash +/ + data_stream/ + / + manifest.yml + manifest.yml +``` + +To define a system test we must define configuration on at least one level: a package or a data stream’s one. + +First, we must define the configuration for deploying a package’s integration service. We can define it on either the package level: + +```bash +/ + _dev/ + deploy/ + / + +``` + +or the data stream’s level: + +```bash +/ + data_stream/ + / + _dev/ + deploy/ + / + +``` + +`` - a name of the supported service deployer: + +* `docker` - Docker Compose +* `k8s` - Kubernetes +* `tf` - Terraform + + +### Docker Compose service deployer [system-docker-compose] + +The `` must include a `docker-compose.yml` file when using the Docker Compose service deployer. The `docker-compose.yml` file defines the integration service for the package. For example, if your package has a logs data stream, the log files from your package’s integration service must be written to a volume. For example, the `apache` package has the following definition in it’s integration service’s `docker-compose.yml` file. + +```bash +version: '2.3' +services: + apache: + # Other properties such as build, ports, etc. + volumes: + - ${SERVICE_LOGS_DIR}:/usr/local/apache2/logs +``` + +Here, `SERVICE_LOGS_DIR` is a special keyword. It is something that we will need later. + + +### Terraform service deployer [system-terraform] + +When using the Terraform service deployer, the `` must include at least one `*.tf` file. The `*.tf` files define the infrastructure using the Terraform syntax. The Terraform-based service can be handy to boot up resources using a selected cloud provider and use them for testing (e.g. observe and collect metrics). + +Sample `main.tf` definition: + +```bash +variable "TEST_RUN_ID" { + default = "detached" +} + +provider "aws" {} + +resource "aws_instance" "i" { + ami = data.aws_ami.latest-amzn.id + monitoring = true + instance_type = "t1.micro" + tags = { + Name = "elastic-package-test-${var.TEST_RUN_ID}" + } +} + +data "aws_ami" "latest-amzn" { + most_recent = true + owners = [ "amazon" ] # AWS + filter { + name = "name" + values = ["amzn2-ami-hvm-*"] + } +} +``` + +Notice the use of the `TEST_RUN_ID` variable. It contains a unique ID, which can help differentiate resources created in potential concurrent test runs. + + +### Kubernetes service deployer [system-kubernetes] + +The Kubernetes service deployer requires the `_dev/deploy/k8s` directory to be present. It can include additional `*.yaml` files to deploy custom applications in the Kubernetes cluster (e.g. Nginx deployment). If no resource definitions (`*.yaml` files ) are needed, the `_dev/deploy/k8s` directory must contain an `.empty` file (to preserve the `k8s` directory under version control). + +The Kubernetes service deployer needs [kind](https://kind.sigs.k8s.io/) to be installed and the cluster to be up and running: + +```bash +wget -qO- https://raw.githubusercontent.com/elastic/elastic-package/main/scripts/kind-config.yaml | kind create cluster --config - +``` + +Before executing system tests, the service deployer applies once the deployment of the {{agent}} to the cluster and links the kind cluster with the Elastic stack network - applications running in the kind cluster can reach {{es}} and {{kib}} instances. The {{agent}}'s deployment is not deleted after tests to shorten the total test execution time, but it can be reused. + +See how to execute system tests for the Kubernetes integration (`pod` data stream): + +```bash +elastic-package stack up -d -v # start the Elastic stack +wget -qO- https://raw.githubusercontent.com/elastic/elastic-package/main/scripts/kind-config.yaml | kind create cluster --config - +elastic-package test system --data-streams pod -v # start system tests for the "pod" data stream +``` + + +### Test case definition [system-test-case] + +Next, we must define at least one configuration for each data stream that we want to system test. You can define multiple test cases for the same data stream. + +*Hint: if you plan to define only one test case, you can consider the filename `test-default-config.yml`.* + +```bash +/ + data_stream/ + / + _dev/ + test/ + system/ + test--config.yml +``` + +The `test--config.yml` file allows you to define values for package and data stream-level variables. For example, the `apache/access` data stream’s `test-access-log-config.yml` is shown below. + +```bash +vars: ~ +input: logfile +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/access.log*" +``` + +The top-level `vars` field corresponds to package-level variables defined in the `apache` package’s `manifest.yml` file. In the above example, we don’t override any of these package-level variables, so their default values, are used in the `apache` package’s `manifest.yml` file. + +The `data_stream.vars` field corresponds to data stream-level variables for the current data stream (`apache/access` in the above example). In the above example we override the `paths` variable. All other variables are populated with their default values, as specified in the `apache/access` data stream’s `manifest.yml` file. + +Notice the use of the `{{SERVICE_LOGS_DIR}}` placeholder. This corresponds to the `${SERVICE_LOGS_DIR}` variable we saw in the `docker-compose.yml` file earlier. In the above example, the `/usr/local/apache2/logs/access.log*` files located inside the Apache integration service container become available at the same path from {{agent}}'s perspective. + +When a data stream’s manifest declares multiple streams with different inputs you can use the `input` option to select the stream to test. The first stream whose input type matches the `input` value will be tested. By default, the first stream declared in the manifest will be tested. + + +#### Placeholders [system-placeholders] + +The `SERVICE_LOGS_DIR` placeholder is not the only one available for use in a data stream’s `test--config.yml` file. The complete list of available placeholders is shown below. + +| Placeholder name | Data type | Description | +| --- | --- | --- | +| `Hostname` | string | Addressable host name of the integration service. | +| `Ports` | []int | Array of addressable ports the integration service is listening on. | +| `Port` | int | Alias for `Ports[0]`. Provided as a convenience. | +| `Logs.Folder.Agent` | string | Path to integration service’s logs folder, as addressable by the Agent. | +| `SERVICE_LOGS_DIR` | string | Alias for `Logs.Folder.Agent`. Provided as a convenience. | + +Placeholders used in the `test--config.yml` must be enclosed in `{{` and `}}` delimiters, per Handlebars syntax. + + +## Running a system test [system-running-test] + +Once the two levels of configurations are defined as described in the previous section, you are ready to run system tests for a package’s data streams. + +First you must deploy the {{stack}}. This corresponds to steps 1 and 2 as described in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section. + +```bash +elastic-package stack up -d +``` + +For a complete listing of options available for this command, run `elastic-package stack up -h` or `elastic-package help stack up`. + +Next, you must set environment variables needed for further `elastic-package` commands. + +```bash +$(elastic-package stack shellinit) +``` + +Next, you must invoke the system tests runner. This corresponds to steps 3 to 7 as described in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section. + +If you want to run system tests for **all data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test system +``` + +If you want to run system tests for **specific data streams** in a package, navigate to the package’s root folder (or any sub-folder under it) and run the following command. + +```bash +elastic-package test system --data-streams [,,...] +``` + +Finally, when you are done running all system tests, bring down the {{stack}}. This corresponds to step 8 in the [Conceptual-process](/extend/pipeline-testing.md#pipeline-concepts) section. + +```bash +elastic-package stack down +``` + + +### Generating sample events [system-sample-events] + +As the system tests exercise an integration end-to-end from running the integration’s service all the way to indexing generated data from the integration’s data streams into {{es}}, it is possible to generate `sample_event.json` files for each of the integration’s data streams while running these tests. + +```bash +elastic-package test system --generate +``` diff --git a/docs/extend/testing-new-indexing-features.md b/docs/extend/testing-new-indexing-features.md new file mode 100644 index 00000000000..ce0bc77e834 --- /dev/null +++ b/docs/extend/testing-new-indexing-features.md @@ -0,0 +1,170 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/testing-new-indexing-features.html +--- + +# How to test new indexing features [testing-new-indexing-features] + +Elasticsearch has been adding new indexing modes and features that allow optimization of storage size and query performance. + +We’d like to enable integration developers to start testing the ingest and query performance of enabling these features before we start making any changes in the integrations themselves or allowing end users to enable these from the Fleet UI. + +Today, each of these can already be enabled by leveraging the `*@custom` component templates that Fleet installs for each integration data stream, to varying degrees of ease of use (details below). We could improve the UX around this for integration developers by adding an explicit API in Fleet to enable this, however it may not be necessary. See [elastic/kibana#132818](https://github.com/elastic/kibana/issues/132818) for discussion around how a feature flag API could be added to ease this a bit more. + +See the following instructions for testing new indexing features: + +* [Testing synthetic source](#integrations-dev-synthetic-source) +* [Testing `doc-value-only` fields](#integrations-dev-doc-value-only-fields) +* [Time-series indexing (TSDS)](#integrations-dev-test-tsds) + +## Testing synthetic source [integrations-dev-synthetic-source] + +* For background, refer to [#85649](https://github.com/elastic/elasticsearch/pull/85649) +* For integrations support, refer to [#340](https://github.com/elastic/package-spec/pull/340) + +This feature is quite easy to enable on an integration using the component template. Here’s how to do this for the `nginx` substatus metrics, for example: + +1. Install the nginx package. +2. Run this dev tools command: + + ```console + PUT /_component_template/metrics-nginx.substatus@custom + { + "template": { + "settings": {}, + "mappings": { + "_source": { + "mode": "synthetic" + } + } + }, + "_meta": { + "package": { + "name": "nginx" + } + } + } + ``` + +3. If a data stream already exists, rollover the data stream to get the new mappings: `POST metrics-nginx.substatus-default/_rollover` + +One challenge with leveraging synthetic source is that it doesn’t support keyword fields that have `ignore_above` configured. It may be worth removing this setting for testing on those fields. This can be done by editing the package in `dev` and installing it via `elastic-package` or overriding it via the custom component template, similar to the [`doc-value-only`](#integrations-dev-doc-value-only-fields) example. + + +## Testing `doc-value-only` fields [integrations-dev-doc-value-only-fields] + +* For background, refer to [Elasticsearch, Kibana, Elastic Cloud 8.1: Faster indexing, less disk storage, and smarter analytics capabilities](https://www.elastic.co/blog/whats-new-elasticsearch-kibana-cloud-8-1-0). +* For integrations support, refer to [#3419](https://github.com/elastic/integrations/issues/3419). + +This feature is more challenging with component templates because it requires adding `index: false` to every long and double field. Providing an API in Fleet would make this a bit easier. Here’s how to do this manually: + +1. Install the `nginx` package. +2. Get the mappings included with the package: `GET /_component_template/logs-nginx.access@package`. +3. Copy the output into your favorite text editor, search for each `"type": "long"` and `"type": "double"`, and add `"index": false`. +4. Update the custom component template with the new mappings. For example, here’s how to set the long fields to `index: false`: + + ```console + PUT /_component_template/merics-nginx.substatus@custom + { + "template": { + "settings": {}, + "mappings": { + "properties": { + "nginx": { + "properties": { + "stubstatus": { + "properties": { + "hostname": { + "ignore_above": 1024, + "type": "keyword" + }, + "current": { + "type": "long", + "index": false + }, + "waiting": { + "type": "long", + "index": false + }, + "accepts": { + "type": "long", + "index": false + }, + "handled": { + "type": "long", + "index": false + }, + "writing": { + "type": "long", + "index": false + }, + "dropped": { + "type": "long", + "index": false + }, + "active": { + "type": "long", + "index": false + }, + "reading": { + "type": "long", + "index": false + }, + "requests": { + "type": "long", + "index": false + } + } + } + } + } + } + } + }, + "_meta": { + "package": { + "name": "nginx" + } + } + } + ``` + +5. If a data stream already exists, rollover the data stream to get the new mappings: `POST metrics-nginx.substatus-default/_rollover` + + +## Time-series indexing (TSDS) [integrations-dev-test-tsds] + +* For background, refer to [#74660](https://github.com/elastic/elasticsearch/issues/74660) +* For integrations support, refer to [#311](https://github.com/elastic/package-spec/issues/311) + +Usage of TSDS indexing requires the following: + +* Mapping parameters must be added for `time_series_dimension` and `time_series_metric` on appropriate fields. This is already supported by the package ecosystem and Fleet, so packages can already define these options. +* The `mode: time_series` and `routing_path` index settings must be added, this can be done by editing the custom component template. + +Note that the `routing_path` setting should correspond to fields with `time_series_dimension` specified. In the future, ES may automate this setting. + +1. Install the kubernetes package (already has TSDS mappings set up) +2. Run this dev tools command: + + ```console + PUT /_component_template/metrics-kubernetes.pod@custom + { + "template": { + "settings": { + "index.mode": "time_series", + "index.routing_path": ["kubernetes.pod.uid"] + }, + "mappings": {} + }, + "_meta": { + "package": { + "name": "kubernetes" + } + } + } + ``` + +3. If a data stream already existed, rollover the data stream to get the new mappings: `POST metrics-kubernetes.pod-default/_rollover` + + diff --git a/docs/extend/testing-validation.md b/docs/extend/testing-validation.md new file mode 100644 index 00000000000..66b74e23199 --- /dev/null +++ b/docs/extend/testing-validation.md @@ -0,0 +1,118 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/testing-and-validation.html +--- + +# Testing and validation [testing-and-validation] + +1. Build the package you’d like to verify (e.g. `apache`): + + ```bash + cd apache + elastic-package build + ``` + +2. Start the testing environment: + + Run from inside the Integrations repository: + + ```bash + elastic-package stack up -d -v + ``` + + The command above will boot up the {{stack}} ({{es}}, {{kib}}, and {{package-registry}}) using Docker containers. It rebuilds the {{package-registry}} Docker image using packages built in step 1. and boots up the {{package-registry}}. + + To reload the already deployed {{package-registry}}, use the following command: + + ```bash + elastic-package stack up -v -d --services package-registry + ``` + +3. Verify that your integration is available in the correct version. For example, MySQL: [http://localhost:8080/search?package=mysql](http://localhost:8080/search?package=mysql) (use `experimental=true` parameter if the package is in experimental version. Alternatively set `release` to `beta` or higher in your package’s `manifest.yml`, if appropriate.) + + ```json + [ + { + "description": "MySQL Integration", + "download": "/epr/mysql/mysql-0.0.1.tar.gz", + "icons": [ + { + "src": "/package/mysql/0.0.1/img/logo_mysql.svg", + "title": "logo mysql", + "size": "32x32", + "type": "image/svg+xml" + } + ], + "name": "mysql", + "path": "/package/mysql/0.0.1", + "title": "MySQL", + "type": "integration", + "version": "0.0.1" + } + ] + ``` + + The `elastic-package stack` provides an enrolled instance of the {{agent}}. Use that one instead of a local application if you can run the service (you’re integrating with) in the Docker network and you don’t need to rebuild the Elastic-Agent or it’s subprocesses (e.g. {{filebeat}} or {{metricbeat}}). The service Docker image can be used for <=7.10.0`. Otherwise the package is also in 8.0.0 but we do not know today if it will actually be compatible with >= 8.0.0. + + ```yaml + conditions: + kibana.version: '^7.10.0' + ``` + +4. Set the proper package owner (either Github team or personal account) + + Good candidates for a team: `elastic/integrations`, `elastic/security-service-integrations` + + Update the `.github/CODEOWNERS` file accordingly. + + + + +## All integrations [_all_integrations] + +### Development [_development] + +1. When you’re developing integrations and you’d like to propagate your changes to the package registry, first rebuild the package: + + ```bash + $ cd packages/apache + $ elastic-package build + ``` + + Then, rebuild and redeploy the Package Registry: + + *It’s important to execute the following command in the Integrations repository.* + + ```bash + $ elastic-package stack up -v -d --services package-registry + ``` + + Explanation: it’s much faster to rebuild and restart the container with the Package Registry, than work with mounted volumes. + + + +### Code reviewers [_code_reviewers] + +1. Ping "Team:Integrations". + + Use the team label to notify relevant team members about the incoming pull request. + + +#### Manifest files [_manifest_files_2] + +1. Descriptions of configuration options should be as short as possible. + + Remember to keep only the meaningful information about the configuration option. + + Good candidates: references to the product configuration, accepted string values, explanation. + + Bad candidates: *Collect metrics from A, B, C, D,…​ X, Y, Z datasets.* + +2. Descriptions should be human readable. + + Try to rephrase sentences like: *Collect foo_Bar3 metrics*, into *Collect Foo Bar metrics*. + +3. Description should be easy to understand. + + Simplify sentences, don’t provide information about the input if not required. + + Bad candidate: *Collect application logs (log input)* + + Good candidates: *Collect application logs*, *Collect standard logs for the application* + +4. Letter casing is important for screenshot descriptions. + + These descriptions are visualized in the Kibana UI. It would be better experience to have them clean and consistent. + + Bad candidate: *filebeat running on ec2 machine* + + Good candidates: *Filebeat running on AWS EC2 machine* + +5. If package relies on some feature or a field, available only in a specific stack or beats version, `kibana.version` condition should be adjusted accordingly in the package’s `manifest.yml`: + + ```yaml + conditions: + kibana.version: '^8.7.0' + ``` + + ::::{note} + The package version with such condition as above will be only available in Kibana version >=8.7.0 + :::: + + + ::::{note} + Changing dashboards and visualizations using an unreleased version of Kibana might be unsafe since the Kibana Team might make changes to the Kibana code and potentially the data models. There is no guarantee that your changes won’t be broken by the time new Kibana version is released. + :::: + + + +#### CI [_ci] + +1. Run `elastic-package check` and `elastic-package test` locally. + + If you want to verify if your integration works as intended, you can execute the same steps as CI: + + ```bash + $ cd packages/apache + $ elastic-package check -v + $ elastic-package test -v + ``` + + Keep in mind that the `elastic-package test` command requires a live cluster running and exported environment variables. The environment variables can be set with `eval "$(elastic-package stack shellinit)"`. + + + +#### Fields [_fields] + +1. Remove empty fields files. + + If you notice that fields file (e.g. `package-fields.yml`) doesn’t contain any field definitions or it defines root only, feel free to remove it. + + Bad candidate: + + ```yaml + - name: mypackage.mydataset + type: group + ``` + + + + + diff --git a/docs/extend/toc.yml b/docs/extend/toc.yml new file mode 100644 index 00000000000..acb4175f9d0 --- /dev/null +++ b/docs/extend/toc.yml @@ -0,0 +1,51 @@ +toc: + - file: index.md + - file: what-is-an-integration.md + children: + - file: integration-definitions.md + - file: quick-start.md + - file: build-new-integration.md + children: + - file: build-overview.md + - file: build-spin-stack.md + - file: build-create-package.md + - file: add-data-stream.md + - file: define-deployment-modes.md + - file: edit-ingest-pipeline.md + - file: add-mapping.md + - file: create-dashboards.md + - file: build-it.md + - file: testing-validation.md + - file: finishing-touches.md + - file: tips-for-building.md + - file: upload-new-integration.md + - file: testing.md + children: + - file: asset-testing.md + - file: pipeline-testing.md + - file: static-testing.md + - file: system-testing.md + - file: _publish_an_integration.md + - file: developer-workflows.md + children: + - file: developer-workflow-fleet-UI.md + - file: developer-workflow-support-old-package.md + - file: developer-workflow-import-beat.md + - file: integrations-guidelines.md + children: + - file: general-guidelines.md + - file: dashboard-guidelines.md + - file: documentation-guidelines.md + - file: integrations-tsds-synthetic-source.md + children: + - file: developer-tsds-guidelines.md + - file: testing-new-indexing-features.md + - file: elastic-package.md + - file: package-spec.md + children: + - file: dev-spec.md + - file: data-stream-spec.md + - file: docs-spec.md + - file: kibana-spec.md + - file: changelog-spec.md + - file: manifest-spec.md \ No newline at end of file diff --git a/docs/extend/upload-new-integration.md b/docs/extend/upload-new-integration.md new file mode 100644 index 00000000000..078d2b82a00 --- /dev/null +++ b/docs/extend/upload-new-integration.md @@ -0,0 +1,47 @@ +--- +navigation_title: "Upload an integration" +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/upload-a-new-integration.html +--- + +# Upload an integration to Kibana [upload-a-new-integration] + + +{{fleet}} supports integration installation through direct upload as a means to support integration developers or users who have created custom integrations that they don’t want to commit upstream back to the [Elastic Integrations repository](https://github.com/elastic/integrations). + +Direct upload can also be useful in air-gapped environments, by providing a way to update integrations without needing to update a self-hosted package registry. + + +## Local development [upload-integration-local] + +If you’ve followed the local development steps in [*Build an integration*](/extend/build-new-integration.md), upload your integration to Kibana with the following command: + +```bash +elastic-package install --zip /path/to/my/custom-integration +``` + +For more information, see [`elastic-package install`](/extend/elastic-package.md#elastic-package-install). + + +## Production deployment [upload-integration-production] + +To upload your integration to a production deployment, first zip the package: + +```bash +$ cd /path/to/my/custom-integration +$ elastic-package build +``` + +You can now use the Kibana API to upload your integration: + +```bash +$ curl -XPOST \ + -H 'content-type: application/zip' \ + -H 'kbn-xsrf: true' \ + http://your.kibana.host/api/fleet/epm/packages \ + -u {username}:{password} \ + --data-binary @my-custom-integration.zip +``` + +More information on this endpoint is available in the [Fleet API Reference](https://www.elastic.co/guide/en/fleet/current/fleet-apis.html). + diff --git a/docs/extend/what-is-an-integration.md b/docs/extend/what-is-an-integration.md new file mode 100644 index 00000000000..948f16610e4 --- /dev/null +++ b/docs/extend/what-is-an-integration.md @@ -0,0 +1,53 @@ +--- +mapped_pages: + - https://www.elastic.co/guide/en/integrations-developer/current/index.html + - https://www.elastic.co/guide/en/integrations-developer/current/what-is-an-integration.html +--- + +# What is an integration? [what-is-an-integration] + +An Elastic integration is a collection of assets that defines how to observe a specific product or service with the {{stack}}: + +* Data ingest, storage, and transformation rules +* Configuration options +* Pre-built, custom dashboards and visualizations +* Documentation +* Tests + +Integrations have a strict, well-defined structure, and offer a number of benefits over other ingest options: + +* Structured around the service that is being observed—​not the monitoring agent +* Easy, less error-prone configuration +* Fewer monitoring agents for users to install +* Deploy in just a few clicks +* Decoupled release process from the {stack} + + +## Integration lifecycle [how-integrations-work] + +1. Create a source package + + All integrations start as a source package. You’ll find most Elastic integrations in the [`elastic/integrations`](https://github.com/elastic/integrations) repository, but a package can live anywhere. + + All packages must adhere to the [package specification](/extend/package-spec.md) — a formal spec used for the creation and validation of new or updated integrations. + +2. Publish the integration to the package registry + + Once an integration (package) has been created, it needs to be built. Built integrations are stored in the [Package Storage repository](https://github.com/elastic/package-storage) and served up via the [{{package-registry}}](https://github.com/elastic/package-registry). The {{fleet}} UI in {{kib}} connects to the {{package-registry}} and allows users to discover, install, and configure Elastic Packages. The {{package-registry}} can also be [deployed on-premise in air-gapped environments](docs-content://reference/ingestion-tools/fleet/air-gapped.md#air-gapped-diy-epr). + +3. Install the integration + + Using {{fleet}} in {{kib}}, install the integration and add it to an {{agent}} policy. When you install a package, its assets are unpacked and installed into {{es}} and {{kib}} using {{stack}} APIs. In addition, configuration for the package is persisted in {{es}} as an {{agent}} policy. + +4. Add the policy with the integration to an {{agent}}. + + Once the policy with an integration is added to an {{agent}}, the {{agent}} will begin to collect and ship data to the {{stack}} based on the Elastic integration. + + Package assets may come into play here. For example, if a package installed ingest pipelines, those will intercept the data and transform it before it is indexed. + +5. Visualize the results + + Integrations can and should ship with custom dashboards and visualizations that are installed with the integration. Use these for a tailored view of your {{observability}} data. + + + diff --git a/docs/images/backport_input_step.png b/docs/images/backport_input_step.png new file mode 100644 index 00000000000..0ff21d04163 Binary files /dev/null and b/docs/images/backport_input_step.png differ diff --git a/docs/images/browse_package_commits.png b/docs/images/browse_package_commits.png new file mode 100644 index 00000000000..52a619a8491 Binary files /dev/null and b/docs/images/browse_package_commits.png differ diff --git a/docs/images/build.png b/docs/images/build.png new file mode 100644 index 00000000000..d91bd12c4fa Binary files /dev/null and b/docs/images/build.png differ diff --git a/docs/images/colors-in-visualizations.png b/docs/images/colors-in-visualizations.png new file mode 100644 index 00000000000..a58c8fe2138 Binary files /dev/null and b/docs/images/colors-in-visualizations.png differ diff --git a/docs/images/datastream-log-message.png b/docs/images/datastream-log-message.png new file mode 100644 index 00000000000..9887f64a27b Binary files /dev/null and b/docs/images/datastream-log-message.png differ diff --git a/docs/images/filter-in-visualization.png b/docs/images/filter-in-visualization.png new file mode 100644 index 00000000000..8b2e29707f0 Binary files /dev/null and b/docs/images/filter-in-visualization.png differ diff --git a/docs/images/grouping-in-visualizations.png b/docs/images/grouping-in-visualizations.png new file mode 100644 index 00000000000..f685eff9ef7 Binary files /dev/null and b/docs/images/grouping-in-visualizations.png differ diff --git a/docs/images/markdown-grouping.png b/docs/images/markdown-grouping.png new file mode 100644 index 00000000000..94389f92cd9 Binary files /dev/null and b/docs/images/markdown-grouping.png differ diff --git a/docs/images/merge_commit_message.png b/docs/images/merge_commit_message.png new file mode 100644 index 00000000000..06d72861d79 Binary files /dev/null and b/docs/images/merge_commit_message.png differ diff --git a/docs/images/package-installed.png b/docs/images/package-installed.png new file mode 100644 index 00000000000..0abb52aaf45 Binary files /dev/null and b/docs/images/package-installed.png differ diff --git a/docs/images/rows-in-visualizations.png b/docs/images/rows-in-visualizations.png new file mode 100644 index 00000000000..a9666e71fa2 Binary files /dev/null and b/docs/images/rows-in-visualizations.png differ diff --git a/docs/images/titles-in-visualizations.png b/docs/images/titles-in-visualizations.png new file mode 100644 index 00000000000..a0577adef3a Binary files /dev/null and b/docs/images/titles-in-visualizations.png differ