From 7b96040a504f5208838e533d0a0117ffa086589b Mon Sep 17 00:00:00 2001
From: David Kilfoyle <david.kilfoyle@elastic.co>
Date: Fri, 9 Feb 2024 10:45:11 -0500
Subject: [PATCH 1/2] Add known issue for stuck agent upgrade (8.12.0, 8.12.1)

---
 .../release-notes/release-notes-8.12.asciidoc | 84 +++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
index 034d5494d..89ecfda5f 100644
--- a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
+++ b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
@@ -132,6 +132,50 @@ So, if you are relying on an integration level custom ingest pipeline introduced
 Refer to the <<data-streams-pipelines,Ingest pipelines>> documentation for details and examples.
 ====
 
+[discrete]
+[[known-issues-8.12.1]]
+=== Known issues
+
+[[known-issue-3263-8121]]
+.Agents upgraded to 8.12.0 are stuck in a non-upgradeable state
+[%collapsible]
+====
+
+*Details*
+
+An issue discovered in {fleet-server} prevents {agents} that have been upgraded to version 8.12.0 from being upgraded again, using the {fleet} UI, to version 8.12.1 or higher.
+
+*Impact* +
+
+As a workaround, we recommend you to use the {kib} {fleet} API to upgrade these agents using the `force` flag.
+
+To upgrade a single {agent}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/<agent_id>/upgrade
+{
+  "version": "8.12.1",
+  "force": true
+}
+----
+
+To bulk upgrade a set of {agents}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/bulk_upgrade
+  {
+    "agents": "agent.version:8.12.0",
+    "version": "8.12.1",
+    "force": true
+  }
+----
+
+This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+
+====
+
 [discrete]
 [[bug-fixes-8.12.1]]
 === Bug fixes
@@ -405,6 +449,46 @@ Note that according to our <<agent-policy-scaling-recommendations,policy scaling
 
 ====
 
+[[known-issue-3263-8120]]
+.Agents upgraded to 8.12.0 are stuck in a non-upgradeable state
+[%collapsible]
+====
+
+*Details*
+
+An issue discovered in {fleet-server} prevents {agents} that have been upgraded to version 8.12.0 from being upgraded again, using the {fleet} UI, to version 8.12.1 or higher.
+
+*Impact* +
+
+As a workaround, we recommend you to use the {kib} {fleet} API to upgrade these agents using the `force` flag.
+
+To upgrade a single {agent}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/<agent_id>/upgrade
+{
+  "version": "8.12.1",
+  "force": true
+}
+----
+
+To bulk upgrade a set of {agents}:
+
+[source,"shell"]
+----
+POST kbn:/api/fleet/agents/bulk_upgrade
+  {
+    "agents": "agent.version:8.12.0",
+    "version": "8.12.1",
+    "force": true
+  }
+----
+
+This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+
+====
+
 [discrete]
 [[new-features-8.12.0]]
 === New features

From cffb37bd9e469630f607ac175c35ec698d0ab7e0 Mon Sep 17 00:00:00 2001
From: David Kilfoyle <david.kilfoyle@elastic.co>
Date: Mon, 12 Feb 2024 09:38:46 -0500
Subject: [PATCH 2/2] Fix workaround steps

---
 .../release-notes/release-notes-8.12.asciidoc | 61 +++++++++++++------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
index 89ecfda5f..79806d67e 100644
--- a/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
+++ b/docs/en/ingest-management/release-notes/release-notes-8.12.asciidoc
@@ -458,35 +458,62 @@ Note that according to our <<agent-policy-scaling-recommendations,policy scaling
 
 An issue discovered in {fleet-server} prevents {agents} that have been upgraded to version 8.12.0 from being upgraded again, using the {fleet} UI, to version 8.12.1 or higher.
 
+This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+
 *Impact* +
 
-As a workaround, we recommend you to use the {kib} {fleet} API to upgrade these agents using the `force` flag.
-
-To upgrade a single {agent}:
+As a workaround, we recommend you to use the {kib} {fleet} API to update any documents in which `upgrade_details` is either `null` or not defined. Note that these steps must be run as a superuser.
 
 [source,"shell"]
 ----
-POST kbn:/api/fleet/agents/<agent_id>/upgrade
-{
-  "version": "8.12.1",
-  "force": true
-}
+ POST _security/role/fleet_superuser
+ {
+    "indices": [
+        {
+            "names": [".fleet*",".kibana*"],
+            "privileges": ["all"],
+            "allow_restricted_indices": true
+        }
+    ]
+  }
 ----
 
-To bulk upgrade a set of {agents}:
-
 [source,"shell"]
 ----
-POST kbn:/api/fleet/agents/bulk_upgrade
-  {
-    "agents": "agent.version:8.12.0",
-    "version": "8.12.1",
-    "force": true
-  }
+POST _security/user/fleet_superuser 
+ {
+    "password": "password",
+    "roles": ["superuser", "fleet_superuser"]
+ }
 ----
 
-This issue is planned to be fixed in versions 8.12.2 and 8.13.0.
+[source,"shell"]
+----
+curl -sk -XPOST --user fleet_superuser:password -H 'content-type:application/json' \
+  -H'x-elastic-product-origin:fleet' \
+  http://localhost:9200/.fleet-agents/_update_by_query \
+  -d '{
+  "script": {
+    "source": "ctx._source.remove(\"upgrade_details\")",
+    "lang": "painless"
+  },
+  "query": {
+    "bool": {
+        "must_not": {
+          "exists": {
+            "field": "upgrade_details"
+          }
+        }
+      }
+    }
+}'
+----
 
+[source,"shell"]
+----
+DELETE _security/user/fleet_superuser
+DELETE _security/role/fleet_superuser
+----
 ====
 
 [discrete]