Generate github/gitlab issue bodies from a template

CHANGES.md

@@ -1,3 +1,12 @@
+UNRELEASED
+==========
+
+Features
+--------
+
+- Allow configuration of the body of created Github/Gitlab issues via a template in the configuration file. ([\#84](https://github.com/matrix-org/rageshake/issues/84))
+
+
 1.11.0 (2023-08-11)
 ===================
 

README.md

@@ -18,6 +18,29 @@ Optional parameters:
  * `-listen <address>`: TCP network address to listen for HTTP requests
    on. Example: `:9110`.
 
+## Issue template
+
+It is possible to specify a template in the configuration file which will be used to build the
+body of any issues created on Github or Gitlab, via the `issue_body_template` setting.
+See [rageshake.sample.yaml](rageshake.sample.yaml) for an example.
+
+See https://pkg.go.dev/text/template#pkg-overview for documentation of the template language.
+
+The following properties are defined on the input (accessible via `.` or `$`):
+
+| Name         | Type                | Description                                                                                       |
+|--------------|---------------------|---------------------------------------------------------------------------------------------------|
+| `ID`         | `string`            | The unique ID for this rageshake.                                                                 |
+| `UserText`   | `string`            | A multi-line string containing the user description of the fault (from `text` in the submission). |
+| `AppName`    | `string`            | A short slug to identify the app making the report (from `app` in the submission).                |
+| `Labels`     | `[]string`          | A list of labels requested by the application.                                                    |
+| `Data`       | `map[string]string` | A map of other key/value pairs included in the submission.                                        |
+| `Logs`       | `[]string`          | A list of log file names.                                                                         |
+| `LogErrors`  | `[]string`          | Set if there are log parsing errors.                                                              |
+| `Files`      | `[]string`          | A list of other files (not logs) uploaded as part of the rageshake.                               |
+| `FileErrors` | `[]string`          | Set if there are file parsing errors.                                                             |
+| `ListingURL` | `string`            | Complete link to the listing URL that contains all uploaded logs.                                 |
+
 ## HTTP endpoints
 
 The following HTTP endpoints are exposed:

main.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"os"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/google/go-github/github"
@@ -37,6 +38,20 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+// DefaultIssueBodyTemplate is the default template used for `issue_body_template` in the config.
+//
+// !!! Keep in step with the documentation in `rageshake.sample.yaml` !!!
+const DefaultIssueBodyTemplate = `User message:
+{{ .UserText }}
+
+{{ range $key, $val := .Data -}}
+{{ $key }}: ` + "`{{ $val }}`" + `
+{{ end }}
+[Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz))
+{{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }})
+{{- end }}
+`
+
 var configPath = flag.String("config", "rageshake.yaml", "The path to the config file. For more information, see the config file in this repository.")
 var bindAddr = flag.String("listen", ":9110", "The port to listen on.")
 
@@ -63,6 +78,8 @@ type config struct {
 	GitlabProjectLabels     map[string][]string `yaml:"gitlab_project_labels"`
 	GitlabIssueConfidential bool                `yaml:"gitlab_issue_confidential"`
 
+	IssueBodyTemplate string `yaml:"issue_body_template"`
+
 	SlackWebhookURL string `yaml:"slack_webhook_url"`
 
 	EmailAddresses []string `yaml:"email_addresses"`
@@ -158,7 +175,16 @@ func main() {
 	log.Printf("Using %s/listing as public URI", apiPrefix)
 
 	rand.Seed(time.Now().UnixNano())
-	http.Handle("/api/submit", &submitServer{ghClient, glClient, apiPrefix, slack, genericWebhookClient, appNameMap, cfg})
+	http.Handle("/api/submit", &submitServer{
+		issueTemplate:        parseIssueTemplate(cfg),
+		ghClient:             ghClient,
+		glClient:             glClient,
+		apiPrefix:            apiPrefix,
+		slack:                slack,
+		genericWebhookClient: genericWebhookClient,
+		allowedAppNameMap:    appNameMap,
+		cfg:                  cfg,
+	})
 
 	// Make sure bugs directory exists
 	_ = os.Mkdir("bugs", os.ModePerm)
@@ -186,6 +212,18 @@ func main() {
 	log.Fatal(http.ListenAndServe(*bindAddr, nil))
 }
 
+func parseIssueTemplate(cfg *config) *template.Template {
+	issueTemplate := cfg.IssueBodyTemplate
+	if issueTemplate == "" {
+		issueTemplate = DefaultIssueBodyTemplate
+	}
+	parsedIssueTemplate, err := template.New("issue").Parse(issueTemplate)
+	if err != nil {
+		log.Fatalf("Invalid `issue_template` in config file: %s", err)
+	}
+	return parsedIssueTemplate
+}
+
 func configureAppNameMap(cfg *config) map[string]bool {
 	if len(cfg.AllowedAppNames) == 0 {
 		fmt.Println("Warning: allowed_app_names is empty. Accepting requests from all app names")

rageshake.sample.yaml

@@ -21,6 +21,19 @@ github_token: secrettoken
 github_project_mappings:
    my-app: octocat/HelloWorld
 
+# A template for the body of Github and Gitlab issues. The default template is as shown below.
+#
+# See `README.md` for more information on what can be specified here.
+issue_body_template: |
+  {{ .UserText }}
+  
+  {{ range $key, $val := .Data -}}
+  {{ $key }}: `{{ $val }}`
+  {{ end }}
+  [Logs]({{ .ListingURL }}) ([archive]({{ .ListingURL }}?format=tar.gz))
+  {{- range $file := .Files}} / [{{ $file }}]({{ $.ListingURL }}/{{ $file }})
+  {{- end }}
+
 # a GitLab personal access token (https://gitlab.com/-/profile/personal_access_tokens), which
 # will be used to create a GitLab issue for each report. It requires
 # `api` scope. If omitted, no issues will be created.
@@ -55,7 +68,6 @@ smtp_server: localhost:25
 smtp_username: myemailuser
 smtp_password: myemailpass
 
-
 # a list of webhook URLs, (see docs/generic_webhook.md)
 generic_webhook_urls: 
   - https://server.example.com/your-server/api

submit.go

@@ -37,6 +37,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"text/template"
 	"time"
 
 	"github.com/google/go-github/github"
@@ -47,6 +48,9 @@ import (
 var maxPayloadSize = 1024 * 1024 * 55 // 55 MB
 
 type submitServer struct {
+	// Template for building github and gitlab issues
+	issueTemplate *template.Template
+
 	// github client for reporting bugs. may be nil, in which case,
 	// reporting is disabled.
 	ghClient *github.Client
@@ -78,6 +82,15 @@ type jsonLogEntry struct {
 	Lines string `json:"lines"`
 }
 
+// `issueBodyTemplatePayload` contains the data made available to the `issue_body_template`.
+//
+// !!! Keep in step with the documentation in `README.md` !!!
+type issueBodyTemplatePayload struct {
+	payload
+	// Complete link to the listing URL that contains all uploaded logs
+	ListingURL string
+}
+
 // Stores additional information created during processing of a payload
 type genericWebhookPayload struct {
 	payload
@@ -87,7 +100,10 @@ type genericWebhookPayload struct {
 	ListingURL string `json:"listing_url"`
 }
 
-// Stores information about a request made to this server
+// `payload` stores information about a request made to this server.
+//
+// !!! Since this is inherited by `issueBodyTemplatePayload`, remember to keep it in step
+// with the documentation in `README.md` !!!
 type payload struct {
 	// A unique ID for this payload, generated within this server 
 	ID         string            `json:"id"`
@@ -580,7 +596,7 @@ func (s *submitServer) submitGithubIssue(ctx context.Context, p payload, listing
 	}
 	owner, repo := splits[0], splits[1]
 
-	issueReq, err := buildGithubIssueRequest(p, listingURL)
+	issueReq, err := buildGithubIssueRequest(p, listingURL, s.issueTemplate)
 	if err != nil {
 		return err
 	}
@@ -605,7 +621,7 @@ func (s *submitServer) submitGitlabIssue(p payload, listingURL string, resp *sub
 	glProj := s.cfg.GitlabProjectMappings[p.AppName]
 	glLabels := s.cfg.GitlabProjectLabels[p.AppName]
 
-	issueReq, err := buildGitlabIssueRequest(p, listingURL, glLabels, s.cfg.GitlabIssueConfidential)
+	issueReq, err := buildGitlabIssueRequest(p, listingURL, s.issueTemplate, glLabels, s.cfg.GitlabIssueConfidential)
 	if err != nil {
 		return err
 	}
@@ -671,31 +687,26 @@ func buildReportBody(p payload, newline, quoteChar string) *bytes.Buffer {
 	return &bodyBuf
 }
 
-func buildGenericIssueRequest(p payload, listingURL string) (title, body string, err error) {
-	bodyBuf := buildReportBody(p, "  \n", "`")
+func buildGenericIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (title, body string, err error) {
+	var bodyBuf bytes.Buffer
 
-	// Add log links to the body
-	fmt.Fprintf(bodyBuf, "\n[Logs](%s)", listingURL)
-	fmt.Fprintf(bodyBuf, " ([archive](%s))", listingURL+"?format=tar.gz")
+	issuePayload := issueBodyTemplatePayload{
+		payload:    p,
+		ListingURL: listingURL,
+	}
 
-	for _, file := range p.Files {
-		fmt.Fprintf(
-			bodyBuf,
-			" / [%s](%s)",
-			file,
-			listingURL+"/"+file,
-		)
+	if err = bodyTemplate.Execute(&bodyBuf, issuePayload); err != nil {
+		return
 	}
 
 	title = buildReportTitle(p)
-
 	body = bodyBuf.String()
 
 	return
 }
 
-func buildGithubIssueRequest(p payload, listingURL string) (*github.IssueRequest, error) {
-	title, body, err := buildGenericIssueRequest(p, listingURL)
+func buildGithubIssueRequest(p payload, listingURL string, bodyTemplate *template.Template) (*github.IssueRequest, error) {
+	title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate)
 	if err != nil {
 		return nil, err
 	}
@@ -712,8 +723,8 @@ func buildGithubIssueRequest(p payload, listingURL string) (*github.IssueRequest
 	}, nil
 }
 
-func buildGitlabIssueRequest(p payload, listingURL string, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) {
-	title, body, err := buildGenericIssueRequest(p, listingURL)
+func buildGitlabIssueRequest(p payload, listingURL string, bodyTemplate *template.Template, labels []string, confidential bool) (*gitlab.CreateIssueOptions, error) {
+	title, body, err := buildGenericIssueRequest(p, listingURL, bodyTemplate)
 	if err != nil {
 		return nil, err
 	}

submit_test.go

@@ -28,6 +28,7 @@ import (
 	"strconv"
 	"strings"
 	"testing"
+	"text/template"
 )
 
 // testParsePayload builds a /submit request with the given body, and calls
@@ -56,11 +57,10 @@ func testParsePayload(t *testing.T, body, contentType string, tempDir string) (*
 	return p, rr.Result()
 }
 
-
 func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool, body string) int {
-        // Submit a request without files to the server and return statusCode
-        // Could be extended with more complicated config; aimed here just to
-        // test options for allowedAppNameMap
+	// Submit a request without files to the server and return statusCode
+	// Could be extended with more complicated config; aimed here just to
+	// test options for allowedAppNameMap
 
 	req, err := http.NewRequest("POST", "/api/submit", strings.NewReader(body))
 	if err != nil {
@@ -70,7 +70,7 @@ func submitSimpleRequestToServer(t *testing.T, allowedAppNameMap map[string]bool
 	w := httptest.NewRecorder()
 
 	var cfg config
-	s := &submitServer{nil, nil, "/", nil, nil, allowedAppNameMap, &cfg}
+	s := &submitServer{nil, nil, nil, "/", nil, nil, allowedAppNameMap, &cfg}
 
 	s.ServeHTTP(w, req)
 	rsp := w.Result()
@@ -406,6 +406,63 @@ func mkTempDir(t *testing.T) string {
  * buildGithubIssueRequest tests
  */
 
+// General test of Github issue formatting.
+func TestBuildGithubIssue(t *testing.T) {
+	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="text"
+
+
+test words.
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="app"
+
+riot-web
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="User-Agent"
+
+xxx
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="user_id"
+
+id
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="device_id"
+
+id
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="version"
+
+1
+------WebKitFormBoundarySsdgl8Nq9voFyhdO
+Content-Disposition: form-data; name="file"; filename="passwd.txt"
+
+file
+------WebKitFormBoundarySsdgl8Nq9voFyhdO--
+`
+	p, _ := testParsePayload(t, body,
+		"multipart/form-data; boundary=----WebKitFormBoundarySsdgl8Nq9voFyhdO",
+		"",
+	)
+
+	if p == nil {
+		t.Fatal("parseRequest returned nil")
+	}
+
+	parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate))
+	issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate)
+	if err != nil {
+		t.Fatalf("Error building issue request: %s", err)
+	}
+
+	if *issueReq.Title != "test words." {
+		t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.")
+	}
+	expectedBody := "User message:\n\ntest words.\n\nUser-Agent: `xxx`\nVersion: `1`\ndevice_id: `id`\nuser_id: `id`\n\n[Logs](http://test/listing/foo) ([archive](http://test/listing/foo?format=tar.gz)) / [passwd.txt](http://test/listing/foo/passwd.txt)\n"
+	if *issueReq.Body != expectedBody {
+		t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody)
+	}
+}
+
 func TestBuildGithubIssueLeadingNewline(t *testing.T) {
 	body := `------WebKitFormBoundarySsdgl8Nq9voFyhdO
 Content-Disposition: form-data; name="text"
@@ -427,12 +484,16 @@ riot-web
 		t.Fatal("parseRequest returned nil")
 	}
 
-	issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo")
+	parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate))
+	issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate)
+	if err != nil {
+		t.Fatalf("Error building issue request: %s", err)
+	}
 
 	if *issueReq.Title != "test words." {
 		t.Errorf("Title: got %s, want %s", *issueReq.Title, "test words.")
 	}
-	expectedBody := "User message:\n\n\ntest words.\n"
+	expectedBody := "User message:\n\ntest words.\n"
 	if !strings.HasPrefix(*issueReq.Body, expectedBody) {
 		t.Errorf("Body: got %s, want %s", *issueReq.Body, expectedBody)
 	}
@@ -453,7 +514,11 @@ Content-Disposition: form-data; name="text"
 		t.Fatal("parseRequest returned nil")
 	}
 
-	issueReq := buildGithubIssueRequest(*p, "http://test/listing/foo")
+	parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate))
+	issueReq, err := buildGithubIssueRequest(*p, "http://test/listing/foo", parsedIssueTemplate)
+	if err != nil {
+		t.Fatalf("Error building issue request: %s", err)
+	}
 
 	if *issueReq.Title != "Untitled report" {
 		t.Errorf("Title: got %s, want %s", *issueReq.Title, "Untitled report")
@@ -464,7 +529,7 @@ Content-Disposition: form-data; name="text"
 	}
 }
 
-func TestTestSortDataKeys(t *testing.T) {
+func TestSortDataKeys(t *testing.T) {
 	expect := `
 Number of logs: 0
 Application: 
@@ -506,9 +571,13 @@ user_id: id
 		}
 	}
 
+	parsedIssueTemplate := template.Must(template.New("issue").Parse(DefaultIssueBodyTemplate))
 	for k, v := range sample {
 		p := payload{Data: v.data}
-		res := buildGithubIssueRequest(p, "")
+		res, err := buildGithubIssueRequest(p, "", parsedIssueTemplate)
+		if err != nil {
+			t.Fatalf("Error building issue request: %s", err)
+		}
 		got := *res.Body
 		if k == 0 {
 			expect = got