From f22b0bae8d7247bfa3e0062dd7a37b128ce0cbd9 Mon Sep 17 00:00:00 2001 From: Jeremy Lewi Date: Sun, 29 Sep 2024 20:04:28 -0700 Subject: [PATCH] Render OpenAI request/response as HTML (#265) This PR adds support for rendering of OpenAI requests/responses as HTML (fix #259). This functionality was already supported for Anthropic. I added an LLM sub-command to support rendering the data as HTML. This makes it reusable and not closely tied to the GetLLMLogs RPC. Fix HTML rendering of our prompts to properly handle the XML tags in our prompt. We need to escape them so they aren't treated as raw HTML tags which don't render. Remove the obsolete logs subcommand. Fix #259 --- app/cmd/llms.go | 127 +++++++++++ app/cmd/logs.go | 61 ------ app/cmd/root.go | 2 +- app/pkg/analyze/logs.go | 18 +- app/pkg/analyze/render.go | 201 +++++++++++++++--- app/pkg/analyze/render_test.go | 70 +++++- app/pkg/analyze/request.html.tmpl | 6 +- app/pkg/analyze/test_data/openai_request.json | 15 ++ 8 files changed, 382 insertions(+), 118 deletions(-) create mode 100644 app/cmd/llms.go delete mode 100644 app/cmd/logs.go create mode 100644 app/pkg/analyze/test_data/openai_request.json diff --git a/app/cmd/llms.go b/app/cmd/llms.go new file mode 100644 index 00000000..2cd1c317 --- /dev/null +++ b/app/cmd/llms.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/jlewi/monogo/helpers" + + "github.com/jlewi/foyle/app/api" + "github.com/jlewi/foyle/app/pkg/analyze" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +// NewLLMsCmd returns a command to work with llms +func NewLLMsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "llms", + } + + cmd.AddCommand(NewLLMsRenderCmd()) + + return cmd +} + +func NewLLMsRenderCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "render", + } + + cmd.AddCommand(NewLLMsRenderRequestCmd()) + cmd.AddCommand(NewLLMsRenderResponseCmd()) + + return cmd +} + +func NewLLMsRenderRequestCmd() *cobra.Command { + var provider string + var inputFile string + var outputFile string + cmd := &cobra.Command{ + Use: "request", + Run: func(cmd *cobra.Command, args []string) { + err := func() error { + p := api.ModelProvider(provider) + + data, err := os.ReadFile(inputFile) + if err != nil { + return errors.Wrapf(err, "Failed to read file %s", inputFile) + } + + htmlData, err := analyze.RenderRequestHTML(string(data), p) + if err != nil { + return err + } + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(htmlData), 0644); err != nil { + return errors.Wrapf(err, "Failed to write file %s", outputFile) + } + } else { + fmt.Fprint(os.Stdout, htmlData) + } + + return nil + }() + + if err != nil { + fmt.Printf("Error rendering request;\n %+v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().StringVarP(&provider, "provider", "p", string(api.ModelProviderOpenAI), "The model provider for the request.") + cmd.Flags().StringVarP(&inputFile, "input", "i", "", "The file containing the JSON representation of the request.") + cmd.Flags().StringVarP(&inputFile, "output", "o", "", "The file to write the output to. If blank output to stdout.") + helpers.IgnoreError(cmd.MarkFlagRequired("input")) + + return cmd +} + +func NewLLMsRenderResponseCmd() *cobra.Command { + var provider string + var inputFile string + var outputFile string + cmd := &cobra.Command{ + Use: "response", + Run: func(cmd *cobra.Command, args []string) { + err := func() error { + p := api.ModelProvider(provider) + + data, err := os.ReadFile(inputFile) + if err != nil { + return errors.Wrapf(err, "Failed to read file %s", inputFile) + } + + htmlData, err := analyze.RenderResponseHTML(string(data), p) + if err != nil { + return err + } + + if outputFile != "" { + if err := os.WriteFile(outputFile, []byte(htmlData), 0644); err != nil { + return errors.Wrapf(err, "Failed to write file %s", outputFile) + } + } else { + fmt.Fprint(os.Stdout, htmlData) + } + + return nil + }() + + if err != nil { + fmt.Printf("Error rendering response;\n %+v\n", err) + os.Exit(1) + } + }, + } + + cmd.Flags().StringVarP(&provider, "provider", "p", string(api.ModelProviderOpenAI), "The model provider for the request.") + cmd.Flags().StringVarP(&inputFile, "input", "i", "", "The file containing the JSON representation of the request.") + cmd.Flags().StringVarP(&inputFile, "output", "o", "", "The file to write the output to. If blank output to stdout.") + helpers.IgnoreError(cmd.MarkFlagRequired("input")) + + return cmd +} diff --git a/app/cmd/logs.go b/app/cmd/logs.go deleted file mode 100644 index 4db0a351..00000000 --- a/app/cmd/logs.go +++ /dev/null @@ -1,61 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/go-logr/zapr" - "github.com/jlewi/foyle/app/pkg/application" - "github.com/jlewi/monogo/helpers" - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -// NewLogsCmd returns a command to manage logs -func NewLogsCmd() *cobra.Command { - cmd := &cobra.Command{ - Use: "logs", - } - - cmd.AddCommand(NewLogsProcessCmd()) - return cmd -} - -// NewLogsProcessCmd returns a command to process the assets -func NewLogsProcessCmd() *cobra.Command { - logDirs := []string{} - var outDir string - cmd := &cobra.Command{ - Use: "process", - Run: func(cmd *cobra.Command, args []string) { - err := func() error { - app := application.NewApp() - if err := app.LoadConfig(cmd); err != nil { - return err - } - if err := app.SetupLogging(false); err != nil { - return err - } - if err := app.OpenDBs(); err != nil { - return err - } - defer helpers.DeferIgnoreError(app.Shutdown) - - logVersion() - - log := zapr.NewLogger(zap.L()) - log.Info("Calling logs process is no longer necessary; logs are processed in real time.", "logDirs", logDirs, "outDir", outDir) - return nil - }() - - if err != nil { - fmt.Printf("Error processing logs;\n %+v\n", err) - os.Exit(1) - } - }, - } - - cmd.Flags().StringArrayVarP(&logDirs, "logs", "", []string{}, "(Optional) Directories containing logs to process") - cmd.Flags().StringVarP(&outDir, "out", "", "", "(Optional) Directory to write the output to") - return cmd -} diff --git a/app/cmd/root.go b/app/cmd/root.go index 754fad27..13596231 100644 --- a/app/cmd/root.go +++ b/app/cmd/root.go @@ -29,7 +29,7 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(NewVersionCmd(appName, os.Stdout)) rootCmd.AddCommand(NewServeCmd()) rootCmd.AddCommand(NewConfigCmd()) - rootCmd.AddCommand(NewLogsCmd()) + rootCmd.AddCommand(NewLLMsCmd()) rootCmd.AddCommand(NewApplyCmd()) rootCmd.AddCommand(NewProtoToJsonCmd()) return rootCmd diff --git a/app/pkg/analyze/logs.go b/app/pkg/analyze/logs.go index c16133e5..32774d5e 100644 --- a/app/pkg/analyze/logs.go +++ b/app/pkg/analyze/logs.go @@ -87,22 +87,8 @@ func readLLMLog(ctx context.Context, traceId string, logFile string) (*logspb.Ge } } - if provider == api.ModelProviderAnthropic && resp.ResponseJson != "" { - html, err := renderAnthropicRequestJson(resp.RequestJson) - if err != nil { - log.Error(err, "Failed to render request") - - } else { - resp.RequestHtml = html - } - - htmlResp, err := renderAnthropicResponseJson(resp.ResponseJson) - if err != nil { - log.Error(err, "Failed to render response") - - } else { - resp.ResponseHtml = htmlResp - } + if err := renderHTML(resp, provider); err != nil { + log.Error(err, "Failed to render HTML") } return resp, nil } diff --git a/app/pkg/analyze/render.go b/app/pkg/analyze/render.go index 1585c2df..bebeec70 100644 --- a/app/pkg/analyze/render.go +++ b/app/pkg/analyze/render.go @@ -6,6 +6,12 @@ import ( "encoding/json" "fmt" "html/template" + "strings" + + "github.com/jlewi/foyle/app/api" + logspb "github.com/jlewi/foyle/protos/go/foyle/logs" + "github.com/pkg/errors" + "github.com/sashabaranov/go-openai" "github.com/go-logr/zapr" "github.com/liushuangls/go-anthropic/v2" @@ -35,26 +41,90 @@ type Message struct { Content template.HTML } -func renderAnthropicRequestJson(jsonValue string) (string, error) { - req := &anthropic.MessagesRequest{} - if err := json.Unmarshal([]byte(jsonValue), req); err != nil { - return "", nil +// renderHTML populates the HTML fields in GetLLMLogsResponse with the rendered HTML. +// The HTML is generated from the JSON of the request and response +func renderHTML(resp *logspb.GetLLMLogsResponse, provider api.ModelProvider) error { + if resp == nil { + return errors.WithStack(errors.New("response is nil")) + } + + reqHtml, reqErr := RenderRequestHTML(resp.GetRequestJson(), provider) + if reqErr != nil { + return reqErr } + resp.RequestHtml = reqHtml - return renderAnthropicRequest(req), nil + respHtml, respErr := RenderResponseHTML(resp.GetResponseJson(), provider) + if respErr != nil { + return respErr + } + resp.ResponseHtml = respHtml + return nil } -func renderAnthropicResponseJson(jsonValue string) (string, error) { - res := &anthropic.MessagesResponse{} - if err := json.Unmarshal([]byte(jsonValue), res); err != nil { - return "", nil +func RenderRequestHTML(jsonValue string, provider api.ModelProvider) (string, error) { + if jsonValue == "" { + return "", errors.WithStack(errors.New("request is empty")) + } + + var data *TemplateData + switch provider { + case api.ModelProviderOpenAI: + req := &openai.ChatCompletionRequest{} + if err := json.Unmarshal([]byte(jsonValue), req); err != nil { + return "", errors.Wrapf(err, "failed to unmarshal request to openai.ChatCompletionRequest; json: %s", jsonValue) + } + data = oaiRequestToTemplateData(req) + case api.ModelProviderAnthropic: + req := &anthropic.MessagesRequest{} + if err := json.Unmarshal([]byte(jsonValue), req); err != nil { + return "", errors.Wrapf(err, "failed to unmarshal request to anthropic.MessagesRequest; json: %s", jsonValue) + } + data = anthropicRequestToTemplateData(req) + default: + return fmt.Sprintf("

Unsupported provider: %v

", provider), nil + } + + var buf bytes.Buffer + if err := requestTemplate.Execute(&buf, data); err != nil { + return "", errors.Wrapf(err, "Failed to execute request template") } - return renderAnthropicResponse(res), nil + return buf.String(), nil } -// renderAnthropicRequest returns a string containing the HTML representation of the request -func renderAnthropicRequest(request *anthropic.MessagesRequest) string { +func RenderResponseHTML(jsonValue string, provider api.ModelProvider) (string, error) { + if jsonValue == "" { + return "", errors.WithStack(errors.New("response is nil")) + } + + var data *ResponseTemplateData + switch provider { + case api.ModelProviderOpenAI: + req := &openai.ChatCompletionResponse{} + if err := json.Unmarshal([]byte(jsonValue), req); err != nil { + return "", errors.Wrapf(err, "failed to unmarshal request to openai.ChatCompletionResponse; json: %s", jsonValue) + } + data = oaiResponseToTemplateData(req) + case api.ModelProviderAnthropic: + req := &anthropic.MessagesResponse{} + if err := json.Unmarshal([]byte(jsonValue), req); err != nil { + return "", errors.Wrapf(err, "failed to unmarshal request to anthropic.MessagesRequest; json: %s", jsonValue) + } + data = anthropicResponseToTemplateData(req) + default: + return fmt.Sprintf("

Unsupported provider: %v

", provider), nil + } + + var buf bytes.Buffer + if err := responseTemplate.Execute(&buf, data); err != nil { + return "", errors.Wrapf(err, "Failed to execute request template") + } + + return buf.String(), nil +} + +func anthropicRequestToTemplateData(request *anthropic.MessagesRequest) *TemplateData { log := zapr.NewLogger(zap.L()) data := &TemplateData{ Model: request.Model, @@ -64,14 +134,16 @@ func renderAnthropicRequest(request *anthropic.MessagesRequest) string { Messages: make([]Message, 0, len(request.Messages)), } + md := converter() + for _, message := range request.Messages { content := "" for _, c := range message.Content { content += *c.Text } - + content = escapePromptTags(content) var buf bytes.Buffer - if err := goldmark.Convert([]byte(content), &buf); err != nil { + if err := md.Convert([]byte(content), &buf); err != nil { log.Error(err, "Failed to convert markdown to HTML") buf.WriteString(fmt.Sprintf("Failed to convert markdown to HTML: error %+v", err)) } @@ -80,12 +152,7 @@ func renderAnthropicRequest(request *anthropic.MessagesRequest) string { Content: template.HTML(buf.String()), }) } - var buf bytes.Buffer - if err := requestTemplate.Execute(&buf, data); err != nil { - log.Error(err, "Failed to execute request template") - return fmt.Sprintf("Failed to execute request template: error %+v", err) - } - return buf.String() + return data } type ResponseTemplateData struct { @@ -100,8 +167,7 @@ type ResponseTemplateData struct { OutputTokens int } -// renderAnthropicResponse returns a string containing the HTML representation of the response -func renderAnthropicResponse(resp *anthropic.MessagesResponse) string { +func anthropicResponseToTemplateData(resp *anthropic.MessagesResponse) *ResponseTemplateData { log := zapr.NewLogger(zap.L()) data := &ResponseTemplateData{ ID: resp.ID, @@ -116,8 +182,10 @@ func renderAnthropicResponse(resp *anthropic.MessagesResponse) string { } for _, message := range resp.Content { + content := message.GetText() + content = escapePromptTags(content) var buf bytes.Buffer - if err := goldmark.Convert([]byte(message.GetText()), &buf); err != nil { + if err := goldmark.Convert([]byte(content), &buf); err != nil { log.Error(err, "Failed to convert markdown to HTML") buf.WriteString(fmt.Sprintf("Failed to convert markdown to HTML: error %+v", err)) } @@ -125,12 +193,89 @@ func renderAnthropicResponse(resp *anthropic.MessagesResponse) string { Content: template.HTML(buf.String()), }) } - var buf bytes.Buffer - if err := responseTemplate.Execute(&buf, data); err != nil { - log.Error(err, "Failed to execute response template") - return fmt.Sprintf("Failed to execute response template: error %+v", err) + return data +} + +func oaiRequestToTemplateData(request *openai.ChatCompletionRequest) *TemplateData { + log := zapr.NewLogger(zap.L()) + data := &TemplateData{ + Model: request.Model, + Tokens: request.MaxTokens, + Temperature: float64(request.Temperature), + // System message for OpenAI is just a message with a role of system prompt + System: "", + Messages: make([]Message, 0, len(request.Messages)), + } + + md := converter() + for _, message := range request.Messages { + content := message.Content + content = escapePromptTags(content) + var buf bytes.Buffer + if err := md.Convert([]byte(content), &buf); err != nil { + log.Error(err, "Failed to convert markdown to HTML") + buf.WriteString(fmt.Sprintf("Failed to convert markdown to HTML: error %+v", err)) + } + data.Messages = append(data.Messages, Message{ + Role: message.Role, + Content: template.HTML(buf.String()), + }) + } + + return data +} + +func oaiResponseToTemplateData(resp *openai.ChatCompletionResponse) *ResponseTemplateData { + log := zapr.NewLogger(zap.L()) + + data := &ResponseTemplateData{ + ID: resp.ID, + Type: "", + Role: "", + Model: resp.Model, + StopSequence: "", + InputTokens: resp.Usage.PromptTokens, + OutputTokens: resp.Usage.CompletionTokens, + Messages: make([]Message, 0, len(resp.Choices)), + } + + if len(resp.Choices) > 0 { + data.StopReason = string(resp.Choices[0].FinishReason) + data.Role = resp.Choices[0].Message.Role + } + + md := converter() + for _, c := range resp.Choices { + var buf bytes.Buffer + content := escapePromptTags(c.Message.Content) + if err := md.Convert([]byte(content), &buf); err != nil { + log.Error(err, "Failed to convert markdown to HTML") + buf.WriteString(fmt.Sprintf("Failed to convert markdown to HTML: error %+v", err)) + } + data.Messages = append(data.Messages, Message{ + Content: template.HTML(buf.String()), + }) + } + return data +} + +func converter() goldmark.Markdown { + md := goldmark.New() + return md +} + +// escapePromptTags escapes the xml tags we use in our prompt so they can be displayed in the HTML +func escapePromptTags(data string) string { + tags := []string{"", "", "", "", "", ""} + + for _, tag := range tags { + escapedTag := tag + escapedTag = strings.ReplaceAll(escapedTag, "<", "<") + escapedTag = strings.ReplaceAll(escapedTag, ">", ">") + + data = strings.ReplaceAll(data, tag, escapedTag) } - return buf.String() + return data } func init() { diff --git a/app/pkg/analyze/render_test.go b/app/pkg/analyze/render_test.go index 35fbb1bd..23318543 100644 --- a/app/pkg/analyze/render_test.go +++ b/app/pkg/analyze/render_test.go @@ -7,6 +7,8 @@ import ( "path/filepath" "testing" + "github.com/jlewi/foyle/app/api" + "github.com/liushuangls/go-anthropic/v2" "github.com/pkg/browser" "google.golang.org/protobuf/proto" @@ -14,14 +16,21 @@ import ( func TestRenderAnthropicRequest(t *testing.T) { type testCase struct { - name string - fname string + name string + fname string + provider api.ModelProvider } tests := []testCase{ { - name: "basic", - fname: "anthropic_request.json", + name: "openai", + fname: "openai_request.json", + provider: api.ModelProviderOpenAI, + }, + { + name: "anthropic", + fname: "anthropic_request.json", + provider: api.ModelProviderAnthropic, }, } @@ -40,14 +49,12 @@ func TestRenderAnthropicRequest(t *testing.T) { t.Fatalf("Failed to read file %s: %v", fname, err) } - req := &anthropic.MessagesRequest{} - if err := json.Unmarshal(data, req); err != nil { - t.Fatalf("Failed to unmarshal request: %v", err) + result, err := RenderRequestHTML(string(data), test.provider) + if err != nil { + t.Fatalf("Failed to render request: %+v", err) } - - result := renderAnthropicRequest(req) if result == "" { - t.Errorf("Request should not be empty") + t.Fatalf("Request should not be empty") } if os.Getenv("OPEN_IN_BROWSER") != "" { @@ -96,7 +103,15 @@ func TestRenderAnthropicResponse(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - result := renderAnthropicResponse(test.resp) + jsonData, err := json.Marshal(test.resp) + if err != nil { + t.Fatalf("Failed to unmarshal request: %v", err) + } + + result, err := RenderResponseHTML(string(jsonData), api.ModelProviderAnthropic) + if err != nil { + t.Errorf("Failed to render response: %v", err) + } if result == "" { t.Errorf("Result should not be empty") } @@ -116,3 +131,36 @@ func TestRenderAnthropicResponse(t *testing.T) { }) } } + +func Test_escapePrompt(t *testing.T) { + type testCase struct { + name string + prompt string + expected string + } + + cases := []testCase{ + { + name: "basic", + prompt: `This is the input + +some example + +`, + expected: `This is the input +<example> +some example +</example> +`, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + result := escapePromptTags(c.prompt) + if result != c.expected { + t.Errorf("Expected %s; got %s", c.expected, result) + } + }) + } +} diff --git a/app/pkg/analyze/request.html.tmpl b/app/pkg/analyze/request.html.tmpl index 632cb7f5..1f3d4f60 100644 --- a/app/pkg/analyze/request.html.tmpl +++ b/app/pkg/analyze/request.html.tmpl @@ -24,6 +24,10 @@ .role { font-weight: bold; } + h2.example-header { + color: #2c3e50; + font-size: 1.5em; + } @@ -48,7 +52,7 @@ {{range .Messages}}
-

Role{{.Role}}

+

Role: {{.Role}}

{{.Content }}
{{end}} diff --git a/app/pkg/analyze/test_data/openai_request.json b/app/pkg/analyze/test_data/openai_request.json new file mode 100644 index 00000000..3983a059 --- /dev/null +++ b/app/pkg/analyze/test_data/openai_request.json @@ -0,0 +1,15 @@ +{ + "max_tokens": 2000, + "messages": [ + { + "content": "You are a helpful AI assistant for software developers. You are helping software engineers write \nmarkdown documents to deploy and operate software. Your job is to help users with tasks related to building, deploying,\nand operating software. You should interpret any questions or commands in that context. You job is to suggest\ncommands the user can execute to accomplish their goals.", + "role": "system" + }, + { + "content": "Continue writing the markdown document by adding a code block with the commands a user should execute.\nFollow these rules\n\n* Set the language inside the code block to bash\n* Use the text at the end of the document to determine what commands to execute next\n* Use the existing text and code blocks in the document to learn phrases that are predictive of specific commands\n* Only respond with a single code block\n* You can put multiple commands into a code block\n* If the text at the end of the document doesn't clearly describe a command to execute simply respond with the tag\n\n\nHere are a bunch of examples of input documents along with the expected output.\n\n\n\nGrab a snapshot of the HC query for model errors\n\n\n\n```bash\n/Users/jlewi/git_hccli/hccli querytourl --query-file=/tmp/query.json --dataset=foyle \n```\n\n\n\n\n\nGrab a snapshot of the HC query for model errors\n\n\n\n```bash\n/Users/jlewi/git_hccli/hccli querytourl --query-file=/tmp/query.json --dataset=foyle \n```\n\n\n\n\n\nGrab a snapshot of the HC query for model errors\n\n\n\n```bash\n/Users/jlewi/git_hccli/hccli querytourl --help \n```\n\n\n\nHere's the actual document containing the problem or task to be solved:\n\n\nH\n\n\n\n", + "role": "user" + } + ], + "model": "gpt-4o", + "temperature": 0.9 +}