json-writer: replace invalid sequences in messages with �

Fixes: #42

Author: lzaoral

src/json-writer.cc

@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2011 Red Hat, Inc.
+ * Copyright (C) 2011-2022 Red Hat, Inc.
  *
  * This file is part of csdiff.
  *
@@ -28,8 +28,18 @@
 
 #include <boost/iostreams/filtering_stream.hpp>
 #include <boost/iostreams/filter/regex.hpp>
+#include <boost/nowide/utf/convert.hpp>
 #include <boost/property_tree/json_parser.hpp>
 
+static inline std::string sanitizeUTF8(const std::string &str)
+{
+    using boost::nowide::utf::convert_string;
+
+    // every non-UTF8 sequence will be replaced with 0xEF 0xBF 0xBD which
+    // corresponds to REPLACEMENT CHARACTER U+FFFD
+    return convert_string<char>(str.data(), str.data() + str.size());
+}
+
 typedef SharedStringPTree PTree;
 
 class SimpleTreeEncoder: public AbstractTreeEncoder {
@@ -77,7 +87,7 @@ void SimpleTreeEncoder::appendDef(const Defect &def)
 
         // describe the event
         evtNode.put<string>("event", evt.event);
-        evtNode.put<string>("message", evt.msg);
+        evtNode.put<string>("message", sanitizeUTF8(evt.msg));
         evtNode.put<int>("verbosity_level", evt.verbosityLevel);
 
         // append the event to the list
@@ -195,10 +205,10 @@ void SarifTreeEncoder::importScanProps(const TScanProps &scanProps)
     scanProps_ = scanProps;
 }
 
-static void sarifEncodeMsg(PTree *pDst, const std::string text)
+static void sarifEncodeMsg(PTree *pDst, const std::string& text)
 {
     PTree msg;
-    msg.put<std::string>("text", text);
+    msg.put<std::string>("text", sanitizeUTF8(text));
     pDst->put_child("message", msg);
 }
 

tests/CMakeLists.txt

@@ -1,4 +1,4 @@
-# Copyright (C) 2011 - 2020 Red Hat, Inc.
+# Copyright (C) 2011 - 2022 Red Hat, Inc.
 #
 # This file is part of csdiff.
 #

tests/csgrep/89-json-writer-illegal-utf8-sequence-args.txt

@@ -0,0 +1 @@
+--mode=json

tests/csgrep/89-json-writer-illegal-utf8-sequence-stdin.txt

@@ -0,0 +1 @@
+test.c: warning: illegal sequence '�' will be replaced with '�'

tests/csgrep/89-json-writer-illegal-utf8-sequence-stdout.txt

@@ -0,0 +1,18 @@
+{
+    "defects": [
+        {
+            "checker": "COMPILER_WARNING",
+            "language": "c/c++",
+            "key_event_idx": 0,
+            "events": [
+                {
+                    "file_name": "test.c",
+                    "line": 0,
+                    "event": "warning",
+                    "message": "illegal sequence '�' will be replaced with '�'",
+                    "verbosity_level": 0
+                }
+            ]
+        }
+    ]
+}

tests/csgrep/90-sarif-writer-illegal-utf8-sequence-args.txt

@@ -0,0 +1 @@
+--mode=sarif

tests/csgrep/90-sarif-writer-illegal-utf8-sequence-stdin.txt

@@ -0,0 +1 @@
+test.c: warning: illegal sequence '�' will be replaced with '�'

tests/csgrep/90-sarif-writer-illegal-utf8-sequence-stdout.txt

@@ -0,0 +1,60 @@
+{
+    "$schema": "https://json.schemastore.org/sarif-2.1.0.json",
+    "version": "",
+    "runs": [
+        {
+            "tool": {
+                "driver": {
+                    "name": "csdiff",
+                    "version": "",
+                    "informationUri": "https://github.com/csutils/csdiff"
+                }
+            },
+            "results": [
+                {
+                    "ruleId": "COMPILER_WARNING: warning",
+                    "locations": [
+                        {
+                            "id": 0,
+                            "physicalLocation": {
+                                "artifactLocation": {
+                                    "uri": "test.c"
+                                }
+                            }
+                        }
+                    ],
+                    "message": {
+                        "text": "illegal sequence '�' will be replaced with '�'"
+                    },
+                    "codeFlows": [
+                        {
+                            "threadFlows": [
+                                {
+                                    "locations": [
+                                        {
+                                            "location": {
+                                                "id": 0,
+                                                "physicalLocation": {
+                                                    "artifactLocation": {
+                                                        "uri": "test.c"
+                                                    }
+                                                },
+                                                "message": {
+                                                    "text": "illegal sequence '�' will be replaced with '�'"
+                                                }
+                                            },
+                                            "nestingLevel": 0,
+                                            "kinds": [
+                                                "warning"
+                                            ]
+                                        }
+                                    ]
+                                }
+                            ]
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/csgrep/CMakeLists.txt

@@ -132,3 +132,5 @@ test_csgrep("85-sarif-writer"                         )
 test_csgrep("86-sarif-parser"                         )
 test_csgrep("87-json-writer-empty"                    )
 test_csgrep("88-csparser-cert-c"                      )
+test_csgrep("89-json-writer-illegal-utf8-sequence"    )
+test_csgrep("90-sarif-writer-illegal-utf8-sequence"   )