pass a char buffer to simplecpp instead of a stream

democlient/democlient.cpp

@@ -59,8 +59,8 @@ class CppcheckExecutor : public ErrorLogger {
         cppcheck.settings().certainty.enable(Certainty::inconclusive);
     }
 
-    void run(const char code[]) {
-        cppcheck.check(FileWithDetails("test.cpp"), code);
+    void run(const char* code) {
+        cppcheck.checkBuffer(FileWithDetails("test.cpp"), reinterpret_cast<const uint8_t*>(code), strlen(code));
     }
 
     void reportOut(const std::string & /*outmsg*/, Color /*c*/) override {}

gui/mainwindow.cpp

@@ -700,7 +700,10 @@ void MainWindow::analyzeCode(const QString& code, const QString& filename)
     checkLockDownUI();
     clearResults();
     mUI->mResults->checkingStarted(1);
-    cppcheck.check(FileWithDetails(filename.toStdString()), code.toStdString());
+    {
+        const std::string code_s = code.toStdString();
+        cppcheck.checkBuffer(FileWithDetails(filename.toStdString()), reinterpret_cast<const std::uint8_t*>(code_s.data()), code_s.size());
+    }
     analysisDone();
 
     // Expand results

lib/cppcheck.cpp

@@ -563,10 +563,9 @@ unsigned int CppCheck::check(const FileWithDetails &file)
     return checkFile(file, emptyString);
 }
 
-unsigned int CppCheck::check(const FileWithDetails &file, const std::string &content)
+unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const uint8_t* data, std::size_t size)
 {
-    std::istringstream iss(content);
-    return checkFile(file, emptyString, &iss);
+    return checkBuffer(file, emptyString, data, size);
 }
 
 unsigned int CppCheck::check(const FileSettings &fs)
@@ -612,15 +611,29 @@ unsigned int CppCheck::check(const FileSettings &fs)
     return returnValue;
 }
 
-static simplecpp::TokenList createTokenList(const std::string& filename, std::vector<std::string>& files, simplecpp::OutputList* outputList, std::istream* fileStream)
+unsigned int CppCheck::checkBuffer(const FileWithDetails &file, const std::string &cfgname, const uint8_t* data, std::size_t size)
 {
-    if (fileStream)
-        return {*fileStream, files, filename, outputList};
+    return checkInternal(file, cfgname,
+                         [&file, data, size](TokenList& list) {
+        list.createTokensFromBuffer(data, size, file.spath());
+    },
+                         [&file, data, size](std::vector<std::string>& files, simplecpp::OutputList* outputList) {
+        return simplecpp::TokenList{data, size, files, file.spath(), outputList};
+    });
+}
 
-    return {filename, files, outputList};
+unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string &cfgname)
+{
+    return checkInternal(file, cfgname,
+                         [&file](TokenList& list) {
+        list.createTokensFromFile(file.spath());
+    },
+                         [&file](std::vector<std::string>& files, simplecpp::OutputList* outputList) {
+        return simplecpp::TokenList{file.spath(), files, outputList};
+    });
 }
 
-unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string &cfgname, std::istream* fileStream)
+unsigned int CppCheck::checkInternal(const FileWithDetails& file, const std::string &cfgname, const CreateTokensFn& createTokens, const CreateTokenListFn& createTokenList)
 {
     // TODO: move to constructor when CppCheck no longer owns the settings
     if (mSettings.checks.isEnabled(Checks::unusedFunction) && !mUnusedFunctionsCheck)
@@ -666,13 +679,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
                 Tokenizer tokenizer(mSettings, *this);
                 // enforce the language since markup files are special and do not adhere to the enforced language
                 tokenizer.list.setLang(Standards::Language::C, true);
-                if (fileStream) {
-                    tokenizer.list.createTokens(*fileStream, file.spath());
-                }
-                else {
-                    std::ifstream in(file.spath());
-                    tokenizer.list.createTokens(in, file.spath());
-                }
+                createTokens(tokenizer.list);
                 mUnusedFunctionsCheck->parseTokens(tokenizer, mSettings);
                 // TODO: set analyzer information
             }
@@ -681,7 +688,7 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
 
         simplecpp::OutputList outputList;
         std::vector<std::string> files;
-        simplecpp::TokenList tokens1 = createTokenList(file.spath(), files, &outputList, fileStream);
+        simplecpp::TokenList tokens1 = createTokenList(files, &outputList);
 
         // If there is a syntax error, report it and stop
         const auto output_it = std::find_if(outputList.cbegin(), outputList.cend(), [](const simplecpp::Output &output){
@@ -805,9 +812,8 @@ unsigned int CppCheck::checkFile(const FileWithDetails& file, const std::string
                     code += "#line " + std::to_string(dir.linenr) + " \"" + dir.file + "\"\n" + dir.str + '\n';
             }
             TokenList tokenlist(&mSettings);
-            std::istringstream istr2(code);
             // TODO: asserts when file has unknown extension
-            tokenlist.createTokens(istr2, Path::identify(*files.begin(), false)); // TODO: check result?
+            tokenlist.createTokensFromBuffer(code.data(), code.size(), Path::identify(*files.begin(), false)); // TODO: check result?
             executeRules("define", tokenlist);
         }
 #endif

lib/cppcheck.h

@@ -49,7 +49,10 @@ class Tokenizer;
 class FileWithDetails;
 class RemarkComment;
 
-namespace simplecpp { class TokenList; }
+namespace simplecpp {
+    class TokenList;
+    struct Output;
+}
 
 /// @addtogroup Core
 /// @{
@@ -99,12 +102,17 @@ class CPPCHECKLIB CppCheck : ErrorLogger {
      * the disk but the content is given in @p content. In errors the @p path
      * is used as a filename.
      * @param file The file to check.
-     * @param content File content as a string.
+     * @param data File content as a buffer.
+     * @param size Size of buffer.
      * @return amount of errors found or 0 if none were found.
      * @note You must set settings before calling this function (by calling
      *  settings()).
      */
-    unsigned int check(const FileWithDetails &file, const std::string &content);
+    unsigned int checkBuffer(const FileWithDetails &file, const uint8_t* data, std::size_t size);
+
+    unsigned int checkString(const FileWithDetails &file, const std::string& data) {
+        return checkBuffer(file, reinterpret_cast<const uint8_t*>(data.data()), data.size());
+    }
 
     /**
      * @brief Get reference to current settings.
@@ -172,14 +180,37 @@ class CPPCHECKLIB CppCheck : ErrorLogger {
     /** @brief There has been an internal error => Report information message */
     void internalError(const std::string &filename, const std::string &msg);
 
+    /**
+     * @brief Check a file
+     * @param file the file
+     * @param cfgname  cfg name
+     * @return number of errors found
+     */
+    unsigned int checkFile(const FileWithDetails& file, const std::string &cfgname);
+
+    /**
+     * @brief Check a file using buffer
+     * @param file the file
+     * @param cfgname  cfg name
+     * @param data the data to be read
+     * @param size the size of the data to be read
+     * @return number of errors found
+     */
+    unsigned int checkBuffer(const FileWithDetails& file, const std::string &cfgname, const uint8_t* data, std::size_t size);
+
+    using CreateTokensFn = std::function<void (TokenList&)>;
+    // TODO: should use simplecpp::OutputList
+    using CreateTokenListFn = std::function<simplecpp::TokenList(std::vector<std::string>&, std::list<simplecpp::Output>*)>;
+
     /**
      * @brief Check a file using stream
      * @param file the file
      * @param cfgname  cfg name
-     * @param fileStream stream the file content can be read from
+     * @param createTokens a function to create the tokens with
+     * @param createTokenList a function to create the TokenList with
      * @return number of errors found
      */
-    unsigned int checkFile(const FileWithDetails& file, const std::string &cfgname, std::istream* fileStream = nullptr);
+    unsigned int checkInternal(const FileWithDetails& file, const std::string &cfgname, const CreateTokensFn& createTokens, const CreateTokenListFn& createTokenList);
 
     /**
      * @brief Check normal tokens

lib/importproject.cpp

@@ -574,8 +574,7 @@ namespace {
             // TODO: improve evaluation
             const Settings s;
             TokenList tokenlist(&s);
-            std::istringstream istr(c);
-            tokenlist.createTokens(istr, Standards::Language::C); // TODO: check result
+            tokenlist.createTokensFromBuffer(c.data(), c.size(), Standards::Language::C); // TODO: check result
             // TODO: put in a helper
             // generate links
             {

lib/library.cpp

@@ -35,7 +35,6 @@
 #include <iostream>
 #include <list>
 #include <memory>
-#include <sstream>
 #include <stdexcept>
 #include <string>
 #include <unordered_set>
@@ -172,8 +171,8 @@ static std::vector<std::string> getnames(const char *names)
 
 static void gettokenlistfromvalid(const std::string& valid, bool cpp, TokenList& tokenList)
 {
-    std::istringstream istr(valid + ',');
-    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+    const std::string str(valid + ',');
+    tokenList.createTokensFromBuffer(str.data(), str.size(), cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
     for (Token *tok = tokenList.front(); tok; tok = tok->next()) {
         if (Token::Match(tok,"- %num%")) {
             tok->str("-" + tok->strAt(1));

lib/programmemory.cpp

@@ -1791,8 +1791,7 @@ static std::shared_ptr<Token> createTokenFromExpression(const std::string& retur
     std::shared_ptr<TokenList> tokenList = std::make_shared<TokenList>(&settings);
     {
         const std::string code = "return " + returnValue + ";";
-        std::istringstream istr(code);
-        if (!tokenList->createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
+        if (!tokenList->createTokensFromBuffer(code.data(), code.size(), cpp ? Standards::Language::CPP : Standards::Language::C))
             return nullptr;
     }
 

lib/symboldatabase.cpp

@@ -7632,8 +7632,8 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                 if (!typestr.empty()) {
                     ValueType valuetype;
                     TokenList tokenList(&mSettings);
-                    std::istringstream istr(typestr+";");
-                    tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+                    const std::string str(typestr+";");
+                    tokenList.createTokensFromBuffer(str.data(), str.size(), tok->isCpp() ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
                     tokenList.simplifyStdType();
                     if (parsedecl(tokenList.front(), &valuetype, mDefaultSignedness, mSettings)) {
                         valuetype.originalTypeName = typestr;
@@ -7722,8 +7722,8 @@ void SymbolDatabase::setValueTypeInTokenList(bool reportDebugWarnings, Token *to
                     continue;
                 }
                 TokenList tokenList(&mSettings);
-                std::istringstream istr(typestr+";");
-                if (tokenList.createTokens(istr, tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
+                const std::string str(typestr+";");
+                if (tokenList.createTokensFromBuffer(str.data(), str.size(), tok->isCpp() ? Standards::Language::CPP : Standards::Language::C)) {
                     ValueType vt;
                     tokenList.simplifyPlatformTypes();
                     tokenList.simplifyStdType();

lib/tokenlist.cpp

@@ -335,18 +335,29 @@ void TokenList::insertTokens(Token *dest, const Token *src, nonneg int n)
 // Tokenize - tokenizes a given file.
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokens(std::istream &code, const std::string& file0)
+bool TokenList::createTokensFromFile(const std::string& file0)
 {
     ASSERT_LANG(!file0.empty());
 
     appendFileIfNew(file0);
 
-    return createTokensInternal(code, file0);
+    return createTokensFromFileInternal(file0);
 }
 
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokens(std::istream &code, Standards::Language lang)
+bool TokenList::createTokensFromBuffer(const uint8_t* data, size_t size, const std::string& file0)
+{
+    ASSERT_LANG(!file0.empty());
+
+    appendFileIfNew(file0);
+
+    return createTokensFromBufferInternal(data, size, file0);
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensFromBuffer(const uint8_t* data, size_t size, Standards::Language lang)
 {
     ASSERT_LANG(lang != Standards::Language::None);
     if (mLang == Standards::Language::None) {
@@ -355,15 +366,27 @@ bool TokenList::createTokens(std::istream &code, Standards::Language lang)
         ASSERT_LANG(lang == mLang);
     }
 
-    return createTokensInternal(code, "");
+    return createTokensFromBufferInternal(data, size, "");
+}
+
+//---------------------------------------------------------------------------
+
+bool TokenList::createTokensFromFileInternal(const std::string& file0)
+{
+    simplecpp::OutputList outputList;
+    simplecpp::TokenList tokens(file0, mFiles, &outputList);
+
+    createTokens(std::move(tokens));
+
+    return outputList.empty();
 }
 
 //---------------------------------------------------------------------------
 
-bool TokenList::createTokensInternal(std::istream &code, const std::string& file0)
+bool TokenList::createTokensFromBufferInternal(const uint8_t* data, size_t size, const std::string& file0)
 {
     simplecpp::OutputList outputList;
-    simplecpp::TokenList tokens(code, mFiles, file0, &outputList);
+    simplecpp::TokenList tokens(data, size, mFiles, file0, &outputList);
 
     createTokens(std::move(tokens));
 

lib/tokenlist.h

@@ -102,11 +102,30 @@ class CPPCHECKLIB TokenList {
      * - multiline strings are not handled.
      * - UTF in the code are not handled.
      * - comments are not handled.
-     * @param code input stream for code
      * @param file0 source file name
      */
-    bool createTokens(std::istream &code, const std::string& file0);
-    bool createTokens(std::istream &code, Standards::Language lang);
+    bool createTokensFromFile(const std::string& file0);
+
+    bool createTokensFromString(const std::string& data, const std::string& file0) {
+        return createTokensFromBuffer(data.data(), data.size(), file0);
+    }
+
+    bool createTokensFromBuffer(const uint8_t* data, size_t size, const std::string& file0);
+    bool createTokensFromBuffer(const char* data, size_t size, const std::string& file0) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size, file0);
+    }
+    template<size_t size>
+    bool createTokensFromString(const char (&data)[size], const std::string& file0) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size-1, file0);
+    }
+    bool createTokensFromBuffer(const uint8_t* data, size_t size, Standards::Language lang);
+    bool createTokensFromBuffer(const char* data, size_t size, Standards::Language lang) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size, lang);
+    }
+    template<size_t size>
+    bool createTokensFromString(const char (&data)[size], Standards::Language lang) {
+        return createTokensFromBuffer(reinterpret_cast<const uint8_t*>(data), size-1, lang);
+    }
 
     void createTokens(simplecpp::TokenList&& tokenList);
 
@@ -204,7 +223,8 @@ class CPPCHECKLIB TokenList {
 private:
     void determineCppC();
 
-    bool createTokensInternal(std::istream &code, const std::string& file0);
+    bool createTokensFromFileInternal(const std::string& file0);
+    bool createTokensFromBufferInternal(const uint8_t* data, std::size_t size, const std::string& file0);
 
     /** Token list */
     TokensFrontBack mTokensFrontBack;

lib/valueflow.cpp

@@ -123,7 +123,6 @@
 #include <memory>
 #include <numeric>
 #include <set>
-#include <sstream>
 #include <string>
 #include <unordered_map>
 #include <unordered_set>
@@ -1005,8 +1004,7 @@ static bool isNotEqual(std::pair<const Token*, const Token*> x, std::pair<const
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const std::string& y, bool cpp)
 {
     TokenList tokenList(nullptr);
-    std::istringstream istr(y);
-    tokenList.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
+    tokenList.createTokensFromBuffer(y.data(), y.size(), cpp ? Standards::Language::CPP : Standards::Language::C); // TODO: check result?
     return isNotEqual(x, std::make_pair(tokenList.front(), tokenList.back()));
 }
 static bool isNotEqual(std::pair<const Token*, const Token*> x, const ValueType* y, bool cpp)

lib/vf_common.cpp

@@ -35,7 +35,6 @@
 #include <cstddef>
 #include <exception>
 #include <limits>
-#include <sstream>
 #include <utility>
 #include <vector>
 
@@ -99,8 +98,8 @@ namespace ValueFlow
     bool getMinMaxValues(const std::string &typestr, const Settings &settings, bool cpp, MathLib::bigint &minvalue, MathLib::bigint &maxvalue)
     {
         TokenList typeTokens(&settings);
-        std::istringstream istr(typestr+";");
-        if (!typeTokens.createTokens(istr, cpp ? Standards::Language::CPP : Standards::Language::C))
+        const std::string str(typestr+";");
+        if (!typeTokens.createTokensFromBuffer(str.data(), str.size(), cpp ? Standards::Language::CPP : Standards::Language::C))
             return false;
         typeTokens.simplifyPlatformTypes();
         typeTokens.simplifyStdType();