Skip to content

Commit

Permalink
Fix #12853: Add extra information to directives (#6527)
Browse files Browse the repository at this point in the history
  • Loading branch information
@olabetskyi
olabetskyi authored Jun 19, 2024
1 parent 3af999b commit e7811ed
Show file tree
Hide file tree
Showing 5 changed files with 185 additions and 34 deletions.
13 changes: 10 additions & 3 deletions addons/cppcheckdata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1281,6 +1281,9 @@ def iterconfigurations(self):
# Iterating <typedef-info>
iter_typedef_info = False

# Iterating <directive>
iter_directive = False

# Use iterable objects to traverse XML tree for dump files incrementally.
# Iterative approach is required to avoid large memory consumption.
# Calling .clear() is necessary to let the element be garbage collected.
Expand Down Expand Up @@ -1312,8 +1315,12 @@ def iterconfigurations(self):
cfg.standards.set_posix(node)

# Parse directives list
elif node.tag == 'directive' and event == 'start':
cfg.directives.append(Directive(node))
elif node.tag == 'directive':
if event == 'start':
cfg.directives.append(Directive(node))
iter_directive = True
elif event == 'end':
iter_directive = False
# Parse macro usage
elif node.tag == 'macro' and event == 'start':
cfg.macro_usage.append(MacroUsage(node))
Expand All @@ -1325,7 +1332,7 @@ def iterconfigurations(self):
# Parse tokens
elif node.tag == 'tokenlist' and event == 'start':
continue
elif node.tag == 'token' and event == 'start':
elif node.tag == 'token' and event == 'start' and not iter_directive:
cfg.tokenlist.append(Token(node))

# Parse scopes
Expand Down
20 changes: 17 additions & 3 deletions lib/preprocessor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,22 @@ static bool sameline(const simplecpp::Token *tok1, const simplecpp::Token *tok2)
return tok1 && tok2 && tok1->location.sameline(tok2->location);
}

Directive::Directive(std::string _file, const int _linenr, const std::string &_str) :
Directive::Directive(const simplecpp::Location & _loc, std::string _str) :
file(_loc.file()),
linenr(_loc.line),
str(std::move(_str))
{}

Directive::Directive(std::string _file, const int _linenr, std::string _str) :
file(std::move(_file)),
linenr(_linenr),
str(trim(_str))
str(std::move(_str))
{}

Directive::DirectiveToken::DirectiveToken(const simplecpp::Token & _tok) :
line(_tok.location.line),
column(_tok.location.col),
tokStr(_tok.str())
{}

char Preprocessor::macroChar = char(1);
Expand Down Expand Up @@ -328,7 +340,7 @@ std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &
continue;
if (tok->next && tok->next->str() == "endfile")
continue;
Directive directive(tok->location.file(), tok->location.line, emptyString);
Directive directive(tok->location, emptyString);
for (const simplecpp::Token *tok2 = tok; tok2 && tok2->location.line == directive.linenr; tok2 = tok2->next) {
if (tok2->comment)
continue;
Expand All @@ -338,6 +350,8 @@ std::list<Directive> Preprocessor::createDirectives(const simplecpp::TokenList &
directive.str += "include";
else
directive.str += tok2->str();

directive.strTokens.emplace_back(*tok2);
}
directives.push_back(std::move(directive));
}
Expand Down
14 changes: 12 additions & 2 deletions lib/preprocessor.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ class SuppressionList;
* Each preprocessor directive (\#include, \#define, \#undef, \#if, \#ifdef, \#else, \#endif)
* will be recorded as an instance of this class.
*
* file and linenr denote the location where where the directive is defined.
* file and linenr denote the location where the directive is defined.
*
*/

Expand All @@ -57,8 +57,18 @@ struct CPPCHECKLIB Directive {
/** the actual directive text */
std::string str;

struct DirectiveToken {
explicit DirectiveToken(const simplecpp::Token & _tok);
int line;
int column;
std::string tokStr;
};

std::vector<DirectiveToken> strTokens;

/** record a directive (possibly filtering src) */
Directive(std::string _file, const int _linenr, const std::string &_str);
Directive(const simplecpp::Location & _loc, std::string _str);
Directive(std::string _file, const int _linenr, std::string _str);
};

class CPPCHECKLIB RemarkComment {
Expand Down
14 changes: 13 additions & 1 deletion lib/tokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5959,7 +5959,19 @@ void Tokenizer::dump(std::ostream &out) const
// could result in invalid XML, so run it through toxml().
outs += "str=\"";
outs += ErrorLogger::toxml(dir.str);
outs +="\"/>";
outs +="\">";
outs += '\n';
for (const auto & strToken : dir.strTokens) {
outs += " <token ";
outs += "column=\"";
outs += std::to_string(strToken.column);
outs += "\" ";
outs += "str=\"";
outs += ErrorLogger::toxml(strToken.tokStr);
outs +="\"/>";
outs += '\n';
}
outs += " </directive>";
outs += '\n';
}
outs += " </directivelist>";
Expand Down
158 changes: 133 additions & 25 deletions test/testtokenize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8075,19 +8075,80 @@ class TestTokenizer : public TestFixture {
"#warning some warning message\n"
"#error some error message\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\"/>\n"
" <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\"/>\n"
" <directive file=\"test.c\" linenr=\"5\" str=\"#else\"/>\n"
" <directive file=\"test.c\" linenr=\"6\" str=\"#endif\"/>\n"
" <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\"/>\n"
" <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\"/>\n"
" <directive file=\"test.c\" linenr=\"9\" str=\"#\"/>\n"
" <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\"/>\n"
" <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\"/>\n"
" <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\"/>\n"
" <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\"/>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro some definition\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro\"/>\n"
" <token column=\"15\" str=\"some\"/>\n"
" <token column=\"20\" str=\"definition\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#undef macro\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"undef\"/>\n"
" <token column=\"8\" str=\"macro\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#ifdef macro\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"ifdef\"/>\n"
" <token column=\"8\" str=\"macro\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"4\" str=\"#elif some (complex) condition\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"elif\"/>\n"
" <token column=\"7\" str=\"some\"/>\n"
" <token column=\"12\" str=\"(\"/>\n"
" <token column=\"13\" str=\"complex\"/>\n"
" <token column=\"20\" str=\")\"/>\n"
" <token column=\"22\" str=\"condition\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"5\" str=\"#else\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"else\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"6\" str=\"#endif\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"endif\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"7\" str=\"#if some other condition\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"if\"/>\n"
" <token column=\"5\" str=\"some\"/>\n"
" <token column=\"10\" str=\"other\"/>\n"
" <token column=\"16\" str=\"condition\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"8\" str=\"#pragma some proprietary content\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"pragma\"/>\n"
" <token column=\"9\" str=\"some\"/>\n"
" <token column=\"14\" str=\"proprietary\"/>\n"
" <token column=\"26\" str=\"content\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"9\" str=\"#\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"10\" str=\"#ident some text\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"ident\"/>\n"
" <token column=\"8\" str=\"some\"/>\n"
" <token column=\"13\" str=\"text\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"11\" str=\"#unknownmacro some unpredictable text\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"unknownmacro\"/>\n"
" <token column=\"15\" str=\"some\"/>\n"
" <token column=\"20\" str=\"unpredictable\"/>\n"
" <token column=\"34\" str=\"text\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"12\" str=\"#warning some warning message\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"warning\"/>\n"
" <token column=\"10\" str=\"some warning message\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"13\" str=\"#error some error message\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"error\"/>\n"
" <token column=\"8\" str=\"some error message\"/>\n"
" </directive>\n"
" </directivelist>\n"
" <tokenlist>\n"
" </tokenlist>\n";
Expand All @@ -8108,17 +8169,49 @@ class TestTokenizer : public TestFixture {
"#endfile\n"
"#define macro5 val\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\"/>\n"
" <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\"/>\n"
" <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\"/>\n"
" <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\"/>\n"
" <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\"/>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro1 val\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro1\"/>\n"
" <token column=\"16\" str=\"val\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#include &quot;inc1.h&quot;\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"file\"/>\n"
" <token column=\"7\" str=\"&quot;inc1.h&quot;\"/>\n"
" </directive>\n"
" <directive file=\"inc1.h\" linenr=\"1\" str=\"#define macro2 val\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro2\"/>\n"
" <token column=\"16\" str=\"val\"/>\n"
" </directive>\n"
" <directive file=\"inc1.h\" linenr=\"2\" str=\"#include &quot;inc2.h&quot;\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"file\"/>\n"
" <token column=\"7\" str=\"&quot;inc2.h&quot;\"/>\n"
" </directive>\n"
" <directive file=\"inc2.h\" linenr=\"1\" str=\"#define macro3 val\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro3\"/>\n"
" <token column=\"16\" str=\"val\"/>\n"
" </directive>\n"
" <directive file=\"inc1.h\" linenr=\"3\" str=\"#define macro4 val\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro4\"/>\n"
" <token column=\"16\" str=\"val\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#define macro5 val\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro5\"/>\n"
" <token column=\"16\" str=\"val\"/>\n"
" </directive>\n"
" </directivelist>\n"
" <tokenlist>\n"
" </tokenlist>\n";

std::ostringstream ostr;
directiveDump(filedata, ostr);
ASSERT_EQUALS(dumpdata, ostr.str());
Expand All @@ -8129,9 +8222,19 @@ class TestTokenizer : public TestFixture {
"#else /* this will be removed too */\n"
"#endif /* this will also be removed */\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\"/>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#else\"/>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#endif\"/>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#ifdef macro2\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"ifdef\"/>\n"
" <token column=\"8\" str=\"macro2\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"2\" str=\"#else\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"else\"/>\n"
" </directive>\n"
" <directive file=\"test.c\" linenr=\"3\" str=\"#endif\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"endif\"/>\n"
" </directive>\n"
" </directivelist>\n"
" <tokenlist>\n"
" </tokenlist>\n";
Expand All @@ -8144,7 +8247,12 @@ class TestTokenizer : public TestFixture {
void testDirectiveRelativePath() {
const char filedata[] = "#define macro 1\n";
const char dumpdata[] = " <directivelist>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro 1\"/>\n"
" <directive file=\"test.c\" linenr=\"1\" str=\"#define macro 1\">\n"
" <token column=\"1\" str=\"#\"/>\n"
" <token column=\"2\" str=\"define\"/>\n"
" <token column=\"9\" str=\"macro\"/>\n"
" <token column=\"15\" str=\"1\"/>\n"
" </directive>\n"
" </directivelist>\n"
" <tokenlist>\n"
" </tokenlist>\n";
Expand Down

0 comments on commit e7811ed

Please sign in to comment.