diff --git a/_xtool/llcppsigfetch/llcppsigfetch.go b/_xtool/llcppsigfetch/llcppsigfetch.go index 86ae9047..afbc67b8 100644 --- a/_xtool/llcppsigfetch/llcppsigfetch.go +++ b/_xtool/llcppsigfetch/llcppsigfetch.go @@ -20,16 +20,16 @@ import ( "fmt" "io" "os" - "path/filepath" "strconv" "strings" - "github.com/goplus/llgo/c" - "github.com/goplus/llgo/c/cjson" "github.com/goplus/llcppg/_xtool/llcppsigfetch/parse" "github.com/goplus/llcppg/_xtool/llcppsymg/args" "github.com/goplus/llcppg/_xtool/llcppsymg/clangutils" "github.com/goplus/llcppg/_xtool/llcppsymg/config" + "github.com/goplus/llcppg/_xtool/llcppsymg/config/cfgparse" + "github.com/goplus/llgo/c" + "github.com/goplus/llgo/c/cjson" ) func main() { @@ -150,8 +150,7 @@ func runFromConfig(cfgFile string, useStdin bool, outputToFile bool, verbose boo os.Exit(1) } - //todo(zzy): reuse the llcppsymg's cflags parse - cflag := ParseCFlags(conf.CFlags) + cflag := cfgparse.ParseCFlags(conf.CFlags) files, notFounds, err := cflag.GenHeaderFilePaths(conf.Include) check(err) @@ -162,7 +161,16 @@ func runFromConfig(cfgFile string, useStdin bool, outputToFile bool, verbose boo } } - context := parse.NewContext(conf.Cplusplus) + // Generate include directory flags (-I flags) + incFlags := make([]string, 0, len(cflag.Paths)) + for _, path := range cflag.Paths { + incFlags = append(incFlags, "-I"+path) + } + + context := parse.NewContext(&parse.ContextConfig{ + Conf: conf.Config, + IncFlags: incFlags, + }) err = context.ProcessFiles(files) check(err) @@ -208,48 +216,6 @@ func outputResult(result *c.Char, outputToFile bool) { } } -// todo(zzy): reuse the llcppsymg's cflags parse https://github.com/goplus/llgo/pull/788 -type CFlags struct { - Paths []string // Include Path -} - -func ParseCFlags(cflags string) *CFlags { - parts := strings.Fields(cflags) - cf := &CFlags{} - for _, part := range parts { - if strings.HasPrefix(part, "-I") { - cf.Paths = append(cf.Paths, part[2:]) - } - } - return cf -} - -func (cf *CFlags) GenHeaderFilePaths(files []string) ([]string, []string, error) { - var foundPaths []string - var notFound []string - - for _, file := range files { - var found bool - for _, path := range cf.Paths { - fullPath := filepath.Join(path, file) - if _, err := os.Stat(fullPath); err == nil { - foundPaths = append(foundPaths, fullPath) - found = true - break - } - } - if !found { - notFound = append(notFound, file) - } - } - - if len(foundPaths) == 0 { - return nil, notFound, fmt.Errorf("failed to find any header files") - } - - return foundPaths, notFound, nil -} - func outputInfo(context *parse.Context, outputToFile bool) { info := context.Output() str := info.Print() diff --git a/_xtool/llcppsigfetch/parse/cvt.go b/_xtool/llcppsigfetch/parse/cvt.go index cce88dcc..b25bab85 100644 --- a/_xtool/llcppsigfetch/parse/cvt.go +++ b/_xtool/llcppsigfetch/parse/cvt.go @@ -7,17 +7,19 @@ import ( "strings" "unsafe" - "github.com/goplus/llgo/c" - "github.com/goplus/llgo/c/cjson" - "github.com/goplus/llgo/c/clang" "github.com/goplus/llcppg/_xtool/llcppsymg/clangutils" "github.com/goplus/llcppg/ast" "github.com/goplus/llcppg/token" + "github.com/goplus/llgo/c" + "github.com/goplus/llgo/c/cjson" + "github.com/goplus/llgo/c/clang" ) type FileEntry struct { - Path string - Doc *ast.File + Path string + IncPath string + IsSys bool + Doc *ast.File } type Converter struct { @@ -58,11 +60,14 @@ func NewConverter(config *clangutils.Config) (*Converter, error) { return nil, err } + files := initFileEntries(unit) + return &Converter{ - Files: make([]*FileEntry, 0), + Files: files, index: index, unit: unit, }, nil + } func (ct *Converter) Dispose() { @@ -71,6 +76,26 @@ func (ct *Converter) Dispose() { ct.unit.Dispose() } +func initFileEntries(unit *clang.TranslationUnit) []*FileEntry { + files := make([]*FileEntry, 0) + clangutils.GetInclusions(unit, func(inced clang.File, incins []clang.SourceLocation) { + loc := unit.GetLocation(inced, 1, 1) + incedFile := toStr(inced.FileName()) + var incPath string + if len(incins) > 0 { + cur := unit.GetCursor(&incins[0]) + incPath = toStr(cur.String()) + } + files = append(files, &FileEntry{ + Path: incedFile, + IncPath: incPath, + IsSys: loc.IsInSystemHeader() != 0, + Doc: &ast.File{}, + }) + }) + return files +} + func (ct *Converter) GetTokens(cursor clang.Cursor) []*ast.Token { ran := cursor.Extent() var numTokens c.Uint @@ -121,42 +146,40 @@ func (ct *Converter) logln(args ...interface{}) { } } -func (ct *Converter) UpdateLoc(cursor clang.Cursor) { +func (ct *Converter) GetCurFile(cursor clang.Cursor) *ast.File { loc := cursor.Location() var file clang.File loc.SpellingLocation(&file, nil, nil, nil) - filePath := toStr(file.FileName()) - if filePath == "" { //todo(zzy): For some built-in macros, there is no file. ct.curLoc = ast.Location{File: ""} - return - } - ct.curLoc = ast.Location{File: filePath} -} - -func (ct *Converter) GetCurFile() *ast.File { - if ct.curLoc.File == "" { ct.logln("GetCurFile: NO FILE") return nil } + ct.curLoc = ast.Location{File: filePath} + // todo(zzy): more efficient for i, entry := range ct.Files { - if entry.Path == ct.curLoc.File { - ct.logln("GetCurFile: found", ct.curLoc.File) + if entry.Path == filePath { + ct.logln("GetCurFile: found", filePath) return ct.Files[i].Doc } } - ct.logln("GetCurFile: Create New ast.File", ct.curLoc.File) - newDoc := &ast.File{} - ct.Files = append(ct.Files, &FileEntry{Path: ct.curLoc.File, Doc: newDoc}) - return newDoc + ct.logln("GetCurFile: Create New ast.File", filePath) + entry := &FileEntry{Path: filePath, Doc: &ast.File{}, IsSys: false} + if loc.IsInSystemHeader() != 0 { + entry.IsSys = true + } + ct.Files = append(ct.Files, entry) + return entry.Doc } func (ct *Converter) CreateDeclBase(cursor clang.Cursor) ast.DeclBase { base := ast.DeclBase{ - Loc: &ct.curLoc, + Loc: &ast.Location{ + File: ct.curLoc.File, + }, Parent: ct.BuildScopingExpr(cursor.SemanticParent()), } commentGroup, isDoc := ct.ParseCommentGroup(cursor) @@ -206,9 +229,7 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul ct.incIndent() defer ct.decIndent() - ct.UpdateLoc(cursor) - - curFile := ct.GetCurFile() + curFile := ct.GetCurFile(cursor) name := toStr(cursor.String()) ct.logf("visitTop: Cursor: %s\n", name) @@ -219,7 +240,11 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul switch cursor.Kind { case clang.CursorInclusionDirective: - include := ct.ProcessInclude(cursor) + include, err := ct.ProcessInclude(cursor) + if err != nil { + ct.logln(err) + return clang.ChildVisit_Continue + } curFile.Includes = append(curFile.Includes, include) ct.logln("visitTop: ProcessInclude END ", include.Path) case clang.CursorMacroDefinition: @@ -276,7 +301,7 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul curFile.Decls = append(curFile.Decls, typedefDecl) ct.logln("visitTop: ProcessTypeDefDecl END", typedefDecl.Name.Name) case clang.CursorNamespace: - VisitChildren(cursor, ct.visitTop) + clangutils.VisitChildren(cursor, ct.visitTop) } return clang.ChildVisit_Continue } @@ -284,19 +309,10 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul func (ct *Converter) Convert() ([]*FileEntry, error) { cursor := ct.unit.Cursor() // visit top decls (struct,class,function & macro,include) - VisitChildren(cursor, ct.visitTop) + clangutils.VisitChildren(cursor, ct.visitTop) return ct.Files, nil } -type Visitor func(cursor, parent clang.Cursor) clang.ChildVisitResult - -func VisitChildren(cursor clang.Cursor, fn Visitor) c.Uint { - return clang.VisitChildren(cursor, func(cursor, parent clang.Cursor, clientData unsafe.Pointer) clang.ChildVisitResult { - cfn := *(*Visitor)(clientData) - return cfn(cursor, parent) - }, unsafe.Pointer(&fn)) -} - func (ct *Converter) ProcessType(t clang.Type) ast.Expr { ct.incIndent() defer ct.decIndent() @@ -521,7 +537,7 @@ func (ct *Converter) ProcessMethodAttributes(cursor clang.Cursor, fn *ast.FuncDe func (ct *Converter) ProcessEnumType(cursor clang.Cursor) *ast.EnumType { items := make([]*ast.EnumItem, 0) - VisitChildren(cursor, func(cursor, parent clang.Cursor) clang.ChildVisitResult { + clangutils.VisitChildren(cursor, func(cursor, parent clang.Cursor) clang.ChildVisitResult { if cursor.Kind == clang.CursorEnumConstantDecl { name := cursor.String() defer name.Dispose() @@ -578,9 +594,14 @@ func (ct *Converter) ProcessMacro(cursor clang.Cursor) *ast.Macro { return macro } -func (ct *Converter) ProcessInclude(cursor clang.Cursor) *ast.Include { +func (ct *Converter) ProcessInclude(cursor clang.Cursor) (*ast.Include, error) { name := toStr(cursor.String()) - return &ast.Include{Path: name} + includedFile := cursor.IncludedFile() + includedPath := toStr(includedFile.FileName()) + if includedPath == "" { + return nil, fmt.Errorf("%s: failed to get included file", name) + } + return &ast.Include{Path: includedPath}, nil } func (ct *Converter) createBaseField(cursor clang.Cursor) *ast.Field { @@ -619,7 +640,7 @@ func (ct *Converter) ProcessFieldList(cursor clang.Cursor) *ast.FieldList { params := &ast.FieldList{} ct.logln("ProcessFieldList: VisitChildren") - VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult { + clangutils.VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult { switch subcsr.Kind { case clang.CursorParmDecl, clang.CursorFieldDecl: // In C language, parameter lists do not have similar parameter grouping in Go. @@ -665,7 +686,7 @@ func (ct *Converter) ProcessFieldList(cursor clang.Cursor) *ast.FieldList { // Note:Public Method is considered func (ct *Converter) ProcessMethods(cursor clang.Cursor) []*ast.FuncDecl { methods := make([]*ast.FuncDecl, 0) - VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult { + clangutils.VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult { if isMethod(subcsr) && subcsr.CXXAccessSpecifier() == clang.CXXPublic { method := ct.ProcessFuncDecl(subcsr) if method != nil { @@ -950,8 +971,7 @@ func isRangeChildOf(childRange, parentRange clang.SourceRange) bool { } func getOffset(location clang.SourceLocation) c.Uint { - var offset c.Uint - location.SpellingLocation(nil, nil, nil, &offset) + _, _, _, offset := clangutils.GetLocation(location) return offset } diff --git a/_xtool/llcppsigfetch/parse/cvt_test/cvt.go b/_xtool/llcppsigfetch/parse/cvt_test/cvt.go index b46d4579..01ae1411 100644 --- a/_xtool/llcppsigfetch/parse/cvt_test/cvt.go +++ b/_xtool/llcppsigfetch/parse/cvt_test/cvt.go @@ -79,7 +79,7 @@ func GetType(option *GetTypeOptions) (clang.Type, *clang.Index, *clang.Translati } cursor := unit.Cursor() var typ clang.Type - parse.VisitChildren(cursor, func(child, parent clang.Cursor) clang.ChildVisitResult { + clangutils.VisitChildren(cursor, func(child, parent clang.Cursor) clang.ChildVisitResult { if child.Kind == clang.CursorVarDecl && (option.ExpectTypeKind == clang.TypeInvalid || option.ExpectTypeKind == child.Type().Kind) { typ = child.Type() return clang.ChildVisit_Break diff --git a/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/llgo.expect b/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/llgo.expect index bf071926..2705b502 100644 --- a/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/llgo.expect +++ b/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/llgo.expect @@ -1,4 +1,9 @@ #stdout +=== TestSystemHeader === +stdio.h is absolute path +include files are all system headers +=== TestInclusionMap === +sys/types.h include path found TestDefine Case 1: { "temp.h": { @@ -110,10 +115,7 @@ TestInclude Case 1: "temp.h": { "_Type": "File", "decls": [], - "includes": [{ - "_Type": "Include", - "Path": "foo.h" - }], + "includes": [], "macros": [] } } @@ -169,7 +171,7 @@ TestMacroExpansionOtherFile: }], "includes": [{ "_Type": "Include", - "Path": "def.h" + "Path": "./testdata/macroexpan/def.h" }], "macros": [] }, diff --git a/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/preprocess.go b/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/preprocess.go index 7457a25b..c221414c 100644 --- a/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/preprocess.go +++ b/_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/preprocess.go @@ -1,14 +1,22 @@ package main import ( + "fmt" + "path/filepath" + "strings" + + "github.com/goplus/llcppg/_xtool/llcppsigfetch/parse" test "github.com/goplus/llcppg/_xtool/llcppsigfetch/parse/cvt_test" "github.com/goplus/llcppg/_xtool/llcppsymg/clangutils" + "github.com/goplus/llcppg/ast" "github.com/goplus/llgo/c" ) func main() { TestDefine() TestInclude() + TestSystemHeader() + TestInclusionMap() TestMacroExpansionOtherFile() } @@ -29,6 +37,71 @@ func TestInclude() { test.RunTest("TestInclude", testCases) } +func TestInclusionMap() { + fmt.Println("=== TestInclusionMap ===") + converter, err := parse.NewConverter(&clangutils.Config{ + File: "#include ", + Temp: true, + IsCpp: false, + }) + if err != nil { + panic(err) + } + found := false + for _, f := range converter.Files { + if f.IncPath == "sys/types.h" { + found = true + } + } + if !found { + panic("sys/types.h not found") + } else { + fmt.Println("sys/types.h include path found") + } +} + +func TestSystemHeader() { + fmt.Println("=== TestSystemHeader ===") + converter, err := parse.NewConverter(&clangutils.Config{ + File: "#include ", + Temp: true, + IsCpp: false, + }) + if err != nil { + panic(err) + } + converter.Convert() + if len(converter.Files) < 2 { + panic("expect 2 files") + } + if converter.Files[0].IsSys { + panic("entry file is not system header") + } + + includePath := converter.Files[0].Doc.Includes[0].Path + if strings.HasSuffix(includePath, "stdio.h") && filepath.IsAbs(includePath) { + fmt.Println("stdio.h is absolute path") + } + + for i := 1; i < len(converter.Files); i++ { + if !converter.Files[i].IsSys { + panic(fmt.Errorf("include file is not system header: %s", converter.Files[i].Path)) + } + for _, decl := range converter.Files[i].Doc.Decls { + switch decl := decl.(type) { + case *ast.TypeDecl: + case *ast.EnumTypeDecl: + case *ast.FuncDecl: + case *ast.TypedefDecl: + if decl.DeclBase.Loc.File != converter.Files[i].Path { + fmt.Println("Decl is not in the file", decl.DeclBase.Loc.File, "expect", converter.Files[i].Path) + } + } + } + } + fmt.Println("include files are all system headers") +} + func TestMacroExpansionOtherFile() { c.Printf(c.Str("TestMacroExpansionOtherFile:\n")) test.RunTestWithConfig(&clangutils.Config{ diff --git a/_xtool/llcppsigfetch/parse/dump.go b/_xtool/llcppsigfetch/parse/dump.go index 434abfb4..c00b6ed3 100644 --- a/_xtool/llcppsigfetch/parse/dump.go +++ b/_xtool/llcppsigfetch/parse/dump.go @@ -1,9 +1,9 @@ package parse import ( + "github.com/goplus/llcppg/ast" "github.com/goplus/llgo/c" "github.com/goplus/llgo/c/cjson" - "github.com/goplus/llcppg/ast" ) func MarshalOutputASTFiles(files []*FileEntry) *cjson.JSON { @@ -11,8 +11,11 @@ func MarshalOutputASTFiles(files []*FileEntry) *cjson.JSON { for _, entry := range files { f := cjson.Object() path := cjson.String(c.AllocaCStr(entry.Path)) + incPath := cjson.String(c.AllocaCStr(entry.IncPath)) + f.SetItem(c.Str("incPath"), incPath) f.SetItem(c.Str("path"), path) f.SetItem(c.Str("doc"), MarshalASTFile(entry.Doc)) + f.SetItem(c.Str("isSys"), boolField(entry.IsSys)) root.AddItem(f) } return root diff --git a/_xtool/llcppsigfetch/parse/parse.go b/_xtool/llcppsigfetch/parse/parse.go index 7507ed83..0971922b 100644 --- a/_xtool/llcppsigfetch/parse/parse.go +++ b/_xtool/llcppsigfetch/parse/parse.go @@ -4,8 +4,10 @@ import ( "errors" "fmt" "os" + "strings" "github.com/goplus/llcppg/_xtool/llcppsymg/clangutils" + "github.com/goplus/llcppg/types" "github.com/goplus/llgo/c/cjson" ) @@ -26,13 +28,21 @@ func SetDebug(dbgFlags dbgFlags) { type Context struct { Files []*FileEntry - IsCpp bool + *ContextConfig } -func NewContext(isCpp bool) *Context { +type ContextConfig struct { + Conf *types.Config + IncFlags []string +} + +func NewContext(cfg *ContextConfig) *Context { return &Context{ Files: make([]*FileEntry, 0), - IsCpp: isCpp, + ContextConfig: &ContextConfig{ + Conf: cfg.Conf, + IncFlags: cfg.IncFlags, + }, } } @@ -43,7 +53,7 @@ func (p *Context) Output() *cjson.JSON { // ProcessFiles processes the given files and adds them to the context func (p *Context) ProcessFiles(files []string) error { if debugParse { - fmt.Fprintln(os.Stderr, "ProcessFiles: files", files, "isCpp", p.IsCpp) + fmt.Fprintln(os.Stderr, "ProcessFiles: files", files, "isCpp", p.Conf.Cplusplus) } for _, file := range files { if err := p.processFile(file); err != nil { @@ -82,7 +92,8 @@ func (p *Context) parseFile(path string) ([]*FileEntry, error) { converter, err := NewConverter(&clangutils.Config{ File: path, Temp: false, - IsCpp: p.IsCpp, + IsCpp: p.Conf.Cplusplus, + Args: p.IncFlags, }) if err != nil { return nil, errors.New("failed to create converter " + path) @@ -90,6 +101,24 @@ func (p *Context) parseFile(path string) ([]*FileEntry, error) { defer converter.Dispose() files, err := converter.Convert() + + // the entry file is the first file in the files list + entryFile := files[0] + if entryFile.IncPath != "" { + return nil, errors.New("entry file " + entryFile.Path + " has include path " + entryFile.IncPath) + } + + for _, include := range p.Conf.Include { + if strings.Contains(entryFile.Path, include) { + entryFile.IncPath = include + break + } + } + + if entryFile.IncPath == "" { + return nil, errors.New("entry file " + entryFile.Path + " is not in include list") + } + if err != nil { return nil, err } diff --git a/_xtool/llcppsymg/clangutils/clangutils.go b/_xtool/llcppsymg/clangutils/clangutils.go index e5821b9d..e32d9ebd 100644 --- a/_xtool/llcppsymg/clangutils/clangutils.go +++ b/_xtool/llcppsymg/clangutils/clangutils.go @@ -18,6 +18,8 @@ type Config struct { type Visitor func(cursor, parent clang.Cursor) clang.ChildVisitResult +type InclusionVisitor func(included_file clang.File, inclusions []clang.SourceLocation) + const TEMP_FILE = "temp.h" func CreateTranslationUnit(config *Config) (*clang.Index, *clang.TranslationUnit, error) { @@ -99,3 +101,11 @@ func VisitChildren(cursor clang.Cursor, fn Visitor) c.Uint { return cfn(cursor, parent) }, unsafe.Pointer(&fn)) } + +func GetInclusions(unit *clang.TranslationUnit, visitor InclusionVisitor) { + clang.GetInclusions(unit, func(inced clang.File, incin *clang.SourceLocation, incilen c.Uint, data c.Pointer) { + ics := unsafe.Slice(incin, incilen) + cfn := *(*InclusionVisitor)(data) + cfn(inced, ics) + }, unsafe.Pointer(&visitor)) +}