llcppsigfetch:abs include path & system header & config mode 's inclu…

…de flagsfrom goplus/llgo#853

_xtool/llcppsigfetch/llcppsigfetch.go

@@ -20,16 +20,16 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
 
-	"github.com/goplus/llgo/c"
-	"github.com/goplus/llgo/c/cjson"
 	"github.com/goplus/llcppg/_xtool/llcppsigfetch/parse"
 	"github.com/goplus/llcppg/_xtool/llcppsymg/args"
 	"github.com/goplus/llcppg/_xtool/llcppsymg/clangutils"
 	"github.com/goplus/llcppg/_xtool/llcppsymg/config"
+	"github.com/goplus/llcppg/_xtool/llcppsymg/config/cfgparse"
+	"github.com/goplus/llgo/c"
+	"github.com/goplus/llgo/c/cjson"
 )
 
 func main() {
@@ -150,8 +150,7 @@ func runFromConfig(cfgFile string, useStdin bool, outputToFile bool, verbose boo
 		os.Exit(1)
 	}
 
-	//todo(zzy): reuse the llcppsymg's cflags parse
-	cflag := ParseCFlags(conf.CFlags)
+	cflag := cfgparse.ParseCFlags(conf.CFlags)
 	files, notFounds, err := cflag.GenHeaderFilePaths(conf.Include)
 	check(err)
 
@@ -162,7 +161,16 @@ func runFromConfig(cfgFile string, useStdin bool, outputToFile bool, verbose boo
 		}
 	}
 
-	context := parse.NewContext(conf.Cplusplus)
+	// Generate include directory flags (-I flags)
+	incFlags := make([]string, 0, len(cflag.Paths))
+	for _, path := range cflag.Paths {
+		incFlags = append(incFlags, "-I"+path)
+	}
+
+	context := parse.NewContext(&parse.ContextConfig{
+		Conf:     conf.Config,
+		IncFlags: incFlags,
+	})
 	err = context.ProcessFiles(files)
 	check(err)
 
@@ -208,48 +216,6 @@ func outputResult(result *c.Char, outputToFile bool) {
 	}
 }
 
-// todo(zzy): reuse the llcppsymg's cflags parse https://github.com/goplus/llgo/pull/788
-type CFlags struct {
-	Paths []string // Include Path
-}
-
-func ParseCFlags(cflags string) *CFlags {
-	parts := strings.Fields(cflags)
-	cf := &CFlags{}
-	for _, part := range parts {
-		if strings.HasPrefix(part, "-I") {
-			cf.Paths = append(cf.Paths, part[2:])
-		}
-	}
-	return cf
-}
-
-func (cf *CFlags) GenHeaderFilePaths(files []string) ([]string, []string, error) {
-	var foundPaths []string
-	var notFound []string
-
-	for _, file := range files {
-		var found bool
-		for _, path := range cf.Paths {
-			fullPath := filepath.Join(path, file)
-			if _, err := os.Stat(fullPath); err == nil {
-				foundPaths = append(foundPaths, fullPath)
-				found = true
-				break
-			}
-		}
-		if !found {
-			notFound = append(notFound, file)
-		}
-	}
-
-	if len(foundPaths) == 0 {
-		return nil, notFound, fmt.Errorf("failed to find any header files")
-	}
-
-	return foundPaths, notFound, nil
-}
-
 func outputInfo(context *parse.Context, outputToFile bool) {
 	info := context.Output()
 	str := info.Print()

_xtool/llcppsigfetch/parse/cvt.go

@@ -7,17 +7,19 @@ import (
 	"strings"
 	"unsafe"
 
-	"github.com/goplus/llgo/c"
-	"github.com/goplus/llgo/c/cjson"
-	"github.com/goplus/llgo/c/clang"
 	"github.com/goplus/llcppg/_xtool/llcppsymg/clangutils"
 	"github.com/goplus/llcppg/ast"
 	"github.com/goplus/llcppg/token"
+	"github.com/goplus/llgo/c"
+	"github.com/goplus/llgo/c/cjson"
+	"github.com/goplus/llgo/c/clang"
 )
 
 type FileEntry struct {
-	Path string
-	Doc  *ast.File
+	Path    string
+	IncPath string
+	IsSys   bool
+	Doc     *ast.File
 }
 
 type Converter struct {
@@ -58,11 +60,14 @@ func NewConverter(config *clangutils.Config) (*Converter, error) {
 		return nil, err
 	}
 
+	files := initFileEntries(unit)
+
 	return &Converter{
-		Files: make([]*FileEntry, 0),
+		Files: files,
 		index: index,
 		unit:  unit,
 	}, nil
+
 }
 
 func (ct *Converter) Dispose() {
@@ -71,6 +76,26 @@ func (ct *Converter) Dispose() {
 	ct.unit.Dispose()
 }
 
+func initFileEntries(unit *clang.TranslationUnit) []*FileEntry {
+	files := make([]*FileEntry, 0)
+	clangutils.GetInclusions(unit, func(inced clang.File, incins []clang.SourceLocation) {
+		loc := unit.GetLocation(inced, 1, 1)
+		incedFile := toStr(inced.FileName())
+		var incPath string
+		if len(incins) > 0 {
+			cur := unit.GetCursor(&incins[0])
+			incPath = toStr(cur.String())
+		}
+		files = append(files, &FileEntry{
+			Path:    incedFile,
+			IncPath: incPath,
+			IsSys:   loc.IsInSystemHeader() != 0,
+			Doc:     &ast.File{},
+		})
+	})
+	return files
+}
+
 func (ct *Converter) GetTokens(cursor clang.Cursor) []*ast.Token {
 	ran := cursor.Extent()
 	var numTokens c.Uint
@@ -121,42 +146,40 @@ func (ct *Converter) logln(args ...interface{}) {
 	}
 }
 
-func (ct *Converter) UpdateLoc(cursor clang.Cursor) {
+func (ct *Converter) GetCurFile(cursor clang.Cursor) *ast.File {
 	loc := cursor.Location()
 	var file clang.File
 	loc.SpellingLocation(&file, nil, nil, nil)
-
 	filePath := toStr(file.FileName())
-
 	if filePath == "" {
 		//todo(zzy): For some built-in macros, there is no file.
 		ct.curLoc = ast.Location{File: ""}
-		return
-	}
-	ct.curLoc = ast.Location{File: filePath}
-}
-
-func (ct *Converter) GetCurFile() *ast.File {
-	if ct.curLoc.File == "" {
 		ct.logln("GetCurFile: NO FILE")
 		return nil
 	}
+	ct.curLoc = ast.Location{File: filePath}
+
 	// todo(zzy): more efficient
 	for i, entry := range ct.Files {
-		if entry.Path == ct.curLoc.File {
-			ct.logln("GetCurFile: found", ct.curLoc.File)
+		if entry.Path == filePath {
+			ct.logln("GetCurFile: found", filePath)
 			return ct.Files[i].Doc
 		}
 	}
-	ct.logln("GetCurFile: Create New ast.File", ct.curLoc.File)
-	newDoc := &ast.File{}
-	ct.Files = append(ct.Files, &FileEntry{Path: ct.curLoc.File, Doc: newDoc})
-	return newDoc
+	ct.logln("GetCurFile: Create New ast.File", filePath)
+	entry := &FileEntry{Path: filePath, Doc: &ast.File{}, IsSys: false}
+	if loc.IsInSystemHeader() != 0 {
+		entry.IsSys = true
+	}
+	ct.Files = append(ct.Files, entry)
+	return entry.Doc
 }
 
 func (ct *Converter) CreateDeclBase(cursor clang.Cursor) ast.DeclBase {
 	base := ast.DeclBase{
-		Loc:    &ct.curLoc,
+		Loc: &ast.Location{
+			File: ct.curLoc.File,
+		},
 		Parent: ct.BuildScopingExpr(cursor.SemanticParent()),
 	}
 	commentGroup, isDoc := ct.ParseCommentGroup(cursor)
@@ -206,9 +229,7 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul
 	ct.incIndent()
 	defer ct.decIndent()
 
-	ct.UpdateLoc(cursor)
-
-	curFile := ct.GetCurFile()
+	curFile := ct.GetCurFile(cursor)
 
 	name := toStr(cursor.String())
 	ct.logf("visitTop: Cursor: %s\n", name)
@@ -219,7 +240,11 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul
 
 	switch cursor.Kind {
 	case clang.CursorInclusionDirective:
-		include := ct.ProcessInclude(cursor)
+		include, err := ct.ProcessInclude(cursor)
+		if err != nil {
+			ct.logln(err)
+			return clang.ChildVisit_Continue
+		}
 		curFile.Includes = append(curFile.Includes, include)
 		ct.logln("visitTop: ProcessInclude END ", include.Path)
 	case clang.CursorMacroDefinition:
@@ -276,27 +301,18 @@ func (ct *Converter) visitTop(cursor, parent clang.Cursor) clang.ChildVisitResul
 		curFile.Decls = append(curFile.Decls, typedefDecl)
 		ct.logln("visitTop: ProcessTypeDefDecl END", typedefDecl.Name.Name)
 	case clang.CursorNamespace:
-		VisitChildren(cursor, ct.visitTop)
+		clangutils.VisitChildren(cursor, ct.visitTop)
 	}
 	return clang.ChildVisit_Continue
 }
 
 func (ct *Converter) Convert() ([]*FileEntry, error) {
 	cursor := ct.unit.Cursor()
 	// visit top decls (struct,class,function & macro,include)
-	VisitChildren(cursor, ct.visitTop)
+	clangutils.VisitChildren(cursor, ct.visitTop)
 	return ct.Files, nil
 }
 
-type Visitor func(cursor, parent clang.Cursor) clang.ChildVisitResult
-
-func VisitChildren(cursor clang.Cursor, fn Visitor) c.Uint {
-	return clang.VisitChildren(cursor, func(cursor, parent clang.Cursor, clientData unsafe.Pointer) clang.ChildVisitResult {
-		cfn := *(*Visitor)(clientData)
-		return cfn(cursor, parent)
-	}, unsafe.Pointer(&fn))
-}
-
 func (ct *Converter) ProcessType(t clang.Type) ast.Expr {
 	ct.incIndent()
 	defer ct.decIndent()
@@ -521,7 +537,7 @@ func (ct *Converter) ProcessMethodAttributes(cursor clang.Cursor, fn *ast.FuncDe
 func (ct *Converter) ProcessEnumType(cursor clang.Cursor) *ast.EnumType {
 	items := make([]*ast.EnumItem, 0)
 
-	VisitChildren(cursor, func(cursor, parent clang.Cursor) clang.ChildVisitResult {
+	clangutils.VisitChildren(cursor, func(cursor, parent clang.Cursor) clang.ChildVisitResult {
 		if cursor.Kind == clang.CursorEnumConstantDecl {
 			name := cursor.String()
 			defer name.Dispose()
@@ -578,9 +594,14 @@ func (ct *Converter) ProcessMacro(cursor clang.Cursor) *ast.Macro {
 	return macro
 }
 
-func (ct *Converter) ProcessInclude(cursor clang.Cursor) *ast.Include {
+func (ct *Converter) ProcessInclude(cursor clang.Cursor) (*ast.Include, error) {
 	name := toStr(cursor.String())
-	return &ast.Include{Path: name}
+	includedFile := cursor.IncludedFile()
+	includedPath := toStr(includedFile.FileName())
+	if includedPath == "" {
+		return nil, fmt.Errorf("%s: failed to get included file", name)
+	}
+	return &ast.Include{Path: includedPath}, nil
 }
 
 func (ct *Converter) createBaseField(cursor clang.Cursor) *ast.Field {
@@ -619,7 +640,7 @@ func (ct *Converter) ProcessFieldList(cursor clang.Cursor) *ast.FieldList {
 
 	params := &ast.FieldList{}
 	ct.logln("ProcessFieldList: VisitChildren")
-	VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult {
+	clangutils.VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult {
 		switch subcsr.Kind {
 		case clang.CursorParmDecl, clang.CursorFieldDecl:
 			// In C language, parameter lists do not have similar parameter grouping in Go.
@@ -665,7 +686,7 @@ func (ct *Converter) ProcessFieldList(cursor clang.Cursor) *ast.FieldList {
 // Note:Public Method is considered
 func (ct *Converter) ProcessMethods(cursor clang.Cursor) []*ast.FuncDecl {
 	methods := make([]*ast.FuncDecl, 0)
-	VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult {
+	clangutils.VisitChildren(cursor, func(subcsr, parent clang.Cursor) clang.ChildVisitResult {
 		if isMethod(subcsr) && subcsr.CXXAccessSpecifier() == clang.CXXPublic {
 			method := ct.ProcessFuncDecl(subcsr)
 			if method != nil {
@@ -950,8 +971,7 @@ func isRangeChildOf(childRange, parentRange clang.SourceRange) bool {
 }
 
 func getOffset(location clang.SourceLocation) c.Uint {
-	var offset c.Uint
-	location.SpellingLocation(nil, nil, nil, &offset)
+	_, _, _, offset := clangutils.GetLocation(location)
 	return offset
 }
 

_xtool/llcppsigfetch/parse/cvt_test/cvt.go

@@ -79,7 +79,7 @@ func GetType(option *GetTypeOptions) (clang.Type, *clang.Index, *clang.Translati
 	}
 	cursor := unit.Cursor()
 	var typ clang.Type
-	parse.VisitChildren(cursor, func(child, parent clang.Cursor) clang.ChildVisitResult {
+	clangutils.VisitChildren(cursor, func(child, parent clang.Cursor) clang.ChildVisitResult {
 		if child.Kind == clang.CursorVarDecl && (option.ExpectTypeKind == clang.TypeInvalid || option.ExpectTypeKind == child.Type().Kind) {
 			typ = child.Type()
 			return clang.ChildVisit_Break

_xtool/llcppsigfetch/parse/cvt_test/preprocess_test/llgo.expect

@@ -1,4 +1,9 @@
 #stdout
+=== TestSystemHeader ===
+stdio.h is absolute path
+include files are all system headers
+=== TestInclusionMap ===
+sys/types.h include path found
 TestDefine Case 1:
 {
 	"temp.h":	{
@@ -110,10 +115,7 @@ TestInclude Case 1:
 	"temp.h":	{
 		"_Type":	"File",
 		"decls":	[],
-		"includes":	[{
-				"_Type":	"Include",
-				"Path":	"foo.h"
-			}],
+		"includes":	[],
 		"macros":	[]
 	}
 }
@@ -169,7 +171,7 @@ TestMacroExpansionOtherFile:
 			}],
 		"includes":	[{
 				"_Type":	"Include",
-				"Path":	"def.h"
+				"Path":	"./testdata/macroexpan/def.h"
 			}],
 		"macros":	[]
 	},