kram-profile - add TotalDebugType, don't add ._ files from archive

Apple stores ._Filename.trace files into zip archives with resource forks if using Finder.  Those are not .trace files, or whatever original extension is.

Some work on threading the report calcs, but not done yet.

kram-profile/README.md

@@ -173,7 +173,7 @@ This is a minimal version of Make.  But code must generate the Ninja file.  Cmak
 Unity builds
 -----------
 
-Not to be confused with the Unity game engine.  But unity builds combine several .cpp files into a single .cpp.  This works around problems with slow linkers, and multile template and inline code instantations.  But code and macros from one .cpp spill into the next.  To facilitate this, be careful about undeffing at the bottoms of files.  kram also uses a common namespaces across headers and source files.  This allows "using namespace" in both, and keeps the sources compartmentalized.
+Not to be confused with the Unity game engine.  But unity builds combine several .cpp files into a single .cpp.  This works around problems with slow linkers, and multiple template and inline code instantations.  But code and macros from one .cpp spill into the next.  To avoid this, be careful about undeffing at the bottoms of files.  kram also uses a common namespaces across headers and source files.  This allows "using namespace" in both, and keeps the namespaces compartmentalized.
 
 Precompiled headers (PCH)
 -----------
@@ -184,39 +184,67 @@ pch spread headers into files.  So the build can break if some don't use it, or
 
 There are broken examples of setting up pch for Makefiles all over the internet.  Maybe cmake has a valid setup, but the jist is below for gcc/clang.  Make sure to verify the parse time is gone in kram-profile by looking at the clang build profiles.
 
+Clang has options to generate a pch .o file.  This must be linked separately into the library.  This is something MSVC pch support for a long time.  gcc doesn't support this.  See the link below, and the pchObj in the makefile example below.
+
+Advanced clang pch usage
+https://maskray.me/blog/2023-07-16-precompiled-headers
+
+
     # gen the .d file, written to tmp and only replaces if it changes
-    cppFlags = ... -MMD -MP (or -MD)
+    cppFlags = ... 
+
+    cppDepFlags = -MMD -MP (or -MD)
 
+    # header must be unique to build (f.e. defines, etc)
+    cppBuild = $(platform)($config)
+
     # setup the files involved, only get 1 pch per DLL/App since
     pchSrc = Precompile.h
-    pchHdr = Precompile-$(platform)($config).h
+    pchHdrSrc = Precompile-$(cppBuild).h
     pchDeps = $(pchHdr).d
-    pchObj = $(pchHdr).gch
+    pchHdr = $(pchHdrSrc).pch
+    pchObj = $(pchHdr).o
     pchIncludesDirs = -Idir1 -Idir2
-            
+
+    # this does code gen, templates, and debuginfo into the h.pch.o file
+    pchFlags = -fpch-codegen -fpch-instantiate-templates -fpch-debuginfo
+             
     # important - only copy the hdr if it changes, don't want full rebuild every time
-    $(pchHdr): $(pchSrc)
-        $cp $< $@
+    # linux (cp -u), win (xcopy), macOS (shell compare then cp)
+    $(pchHdrSrc): $(pchSrc)
+        cp $< $@
         
     # this will output the .d and .gch file
-    $(pchObj): $(pchHdr)
-        clang++ -x c++header $(cppFlags) -c $< -o $@ -$(pchIncludesDirs)
+    $(pchHdr): $(pchHdrSrc)
+        clang++ -x c++header $(cppFlags) $(cppDepFlags) $(pchFlags) $(pchIncludesDirs) -c $< -o $@ 
         
     # this makes sure that the pch is rebuilt if hdrs within pchHdr changee
     # the - sign ignores the deps file on the first run where it does not exist.
     $(pchDeps): ;
     -include $(pchDeps)
 
+    # optional code to build .o from .pch 
+    # must link this in with the lib/exe, don't use "-x c++" here - it's ast not C++ code
+    #  speeds the build, since code isn't prepended to each .o file, and then linked.
+    $(pchObj): $(pchHdr)
+        clang++ $(cppFlags) -c $< -o $@
+
     ....
 
-    # force include Precompile.h, 
-    # and then use the pch obj to avoid parsing (appends to top of .o)
-    cppPchFlags = -include $(pchHdr) -include-pch $(pchObj))
+    # prefix Precompile.h.pch to each .o file
+    cppPchFlags = -include-pch $(pchHdr)
 
     # now build the files
     *.cpp: ... $(pchHdr)
-        clang++ $(cppFlags) -c $< -o $@ $(cppPchFlags)
+        clang++ $(cppFlags) $(cppPchFlags) -c $< -o $@ 
 
+    # link the pchObj into the lib or ese
+    allObjs = *.o $(pchObj)
+
+    $(libOrExe): $(allObjs)
+        clang++ $< -o $@
+        
+        
 SIMD
 -----------
 

kram-profile/kram-profile/File.swift

@@ -33,6 +33,7 @@ class BuildStats {
     var totalInstantiateFunction = 0
     var totalInstantiateClass = 0
     var totalCodeGenFunction = 0
+    var totalDebugType = 0
 
     var totalBackend = 0
     var totalOptimizer = 0
@@ -47,6 +48,7 @@ class BuildStats {
         totalInstantiateFunction += rhs.totalInstantiateFunction
         totalInstantiateClass += rhs.totalInstantiateClass
         totalCodeGenFunction += rhs.totalCodeGenFunction
+        totalDebugType += rhs.totalDebugType
 
         totalBackend += rhs.totalBackend
         totalOptimizer += rhs.totalOptimizer
@@ -65,6 +67,7 @@ class BuildStats {
         totalInstantiateFunction /= s
         totalInstantiateClass /= s
         totalCodeGenFunction /= s
+        totalDebugType /= s
 
         totalBackend /= s
         totalOptimizer /= s
@@ -460,10 +463,17 @@ func loadFileContent(_ file: File) -> Data {
 
 func isSupportedFilename(_ url: URL) -> Bool {
     let ext = url.pathExtension
-    
+
     // what ext does trace.zip, or trace.gz come in as ?
     // should this limit compressed files to the names supported below
 
+    // Apple and Microsoft store resource fork data in "._Filename.trace" files
+    // so need to ignore these in the lists.  These don't occur from CLI zip,
+    // only from using Finder "Compress"
+    if url.lastPathComponent.starts(with: "._") {
+        return false
+    }
+
     if ext == "gz" {
         return true
     }

kram-profile/kram-profile/kram_profileApp.swift

@@ -1522,10 +1522,34 @@ func updateBuildTimingsTask(_ files: [File]) {
 
     if counter == 0 { return }
 
+    #if false
+
+
+        let backgroundTaskGroup = await withTaskGroup(of: Void.self) { group in
+            for file in files {
+                if file.fileType == .Build {
+                    _ = group.addTaskUnlessCancelled {
+                        guard Task.isCancelled == false else { return }
+
+                        do {
+                            try await updateBuildTimingTask(file)
+                        }
+                        catch {
+                            log.error(error.localizedDescription)
+                        }
+
+                    }
+                }
+            }
+        }
+
+    #else
+
     let _ = Task(priority: .medium, operation: {
         let timer = Timer()
 
         for file in files {
+
             if file.fileType == .Build {
                 do {
                     try updateBuildTimingTask(file)
@@ -1539,9 +1563,10 @@ func updateBuildTimingsTask(_ files: [File]) {
         timer.stop()
         log.info("finished updating build timings in \(double:timer.timeElapsed(), decimals:3)s")
     })
+    #endif
 }
 
-func updateBuildTimingTask(_ file: File) throws {
+func updateBuildTimingTask(_ file: File) /*async */ throws {
     assert(file.fileType == .Build)
 
     // only checking this, and not duration == 0
@@ -1751,6 +1776,9 @@ func generateStatsForTotalTrack(_ events: [PerfettoEvent]) -> BuildStats {
         else if event.name == "Total CodeGen Function" {
             stats.totalCodeGenFunction = event.dur!
         }
+        else if event.name == "Total DebugType" {
+            stats.totalDebugType = event.dur!
+        }
 
         // backend
         else if event.name == "Total Backend" {
@@ -1829,7 +1857,6 @@ func convertStatsToTotalTrack(_ stats: BuildStats) -> [PerfettoEvent] {
     event.ts = stats.frontendStart + stats.totalSource
     totalEvents.append(event)
 
-
     // This total can exceed when backend start, so clamp it too
     let tsCodeGenFunction = stats.frontendStart + stats.totalSource + stats.totalInstantiateFunction
 
@@ -1842,6 +1869,19 @@ func convertStatsToTotalTrack(_ stats: BuildStats) -> [PerfettoEvent] {
     event.ts = tsCodeGenFunction
     totalEvents.append(event)
 
+
+    // can gen a lot of debug types, and clamp to backend
+    let tsDebugType = tsCodeGenFunction + totalCodeGenFunction
+
+    var totalDebugType = stats.totalDebugType
+    if totalDebugType + totalDebugType > stats.backendStart {
+        totalDebugType = stats.backendStart - tsDebugType
+    }
+
+    event = makeDurEvent(tid, "Debug", totalDebugType, total)
+    event.ts = tsDebugType
+    totalEvents.append(event)
+
     // backend
     event = makeDurEvent(tid, "Backend", stats.totalBackend, total)
     event.ts = stats.backendStart