From 2aa40c7cbb14f2b710138833e6e7db307a72a24c Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 17 Dec 2020 03:43:41 -0800 Subject: [PATCH 01/11] Add initial support -fcoverage-mapping support This is enought make it work up until llvm-cov tries to read the named data sections in the binary and can't find them. For this final part to work we probably need to switch the object format to using multiple code and data sections: https://github.com/WebAssembly/tool-conventions/issues/138 Not sure if its worth submitting this part in isolation without a fully working solution? See #13046 --- emcc.py | 1 - .../include/profile/InstrProfData.inc | 754 ++++++++++++ .../compiler-rt/lib/profile/GCDAProfiling.c | 656 +++++++++++ .../compiler-rt/lib/profile/InstrProfiling.c | 84 ++ .../compiler-rt/lib/profile/InstrProfiling.h | 310 +++++ .../lib/profile/InstrProfilingBuffer.c | 129 +++ .../lib/profile/InstrProfilingFile.c | 1029 +++++++++++++++++ .../lib/profile/InstrProfilingInternal.h | 196 ++++ .../lib/profile/InstrProfilingMerge.c | 132 +++ .../lib/profile/InstrProfilingMergeFile.c | 45 + .../lib/profile/InstrProfilingNameVar.c | 17 + .../profile/InstrProfilingPlatformDarwin.c | 67 ++ .../profile/InstrProfilingPlatformFuchsia.c | 197 ++++ .../lib/profile/InstrProfilingPlatformLinux.c | 83 ++ .../lib/profile/InstrProfilingPlatformOther.c | 100 ++ .../profile/InstrProfilingPlatformWindows.c | 68 ++ .../lib/profile/InstrProfilingPort.h | 144 +++ .../lib/profile/InstrProfilingRuntime.cpp | 30 + .../lib/profile/InstrProfilingUtil.c | 332 ++++++ .../lib/profile/InstrProfilingUtil.h | 74 ++ .../lib/profile/InstrProfilingValue.c | 371 ++++++ .../lib/profile/InstrProfilingWriter.c | 300 +++++ system/lib/update_compiler_rt.py | 1 + tests/test_core.py | 14 +- tools/shared.py | 3 + tools/system_libs.py | 7 +- 26 files changed, 5141 insertions(+), 3 deletions(-) create mode 100644 system/lib/compiler-rt/include/profile/InstrProfData.inc create mode 100644 system/lib/compiler-rt/lib/profile/GCDAProfiling.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfiling.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfiling.h create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingFile.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPort.h create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingValue.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c diff --git a/emcc.py b/emcc.py index 8d6f8f4dd87c9..2925b3ed79715 100755 --- a/emcc.py +++ b/emcc.py @@ -1374,7 +1374,6 @@ def filter_out_duplicate_dynamic_libs(inputs): if not compile_only and not options.post_link: ldflags = shared.emsdk_ldflags(newargs) for f in ldflags: - newargs.append(f) add_link_flag(len(newargs), f) # SSEx is implemented on top of SIMD128 instruction set, but do not pass SSE flags to LLVM diff --git a/system/lib/compiler-rt/include/profile/InstrProfData.inc b/system/lib/compiler-rt/include/profile/InstrProfData.inc new file mode 100644 index 0000000000000..99f41d8fef077 --- /dev/null +++ b/system/lib/compiler-rt/include/profile/InstrProfData.inc @@ -0,0 +1,754 @@ +/*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the master file that defines all the data structure, signature, + * constant literals that are shared across profiling runtime library, + * compiler (instrumentation), and host tools (reader/writer). The entities + * defined in this file affect the profile runtime ABI, the raw profile format, + * or both. + * + * The file has two identical copies. The master copy lives in LLVM and + * the other one sits in compiler-rt/lib/profile directory. To make changes + * in this file, first modify the master copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * + * The first part of the file includes macros that defines types, names, and + * initializers for the member fields of the core data structures. The field + * declarations for one structure is enabled by defining the field activation + * macro associated with that structure. Only one field activation record + * can be defined at one time and the rest definitions will be filtered out by + * the preprocessor. + * + * Examples of how the template is used to instantiate structure definition: + * 1. To declare a structure: + * + * struct ProfData { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Type Name; + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 2. To construct LLVM type arrays for the struct type: + * + * Type *DataTypes[] = { + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * LLVMType, + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * 4. To construct constant array for the initializers: + * #define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) \ + * Initializer, + * Constant *ConstantVals[] = { + * #include "llvm/ProfileData/InstrProfData.inc" + * }; + * + * + * The second part of the file includes definitions all other entities that + * are related to runtime ABI and format. When no field activation macro is + * defined, this file can be included to introduce the definitions. + * +\*===----------------------------------------------------------------------===*/ + +/* Functions marked with INSTR_PROF_VISIBILITY must have hidden visibility in + * the compiler runtime. */ +#ifndef INSTR_PROF_VISIBILITY +#define INSTR_PROF_VISIBILITY +#endif + +/* INSTR_PROF_DATA start. */ +/* Definition of member fields of the per-function control structure. */ +#ifndef INSTR_PROF_DATA +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(getPGOFuncNameVarInitializer(Inc->getName())))) +INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + Inc->getHash()->getZExtValue())) +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ + ConstantExpr::getBitCast(CounterPtr, \ + llvm::Type::getInt64PtrTy(Ctx))) +/* This is used to map function pointers for the indirect call targets to + * function name hashes during the conversion from raw to merged profile + * data. + */ +INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ + FunctionAddr) +INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ + ValuesPtrExpr) +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) +INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) +#undef INSTR_PROF_DATA +/* INSTR_PROF_DATA end. */ + + +/* This is an internal data structure used by value profiler. It + * is defined here to allow serialization code sharing by LLVM + * to be used in unit test. + * + * typedef struct ValueProfNode { + * // InstrProfValueData VData; + * uint64_t Value; + * uint64_t Count; + * struct ValueProfNode *Next; + * } ValueProfNode; + */ +/* INSTR_PROF_VALUE_NODE start. */ +#ifndef INSTR_PROF_VALUE_NODE +#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ + ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) +INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ + ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) +INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ + ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) +#undef INSTR_PROF_VALUE_NODE +/* INSTR_PROF_VALUE_NODE end. */ + +/* INSTR_PROF_RAW_HEADER start */ +/* Definition of member fields of the raw profile header data structure. */ +#ifndef INSTR_PROF_RAW_HEADER +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) +INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) +INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) +INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) +INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) +#undef INSTR_PROF_RAW_HEADER +/* INSTR_PROF_RAW_HEADER end */ + +/* VALUE_PROF_FUNC_PARAM start */ +/* Definition of parameter types of the runtime API used to do value profiling + * for a given value site. + */ +#ifndef VALUE_PROF_FUNC_PARAM +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) +#define INSTR_PROF_COMMA +#else +#define INSTR_PROF_DATA_DEFINED +#define INSTR_PROF_COMMA , +#endif +VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA +#ifndef VALUE_RANGE_PROF +VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) +#else /* VALUE_RANGE_PROF */ +VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ + INSTR_PROF_COMMA +VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) +#endif /*VALUE_RANGE_PROF */ +#undef VALUE_PROF_FUNC_PARAM +#undef INSTR_PROF_COMMA +/* VALUE_PROF_FUNC_PARAM end */ + +/* VALUE_PROF_KIND start */ +#ifndef VALUE_PROF_KIND +#define VALUE_PROF_KIND(Enumerator, Value, Descr) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +/* For indirect function call value profiling, the addresses of the target + * functions are profiled by the instrumented code. The target addresses are + * written in the raw profile data and converted to target function name's MD5 + * hash by the profile reader during deserialization. Typically, this happens + * when the raw profile data is read during profile merging. + * + * For this remapping the ProfData is used. ProfData contains both the function + * name hash and the function address. + */ +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") +/* For memory intrinsic functions size profiling. */ +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +/* These two kinds must be the last to be + * declared. This is to make sure the string + * array created with the template can be + * indexed with the kind value. + */ +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") + +#undef VALUE_PROF_KIND +/* VALUE_PROF_KIND end */ + +/* COVMAP_FUNC_RECORD start */ +/* Definition of member fields of the function record structure in coverage + * map. + */ +#ifndef COVMAP_FUNC_RECORD +#define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +#ifdef COVMAP_V1 +COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ + NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ + llvm::Type::getInt8PtrTy(Ctx))) +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + NameValue.size())) +#else +COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ + llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + llvm::IndexedInstrProf::ComputeHash(NameValue))) +#endif +COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ + llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ + CoverageMapping.size())) +COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ + llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) +#undef COVMAP_FUNC_RECORD +/* COVMAP_FUNC_RECORD end. */ + +/* COVMAP_HEADER start */ +/* Definition of member fields of coverage map header. + */ +#ifndef COVMAP_HEADER +#define COVMAP_HEADER(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ + llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) +COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ + llvm::ConstantInt::get(Int32Ty, FilenamesSize)) +COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ + llvm::ConstantInt::get(Int32Ty, CoverageMappingSize)) +COVMAP_HEADER(uint32_t, Int32Ty, Version, \ + llvm::ConstantInt::get(Int32Ty, CovMapVersion::CurrentVersion)) +#undef COVMAP_HEADER +/* COVMAP_HEADER end. */ + + +#ifdef INSTR_PROF_SECT_ENTRY +#define INSTR_PROF_DATA_DEFINED +INSTR_PROF_SECT_ENTRY(IPSK_data, \ + INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ + INSTR_PROF_DATA_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ + INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ + INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_name, \ + INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ + INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vals, \ + INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ + INSTR_PROF_VALS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ + INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") + +#undef INSTR_PROF_SECT_ENTRY +#endif + + +#ifdef INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_DATA_DEFINED + +#define INSTR_PROF_MAX_NUM_VAL_PER_SITE 255 +/*! + * This is the header of the data structure that defines the on-disk + * layout of the value profile data of a particular kind for one function. + */ +typedef struct ValueProfRecord { + /* The kind of the value profile record. */ + uint32_t Kind; + /* + * The number of value profile sites. It is guaranteed to be non-zero; + * otherwise the record for this kind won't be emitted. + */ + uint32_t NumValueSites; + /* + * The first element of the array that stores the number of profiled + * values for each value site. The size of the array is NumValueSites. + * Since NumValueSites is greater than zero, there is at least one + * element in the array. + */ + uint8_t SiteCountArray[1]; + + /* + * The fake declaration is for documentation purpose only. + * Align the start of next field to be on 8 byte boundaries. + uint8_t Padding[X]; + */ + + /* The array of value profile data. The size of the array is the sum + * of all elements in SiteCountArray[]. + InstrProfValueData ValueData[]; + */ + +#ifdef __cplusplus + /*! + * Return the number of value sites. + */ + uint32_t getNumValueSites() const { return NumValueSites; } + /*! + * Read data from this record and save it to Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfSymtab *SymTab); + /* + * In-place byte swap: + * Do byte swap for this instance. \c Old is the original order before + * the swap, and \c New is the New byte order. + */ + void swapBytes(support::endianness Old, support::endianness New); +#endif +} ValueProfRecord; + +/*! + * Per-function header/control data structure for value profiling + * data in indexed format. + */ +typedef struct ValueProfData { + /* + * Total size in bytes including this field. It must be a multiple + * of sizeof(uint64_t). + */ + uint32_t TotalSize; + /* + *The number of value profile kinds that has value profile data. + * In this implementation, a value profile kind is considered to + * have profile data if the number of value profile sites for the + * kind is not zero. More aggressively, the implementation can + * choose to check the actual data value: if none of the value sites + * has any profiled values, the kind can be skipped. + */ + uint32_t NumValueKinds; + + /* + * Following are a sequence of variable length records. The prefix/header + * of each record is defined by ValueProfRecord type. The number of + * records is NumValueKinds. + * ValueProfRecord Record_1; + * ValueProfRecord Record_N; + */ + +#if __cplusplus + /*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ + static uint32_t getSize(const InstrProfRecord &Record); + /*! + * Return a pointer to \c ValueProfData instance ready to be streamed. + */ + static std::unique_ptr + serializeFrom(const InstrProfRecord &Record); + /*! + * Check the integrity of the record. + */ + Error checkIntegrity(); + /*! + * Return a pointer to \c ValueProfileData instance ready to be read. + * All data in the instance are properly byte swapped. The input + * data is assumed to be in little endian order. + */ + static Expected> + getValueProfData(const unsigned char *SrcBuffer, + const unsigned char *const SrcBufferEnd, + support::endianness SrcDataEndianness); + /*! + * Swap byte order from \c Endianness order to host byte order. + */ + void swapBytesToHost(support::endianness Endianness); + /*! + * Swap byte order from host byte order to \c Endianness order. + */ + void swapBytesFromHost(support::endianness Endianness); + /*! + * Return the total size of \c ValueProfileData. + */ + uint32_t getSize() const { return TotalSize; } + /*! + * Read data from this data and save it to \c Record. + */ + void deserializeTo(InstrProfRecord &Record, + InstrProfSymtab *SymTab); + void operator delete(void *ptr) { ::operator delete(ptr); } +#endif +} ValueProfData; + +/* + * The closure is designed to abstact away two types of value profile data: + * - InstrProfRecord which is the primary data structure used to + * represent profile data in host tools (reader, writer, and profile-use) + * - value profile runtime data structure suitable to be used by C + * runtime library. + * + * Both sources of data need to serialize to disk/memory-buffer in common + * format: ValueProfData. The abstraction allows compiler-rt's raw profiler + * writer to share the same format and code with indexed profile writer. + * + * For documentation of the member methods below, refer to corresponding methods + * in class InstrProfRecord. + */ +typedef struct ValueProfRecordClosure { + const void *Record; + uint32_t (*GetNumValueKinds)(const void *Record); + uint32_t (*GetNumValueSites)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueData)(const void *Record, uint32_t VKind); + uint32_t (*GetNumValueDataForSite)(const void *R, uint32_t VK, uint32_t S); + + /* + * After extracting the value profile data from the value profile record, + * this method is used to map the in-memory value to on-disk value. If + * the method is null, value will be written out untranslated. + */ + uint64_t (*RemapValueData)(uint32_t, uint64_t Value); + void (*GetValueForSite)(const void *R, InstrProfValueData *Dst, uint32_t K, + uint32_t S); + ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); +} ValueProfRecordClosure; + +INSTR_PROF_VISIBILITY ValueProfRecord * +getFirstValueProfRecord(ValueProfData *VPD); +INSTR_PROF_VISIBILITY ValueProfRecord * +getValueProfRecordNext(ValueProfRecord *VPR); +INSTR_PROF_VISIBILITY InstrProfValueData * +getValueProfRecordValueData(ValueProfRecord *VPR); +INSTR_PROF_VISIBILITY uint32_t +getValueProfRecordHeaderSize(uint32_t NumValueSites); + +#undef INSTR_PROF_VALUE_PROF_DATA +#endif /* INSTR_PROF_VALUE_PROF_DATA */ + + +#ifdef INSTR_PROF_COMMON_API_IMPL +#define INSTR_PROF_DATA_DEFINED +#ifdef __cplusplus +#define INSTR_PROF_INLINE inline +#define INSTR_PROF_NULLPTR nullptr +#else +#define INSTR_PROF_INLINE +#define INSTR_PROF_NULLPTR NULL +#endif + +#ifndef offsetof +#define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) +#endif + +/*! + * Return the \c ValueProfRecord header size including the + * padding bytes. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { + uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + + sizeof(uint8_t) * NumValueSites; + /* Round the size to multiple of 8 bytes. */ + Size = (Size + 7) & ~7; + return Size; +} + +/*! + * Return the total size of the value profile record including the + * header and the value data. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +uint32_t getValueProfRecordSize(uint32_t NumValueSites, + uint32_t NumValueData) { + return getValueProfRecordHeaderSize(NumValueSites) + + sizeof(InstrProfValueData) * NumValueData; +} + +/*! + * Return the pointer to the start of value data array. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { + return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( + This->NumValueSites)); +} + +/*! + * Return the total number of value data for \c This record. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { + uint32_t NumValueData = 0; + uint32_t I; + for (I = 0; I < This->NumValueSites; I++) + NumValueData += This->SiteCountArray[I]; + return NumValueData; +} + +/*! + * Use this method to advance to the next \c This \c ValueProfRecord. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { + uint32_t NumValueData = getValueProfRecordNumValueData(This); + return (ValueProfRecord *)((char *)This + + getValueProfRecordSize(This->NumValueSites, + NumValueData)); +} + +/*! + * Return the first \c ValueProfRecord instance. + */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { + return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); +} + +/* Closure based interfaces. */ + +/*! + * Return the total size in bytes of the on-disk value profile data + * given the data stored in Record. + */ +INSTR_PROF_VISIBILITY uint32_t +getValueProfDataSize(ValueProfRecordClosure *Closure) { + uint32_t Kind; + uint32_t TotalSize = sizeof(ValueProfData); + const void *Record = Closure->Record; + + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); + if (!NumValueSites) + continue; + TotalSize += getValueProfRecordSize(NumValueSites, + Closure->GetNumValueData(Record, Kind)); + } + return TotalSize; +} + +/*! + * Extract value profile data of a function for the profile kind \c ValueKind + * from the \c Closure and serialize the data into \c This record instance. + */ +INSTR_PROF_VISIBILITY void +serializeValueProfRecordFrom(ValueProfRecord *This, + ValueProfRecordClosure *Closure, + uint32_t ValueKind, uint32_t NumValueSites) { + uint32_t S; + const void *Record = Closure->Record; + This->Kind = ValueKind; + This->NumValueSites = NumValueSites; + InstrProfValueData *DstVD = getValueProfRecordValueData(This); + + for (S = 0; S < NumValueSites; S++) { + uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); + This->SiteCountArray[S] = ND; + Closure->GetValueForSite(Record, DstVD, ValueKind, S); + DstVD += ND; + } +} + +/*! + * Extract value profile data of a function from the \c Closure + * and serialize the data into \c DstData if it is not NULL or heap + * memory allocated by the \c Closure's allocator method. If \c + * DstData is not null, the caller is expected to set the TotalSize + * in DstData. + */ +INSTR_PROF_VISIBILITY ValueProfData * +serializeValueProfDataFrom(ValueProfRecordClosure *Closure, + ValueProfData *DstData) { + uint32_t Kind; + uint32_t TotalSize = + DstData ? DstData->TotalSize : getValueProfDataSize(Closure); + + ValueProfData *VPD = + DstData ? DstData : Closure->AllocValueProfData(TotalSize); + + VPD->TotalSize = TotalSize; + VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); + ValueProfRecord *VR = getFirstValueProfRecord(VPD); + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); + if (!NumValueSites) + continue; + serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); + VR = getValueProfRecordNext(VR); + } + return VPD; +} + +#undef INSTR_PROF_COMMON_API_IMPL +#endif /* INSTR_PROF_COMMON_API_IMPL */ + +/*============================================================================*/ + +#ifndef INSTR_PROF_DATA_DEFINED + +#ifndef INSTR_PROF_DATA_INC +#define INSTR_PROF_DATA_INC + +/* Helper macros. */ +#define INSTR_PROF_SIMPLE_QUOTE(x) #x +#define INSTR_PROF_QUOTE(x) INSTR_PROF_SIMPLE_QUOTE(x) +#define INSTR_PROF_SIMPLE_CONCAT(x,y) x ## y +#define INSTR_PROF_CONCAT(x,y) INSTR_PROF_SIMPLE_CONCAT(x,y) + +/* Magic number to detect file format and endianness. + * Use 255 at one end, since no UTF-8 file can use that character. Avoid 0, + * so that utilities, like strings, don't grab it as a string. 129 is also + * invalid UTF-8, and high enough to be interesting. + * Use "lprofr" in the centre to stand for "LLVM Profile Raw", or "lprofR" + * for 32-bit platforms. + */ +#define INSTR_PROF_RAW_MAGIC_64 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129 +#define INSTR_PROF_RAW_MAGIC_32 (uint64_t)255 << 56 | (uint64_t)'l' << 48 | \ + (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | (uint64_t)'o' << 24 | \ + (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 + +/* Raw profile format version (start from 1). */ +#define INSTR_PROF_RAW_VERSION 5 +/* Indexed profile format version (start from 1). */ +#define INSTR_PROF_INDEX_VERSION 5 +/* Coverage mapping format vresion (start from 0). */ +#define INSTR_PROF_COVMAP_VERSION 2 + +/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the + * version for other variants of profile. We set the lowest bit of the upper 8 + * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton + * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. + */ +#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) +#define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version +#define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime + +/* The variable that holds the name of the profile data + * specified via command line. */ +#define INSTR_PROF_PROFILE_NAME_VAR __llvm_profile_filename + +/* section name strings common to all targets other + than WIN32 */ +#define INSTR_PROF_DATA_COMMON __llvm_prf_data +#define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_VALS_COMMON __llvm_prf_vals +#define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_COVMAP_COMMON __llvm_covmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" + +#ifdef _WIN32 +/* Runtime section names and name strings. */ +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +/* Array of pointers. Each pointer points to a list + * of value nodes associated with one value site. + */ +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COFF +/* Value profile nodes section. */ +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF +#else +/* Runtime section names and name strings. */ +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +/* Array of pointers. Each pointer points to a list + * of value nodes associated with one value site. + */ +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) +/* Value profile nodes section. */ +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) +#endif + +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) + +/* Macros to define start/stop section symbol for a given + * section on Linux. For instance + * INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) will + * expand to __start___llvm_prof_data + */ +#define INSTR_PROF_SECT_START(Sect) \ + INSTR_PROF_CONCAT(__start_,Sect) +#define INSTR_PROF_SECT_STOP(Sect) \ + INSTR_PROF_CONCAT(__stop_,Sect) + +/* Value Profiling API linkage name. */ +#define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target +#define INSTR_PROF_VALUE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range +#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) + +/* InstrProfile per-function control data alignment. */ +#define INSTR_PROF_DATA_ALIGNMENT 8 + +/* The data structure that represents a tracked value by the + * value profiler. + */ +typedef struct InstrProfValueData { + /* Profiled value. */ + uint64_t Value; + /* Number of times the value appears in the training run. */ + uint64_t Count; +} InstrProfValueData; + +#endif /* INSTR_PROF_DATA_INC */ + +#ifndef INSTR_ORDER_FILE_INC +/* The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). */ +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ +#else +#undef INSTR_PROF_DATA_DEFINED +#endif diff --git a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c new file mode 100644 index 0000000000000..498c05900bf27 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c @@ -0,0 +1,656 @@ +/*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------===*| +|* +|* This file implements the call back routines for the gcov profiling +|* instrumentation pass. Link against this library when running code through +|* the -insert-gcov-profiling LLVM pass. +|* +|* We emit files in a corrupt version of GCOV's "gcda" file format. These files +|* are only close enough that LCOV will happily parse them. Anything that lcov +|* ignores is missing. +|* +|* TODO: gcov is multi-process safe by having each exit open the existing file +|* and append to it. We'd like to achieve that and be thread-safe too. +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include +#include +#include +#include +#include + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#include +#include "WindowsMMap.h" +#else +#include +#include +#endif + +#if defined(__FreeBSD__) && defined(__i386__) +#define I386_FREEBSD 1 +#else +#define I386_FREEBSD 0 +#endif + +#if !defined(_MSC_VER) && !I386_FREEBSD +#include +#endif + +#if defined(_MSC_VER) +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +#elif I386_FREEBSD +/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to + * FreeBSD 10, r232261) when compiled in 32-bit mode. + */ +typedef unsigned char uint8_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + +/* #define DEBUG_GCDAPROFILING */ + +/* + * --- GCOV file format I/O primitives --- + */ + +/* + * The current file name we're outputting. Used primarily for error logging. + */ +static char *filename = NULL; + +/* + * The current file we're outputting. + */ +static FILE *output_file = NULL; + +/* + * Buffer that we write things into. + */ +#define WRITE_BUFFER_SIZE (128 * 1024) +static unsigned char *write_buffer = NULL; +static uint64_t cur_buffer_size = 0; +static uint64_t cur_pos = 0; +static uint64_t file_size = 0; +static int new_file = 0; +#if defined(_WIN32) +static HANDLE mmap_handle = NULL; +#endif +static int fd = -1; + +typedef void (*fn_ptr)(); + +typedef void* dynamic_object_id; +// The address of this variable identifies a given dynamic object. +static dynamic_object_id current_id; +#define CURRENT_ID (¤t_id) + +struct fn_node { + dynamic_object_id id; + fn_ptr fn; + struct fn_node* next; +}; + +struct fn_list { + struct fn_node *head, *tail; +}; + +/* + * A list of functions to write out the data, shared between all dynamic objects. + */ +struct fn_list writeout_fn_list; + +/* + * A list of flush functions that our __gcov_flush() function should call, shared between all dynamic objects. + */ +struct fn_list flush_fn_list; + +static void fn_list_insert(struct fn_list* list, fn_ptr fn) { + struct fn_node* new_node = malloc(sizeof(struct fn_node)); + new_node->fn = fn; + new_node->next = NULL; + new_node->id = CURRENT_ID; + + if (!list->head) { + list->head = list->tail = new_node; + } else { + list->tail->next = new_node; + list->tail = new_node; + } +} + +static void fn_list_remove(struct fn_list* list) { + struct fn_node* curr = list->head; + struct fn_node* prev = NULL; + struct fn_node* next = NULL; + + while (curr) { + next = curr->next; + + if (curr->id == CURRENT_ID) { + if (curr == list->head) { + list->head = next; + } + + if (curr == list->tail) { + list->tail = prev; + } + + if (prev) { + prev->next = next; + } + + free(curr); + } else { + prev = curr; + } + + curr = next; + } +} + +static void resize_write_buffer(uint64_t size) { + if (!new_file) return; + size += cur_pos; + if (size <= cur_buffer_size) return; + size = (size - 1) / WRITE_BUFFER_SIZE + 1; + size *= WRITE_BUFFER_SIZE; + write_buffer = realloc(write_buffer, size); + cur_buffer_size = size; +} + +static void write_bytes(const char *s, size_t len) { + resize_write_buffer(len); + memcpy(&write_buffer[cur_pos], s, len); + cur_pos += len; +} + +static void write_32bit_value(uint32_t i) { + write_bytes((char*)&i, 4); +} + +static void write_64bit_value(uint64_t i) { + // GCOV uses a lo-/hi-word format even on big-endian systems. + // See also GCOVBuffer::readInt64 in LLVM. + uint32_t lo = (uint32_t) i; + uint32_t hi = (uint32_t) (i >> 32); + write_32bit_value(lo); + write_32bit_value(hi); +} + +static uint32_t length_of_string(const char *s) { + return (strlen(s) / 4) + 1; +} + +static void write_string(const char *s) { + uint32_t len = length_of_string(s); + write_32bit_value(len); + write_bytes(s, strlen(s)); + write_bytes("\0\0\0\0", 4 - (strlen(s) % 4)); +} + +static uint32_t read_32bit_value() { + uint32_t val; + + if (new_file) + return (uint32_t)-1; + + val = *(uint32_t*)&write_buffer[cur_pos]; + cur_pos += 4; + return val; +} + +static uint32_t read_le_32bit_value() { + uint32_t val = 0; + int i; + + if (new_file) + return (uint32_t)-1; + + for (i = 0; i < 4; i++) + val |= write_buffer[cur_pos++] << (8*i); + return val; +} + +static uint64_t read_64bit_value() { + // GCOV uses a lo-/hi-word format even on big-endian systems. + // See also GCOVBuffer::readInt64 in LLVM. + uint32_t lo = read_32bit_value(); + uint32_t hi = read_32bit_value(); + return ((uint64_t)hi << 32) | ((uint64_t)lo); +} + +static char *mangle_filename(const char *orig_filename) { + char *new_filename; + size_t prefix_len; + int prefix_strip; + const char *prefix = lprofGetPathPrefix(&prefix_strip, &prefix_len); + + if (prefix == NULL) + return strdup(orig_filename); + + new_filename = malloc(prefix_len + 1 + strlen(orig_filename) + 1); + lprofApplyPathPrefix(new_filename, orig_filename, prefix, prefix_len, + prefix_strip); + + return new_filename; +} + +static int map_file() { + fseek(output_file, 0L, SEEK_END); + file_size = ftell(output_file); + + /* A size of 0 is invalid to `mmap'. Return a fail here, but don't issue an + * error message because it should "just work" for the user. */ + if (file_size == 0) + return -1; + +#if defined(_WIN32) + HANDLE mmap_fd; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + + mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); + if (mmap_handle == NULL) { + fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n", + filename, GetLastError()); + return -1; + } + + write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); + if (write_buffer == NULL) { + fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename, + GetLastError()); + CloseHandle(mmap_handle); + return -1; + } +#else + write_buffer = mmap(0, file_size, PROT_READ | PROT_WRITE, + MAP_FILE | MAP_SHARED, fd, 0); + if (write_buffer == (void *)-1) { + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot map: %s\n", filename, + strerror(errnum)); + return -1; + } +#endif + + return 0; +} + +static void unmap_file() { +#if defined(_WIN32) + if (!FlushViewOfFile(write_buffer, file_size)) { + fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename, + GetLastError()); + } + + if (!UnmapViewOfFile(write_buffer)) { + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, + GetLastError()); + } + + if (!CloseHandle(mmap_handle)) { + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n", + filename, GetLastError()); + } + + mmap_handle = NULL; +#else + if (msync(write_buffer, file_size, MS_SYNC) == -1) { + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename, + strerror(errnum)); + } + + /* We explicitly ignore errors from unmapping because at this point the data + * is written and we don't care. + */ + (void)munmap(write_buffer, file_size); +#endif + + write_buffer = NULL; + file_size = 0; +} + +/* + * --- LLVM line counter API --- + */ + +/* A file in this case is a translation unit. Each .o file built with line + * profiling enabled will emit to a different file. Only one file may be + * started at a time. + */ +COMPILER_RT_VISIBILITY +void llvm_gcda_start_file(const char *orig_filename, const char version[4], + uint32_t checksum) { + const char *mode = "r+b"; + filename = mangle_filename(orig_filename); + + /* Try just opening the file. */ + new_file = 0; + fd = open(filename, O_RDWR | O_BINARY); + + if (fd == -1) { + /* Try opening the file, creating it if necessary. */ + new_file = 1; + mode = "w+b"; + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); + if (fd == -1) { + /* Try creating the directories first then opening the file. */ + __llvm_profile_recursive_mkdir(filename); + fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); + if (fd == -1) { + /* Bah! It's hopeless. */ + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot open: %s\n", filename, + strerror(errnum)); + return; + } + } + } + + /* Try to flock the file to serialize concurrent processes writing out to the + * same GCDA. This can fail if the filesystem doesn't support it, but in that + * case we'll just carry on with the old racy behaviour and hope for the best. + */ + lprofLockFd(fd); + output_file = fdopen(fd, mode); + + /* Initialize the write buffer. */ + write_buffer = NULL; + cur_buffer_size = 0; + cur_pos = 0; + + if (new_file) { + resize_write_buffer(WRITE_BUFFER_SIZE); + memset(write_buffer, 0, WRITE_BUFFER_SIZE); + } else { + if (map_file() == -1) { + /* mmap failed, try to recover by clobbering */ + new_file = 1; + write_buffer = NULL; + cur_buffer_size = 0; + resize_write_buffer(WRITE_BUFFER_SIZE); + memset(write_buffer, 0, WRITE_BUFFER_SIZE); + } + } + + /* gcda file, version, stamp checksum. */ + write_bytes("adcg", 4); + write_bytes(version, 4); + write_32bit_value(checksum); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: [%s]\n", orig_filename); +#endif +} + +/* Given an array of pointers to counters (counters), increment the n-th one, + * where we're also given a pointer to n (predecessor). + */ +COMPILER_RT_VISIBILITY +void llvm_gcda_increment_indirect_counter(uint32_t *predecessor, + uint64_t **counters) { + uint64_t *counter; + uint32_t pred; + + pred = *predecessor; + if (pred == 0xffffffff) + return; + counter = counters[pred]; + + /* Don't crash if the pred# is out of sync. This can happen due to threads, + or because of a TODO in GCOVProfiling.cpp buildEdgeLookupTable(). */ + if (counter) + ++*counter; +#ifdef DEBUG_GCDAPROFILING + else + fprintf(stderr, + "llvmgcda: increment_indirect_counter counters=%08llx, pred=%u\n", + *counter, *predecessor); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_emit_function(uint32_t ident, const char *function_name, + uint32_t func_checksum, uint8_t use_extra_checksum, + uint32_t cfg_checksum) { + uint32_t len = 2; + + if (use_extra_checksum) + len++; +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: function id=0x%08x name=%s\n", ident, + function_name ? function_name : "NULL"); +#endif + if (!output_file) return; + + /* function tag */ + write_bytes("\0\0\0\1", 4); + if (function_name) + len += 1 + length_of_string(function_name); + write_32bit_value(len); + write_32bit_value(ident); + write_32bit_value(func_checksum); + if (use_extra_checksum) + write_32bit_value(cfg_checksum); + if (function_name) + write_string(function_name); +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { + uint32_t i; + uint64_t *old_ctrs = NULL; + uint32_t val = 0; + uint64_t save_cur_pos = cur_pos; + + if (!output_file) return; + + val = read_le_32bit_value(); + + if (val != (uint32_t)-1) { + /* There are counters present in the file. Merge them. */ + if (val != 0x01a10000) { + fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: " + "corrupt arc tag (0x%08x)\n", + filename, val); + return; + } + + val = read_32bit_value(); + if (val == (uint32_t)-1 || val / 2 != num_counters) { + fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: " + "mismatched number of counters (%d)\n", + filename, val); + return; + } + + old_ctrs = malloc(sizeof(uint64_t) * num_counters); + for (i = 0; i < num_counters; ++i) + old_ctrs[i] = read_64bit_value(); + } + + cur_pos = save_cur_pos; + + /* Counter #1 (arcs) tag */ + write_bytes("\0\0\xa1\1", 4); + write_32bit_value(num_counters * 2); + for (i = 0; i < num_counters; ++i) { + counters[i] += (old_ctrs ? old_ctrs[i] : 0); + write_64bit_value(counters[i]); + } + + free(old_ctrs); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: %u arcs\n", num_counters); + for (i = 0; i < num_counters; ++i) + fprintf(stderr, "llvmgcda: %llu\n", (unsigned long long)counters[i]); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_summary_info() { + const uint32_t obj_summary_len = 9; /* Length for gcov compatibility. */ + uint32_t i; + uint32_t runs = 1; + static uint32_t run_counted = 0; // We only want to increase the run count once. + uint32_t val = 0; + uint64_t save_cur_pos = cur_pos; + + if (!output_file) return; + + val = read_le_32bit_value(); + + if (val != (uint32_t)-1) { + /* There are counters present in the file. Merge them. */ + if (val != 0xa1000000) { + fprintf(stderr, "profiling: %s: cannot merge previous run count: " + "corrupt object tag (0x%08x)\n", + filename, val); + return; + } + + val = read_32bit_value(); /* length */ + if (val != obj_summary_len) { + fprintf(stderr, "profiling: %s: cannot merge previous run count: " + "mismatched object length (%d)\n", + filename, val); + return; + } + + read_32bit_value(); /* checksum, unused */ + read_32bit_value(); /* num, unused */ + uint32_t prev_runs = read_32bit_value(); + /* Add previous run count to new counter, if not already counted before. */ + runs = run_counted ? prev_runs : prev_runs + 1; + } + + cur_pos = save_cur_pos; + + /* Object summary tag */ + write_bytes("\0\0\0\xa1", 4); + write_32bit_value(obj_summary_len); + write_32bit_value(0); /* checksum, unused */ + write_32bit_value(0); /* num, unused */ + write_32bit_value(runs); + for (i = 3; i < obj_summary_len; ++i) + write_32bit_value(0); + + /* Program summary tag */ + write_bytes("\0\0\0\xa3", 4); /* tag indicates 1 program */ + write_32bit_value(0); /* 0 length */ + + run_counted = 1; + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: %u runs\n", runs); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_gcda_end_file() { + /* Write out EOF record. */ + if (output_file) { + write_bytes("\0\0\0\0\0\0\0\0", 8); + + if (new_file) { + fwrite(write_buffer, cur_pos, 1, output_file); + free(write_buffer); + } else { + unmap_file(); + } + + fflush(output_file); + lprofUnlockFd(fd); + fclose(output_file); + output_file = NULL; + write_buffer = NULL; + } + free(filename); + +#ifdef DEBUG_GCDAPROFILING + fprintf(stderr, "llvmgcda: -----\n"); +#endif +} + +COMPILER_RT_VISIBILITY +void llvm_register_writeout_function(fn_ptr fn) { + fn_list_insert(&writeout_fn_list, fn); +} + +COMPILER_RT_VISIBILITY +void llvm_writeout_files(void) { + struct fn_node *curr = writeout_fn_list.head; + + while (curr) { + if (curr->id == CURRENT_ID) { + curr->fn(); + } + curr = curr->next; + } +} + +COMPILER_RT_VISIBILITY +void llvm_delete_writeout_function_list(void) { + fn_list_remove(&writeout_fn_list); +} + +COMPILER_RT_VISIBILITY +void llvm_register_flush_function(fn_ptr fn) { + fn_list_insert(&flush_fn_list, fn); +} + +void __gcov_flush() { + struct fn_node* curr = flush_fn_list.head; + + while (curr) { + curr->fn(); + curr = curr->next; + } +} + +COMPILER_RT_VISIBILITY +void llvm_delete_flush_function_list(void) { + fn_list_remove(&flush_fn_list); +} + +COMPILER_RT_VISIBILITY +void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn) { + static int atexit_ran = 0; + + if (wfn) + llvm_register_writeout_function(wfn); + + if (ffn) + llvm_register_flush_function(ffn); + + if (atexit_ran == 0) { + atexit_ran = 1; + + /* Make sure we write out the data and delete the data structures. */ + atexit(llvm_delete_flush_function_list); + atexit(llvm_delete_writeout_function_list); + atexit(llvm_writeout_files); + } +} + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.c b/system/lib/compiler-rt/lib/profile/InstrProfiling.c new file mode 100644 index 0000000000000..087d1cdd2efe1 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.c @@ -0,0 +1,84 @@ +/*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include +#include +#include +#include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + + +COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION; + +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { + return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64) + : (INSTR_PROF_RAW_MAGIC_32); +} + +static unsigned ProfileDumped = 0; + +COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() { + return ProfileDumped; +} + +COMPILER_RT_VISIBILITY void lprofSetProfileDumped() { + ProfileDumped = 1; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped() { + lprofSetProfileDumped(); +} + +/* Return the number of bytes needed to add to SizeInBytes to make it + * the result a multiple of 8. + */ +COMPILER_RT_VISIBILITY uint8_t +__llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { + return 7 & (sizeof(uint64_t) - SizeInBytes % sizeof(uint64_t)); +} + +COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { + return __llvm_profile_raw_version; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { + uint64_t *I = __llvm_profile_begin_counters(); + uint64_t *E = __llvm_profile_end_counters(); + + memset(I, 0, sizeof(uint64_t) * (E - I)); + + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const __llvm_profile_data *DI; + for (DI = DataBegin; DI < DataEnd; ++DI) { + uint64_t CurrentVSiteCount = 0; + uint32_t VKI, i; + if (!DI->Values) + continue; + + ValueProfNode **ValueCounters = (ValueProfNode **)DI->Values; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + CurrentVSiteCount += DI->NumValueSites[VKI]; + + for (i = 0; i < CurrentVSiteCount; ++i) { + ValueProfNode *CurrentVNode = ValueCounters[i]; + + while (CurrentVNode) { + CurrentVNode->Count = 0; + CurrentVNode = CurrentVNode->Next; + } + } + } + ProfileDumped = 0; +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.h b/system/lib/compiler-rt/lib/profile/InstrProfiling.h new file mode 100644 index 0000000000000..3a3bab3d0b4c8 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.h @@ -0,0 +1,310 @@ +/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_H_ +#define PROFILE_INSTRPROFILING_H_ + +#include "InstrProfilingPort.h" +#include + +#define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY +#include "profile/InstrProfData.inc" + +enum ValueKind { +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, +#include "profile/InstrProfData.inc" +}; + +typedef void *IntPtrT; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) + __llvm_profile_data { +#define INSTR_PROF_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} __llvm_profile_data; + +typedef struct __llvm_profile_header { +#define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} __llvm_profile_header; + +typedef struct ValueProfNode * PtrToNodeT; +typedef struct ValueProfNode { +#define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} ValueProfNode; + +/*! + * \brief Return 1 if profile counters are continuously synced to the raw + * profile via an mmap(). This is in contrast to the default mode, in which + * the raw profile is written out at program exit time. + */ +int __llvm_profile_is_continuous_mode_enabled(void); + +/*! + * \brief Enable continuous mode. + * + * See \ref __llvm_profile_is_continuous_mode_enabled. The behavior is undefined + * if continuous mode is already enabled, or if it cannot be enable due to + * conflicting options. + */ +void __llvm_profile_enable_continuous_mode(void); + +/*! + * \brief Get number of bytes necessary to pad the argument to eight + * byte boundary. + */ +uint8_t __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes); + +/*! + * \brief Get required size for profile buffer. + */ +uint64_t __llvm_profile_get_size_for_buffer(void); + +/*! + * \brief Write instrumentation data to the given buffer. + * + * \pre \c Buffer is the start of a buffer at least as big as \a + * __llvm_profile_get_size_for_buffer(). + */ +int __llvm_profile_write_buffer(char *Buffer); + +const __llvm_profile_data *__llvm_profile_begin_data(void); +const __llvm_profile_data *__llvm_profile_end_data(void); +const char *__llvm_profile_begin_names(void); +const char *__llvm_profile_end_names(void); +uint64_t *__llvm_profile_begin_counters(void); +uint64_t *__llvm_profile_end_counters(void); +ValueProfNode *__llvm_profile_begin_vnodes(); +ValueProfNode *__llvm_profile_end_vnodes(); +uint32_t *__llvm_profile_begin_orderfile(); + +/*! + * \brief Clear profile counters to zero. + * + */ +void __llvm_profile_reset_counters(void); + +/*! + * \brief Merge profile data from buffer. + * + * Read profile data form buffer \p Profile and merge with + * in-process profile counters. The client is expected to + * have checked or already knows the profile data in the + * buffer matches the in-process counter structure before + * calling it. + */ +void __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); + +/*! \brief Check if profile in buffer matches the current binary. + * + * Returns 0 (success) if the profile data in buffer \p Profile with size + * \p Size was generated by the same binary and therefore matches + * structurally the in-process counters. If the profile data in buffer is + * not compatible, the interface returns 1 (failure). + */ +int __llvm_profile_check_compatibility(const char *Profile, + uint64_t Size); + +/*! + * \brief Counts the number of times a target value is seen. + * + * Records the target value for the CounterIndex if not seen before. Otherwise, + * increments the counter associated w/ the target value. + * void __llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + * uint32_t CounterIndex); + */ +void INSTR_PROF_VALUE_PROF_FUNC( +#define VALUE_PROF_FUNC_PARAM(ArgType, ArgName, ArgLLVMType) ArgType ArgName +#include "profile/InstrProfData.inc" + ); + +void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, + uint64_t CounterValue); + +/*! + * \brief Write instrumentation data to the current file. + * + * Writes to the file with the last name given to \a * + * __llvm_profile_set_filename(), + * or if it hasn't been called, the \c LLVM_PROFILE_FILE environment variable, + * or if that's not set, the last name set to INSTR_PROF_PROFILE_NAME_VAR, + * or if that's not set, \c "default.profraw". + */ +int __llvm_profile_write_file(void); + +int __llvm_orderfile_write_file(void); +/*! + * \brief this is a wrapper interface to \c __llvm_profile_write_file. + * After this interface is invoked, a arleady dumped flag will be set + * so that profile won't be dumped again during program exit. + * Invocation of interface __llvm_profile_reset_counters will clear + * the flag. This interface is designed to be used to collect profile + * data from user selected hot regions. The use model is + * __llvm_profile_reset_counters(); + * ... hot region 1 + * __llvm_profile_dump(); + * .. some other code + * __llvm_profile_reset_counters(); + * ... hot region 2 + * __llvm_profile_dump(); + * + * It is expected that on-line profile merging is on with \c %m specifier + * used in profile filename . If merging is not turned on, user is expected + * to invoke __llvm_profile_set_filename to specify different profile names + * for different regions before dumping to avoid profile write clobbering. + */ +int __llvm_profile_dump(void); + +int __llvm_orderfile_dump(void); + +/*! + * \brief Set the filename for writing instrumentation data. + * + * Sets the filename to be used for subsequent calls to + * \a __llvm_profile_write_file(). + * + * \c Name is not copied, so it must remain valid. Passing NULL resets the + * filename logic to the default behaviour. + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the filename within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. + */ +void __llvm_profile_set_filename(const char *Name); + +/*! + * \brief Set the FILE object for writing instrumentation data. + * + * Sets the FILE object to be used for subsequent calls to + * \a __llvm_profile_write_file(). The profile file name set by environment + * variable, command-line option, or calls to \a __llvm_profile_set_filename + * will be ignored. + * + * \c File will not be closed after a call to \a __llvm_profile_write_file() but + * it may be flushed. Passing NULL restores default behavior. + * + * If \c EnableMerge is nonzero, the runtime will always merge profiling data + * with the contents of the profiling file. If EnableMerge is zero, the runtime + * may still merge the data if it would have merged for another reason (for + * example, because of a %m specifier in the file name). + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the file object within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. + */ +void __llvm_profile_set_file_object(FILE *File, int EnableMerge); + +/*! \brief Register to write instrumentation data to file at exit. */ +int __llvm_profile_register_write_file_atexit(void); + +/*! \brief Initialize file handling. */ +void __llvm_profile_initialize_file(void); + +/*! + * \brief Return path prefix (excluding the base filename) of the profile data. + * This is useful for users using \c -fprofile-generate=./path_prefix who do + * not care about the default raw profile name. It is also useful to collect + * more than more profile data files dumped in the same directory (Online + * merge mode is turned on for instrumented programs with shared libs). + * Side-effect: this API call will invoke malloc with dynamic memory allocation. + */ +const char *__llvm_profile_get_path_prefix(); + +/*! + * \brief Return filename (including path) of the profile data. Note that if the + * user calls __llvm_profile_set_filename later after invoking this interface, + * the actual file name may differ from what is returned here. + * Side-effect: this API call will invoke malloc with dynamic memory allocation + * (the returned pointer must be passed to `free` to avoid a leak). + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only retrieves the filename from the copy + * of the runtime available to the calling image. + */ +const char *__llvm_profile_get_filename(); + +/*! \brief Get the magic token for the file format. */ +uint64_t __llvm_profile_get_magic(void); + +/*! \brief Get the version of the file format. */ +uint64_t __llvm_profile_get_version(void); + +/*! \brief Get the number of entries in the profile data section. */ +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End); + +/* ! \brief Given the sizes of the data and counter information, return the + * number of padding bytes before and after the counters, and after the names, + * in the raw profile. + * + * Note: In this context, "size" means "number of entries", i.e. the first two + * arguments must be the result of __llvm_profile_get_data_size() and of + * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp. + * + * Note: When mmap() mode is disabled, no padding bytes before/after counters + * are needed. However, in mmap() mode, the counter section in the raw profile + * must be page-aligned: this API computes the number of padding bytes + * needed to achieve that. + */ +void __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterNames); + +/*! + * \brief Set the flag that profile data has been dumped to the file. + * This is useful for users to disable dumping profile data to the file for + * certain processes in case the processes don't have permission to write to + * the disks, and trying to do so would result in side effects such as crashes. + */ +void __llvm_profile_set_dumped(); + +/*! + * This variable is defined in InstrProfilingRuntime.cpp as a hidden + * symbol. Its main purpose is to enable profile runtime user to + * bypass runtime initialization code -- if the client code explicitly + * define this variable, then InstProfileRuntime.o won't be linked in. + * Note that this variable's visibility needs to be hidden so that the + * definition of this variable in an instrumented shared library won't + * affect runtime initialization decision of the main program. + * __llvm_profile_profile_runtime. */ +COMPILER_RT_VISIBILITY extern int INSTR_PROF_PROFILE_RUNTIME_VAR; + +/*! + * This variable is defined in InstrProfiling.c. Its main purpose is to + * encode the raw profile version value and other format related information + * such as whether the profile is from IR based instrumentation. The variable + * is defined as weak so that compiler can emit an overriding definition + * depending on user option. Since we don't support mixing FE and IR based + * data in the same raw profile data file (in other words, shared libs and + * main program are expected to be instrumented in the same way), there is + * no need for this variable to be hidden. + */ +extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */ + +/*! + * This variable is a weak symbol defined in InstrProfiling.c. It allows + * compiler instrumentation to provide overriding definition with value + * from compiler command line. This variable has default visibility. + */ +extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */ + +#endif /* PROFILE_INSTRPROFILING_H_ */ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c new file mode 100644 index 0000000000000..174280fd4b528 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -0,0 +1,129 @@ +/*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" + +/* When continuous mode is enabled (%c), this parameter is set to 1. + * + * This parameter is defined here in InstrProfilingBuffer.o, instead of in + * InstrProfilingFile.o, to sequester all libc-dependent code in + * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this + * layering is violated. */ +static int ContinuouslySyncProfile = 0; + +COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) { + return ContinuouslySyncProfile; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { + ContinuouslySyncProfile = 1; +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_size_for_buffer(void) { + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + + return __llvm_profile_get_size_for_buffer_internal( + DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { + intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / + sizeof(__llvm_profile_data); +} + +/// Calculate the number of padding bytes needed to add to \p Offset in order +/// for (\p Offset + Padding) to be page-aligned. +static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset, + unsigned PageSize) { + uint64_t OffsetModPage = Offset % PageSize; + if (OffsetModPage > 0) + return PageSize - OffsetModPage; + return 0; +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, + uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, + uint64_t *PaddingBytesAfterNames) { + if (!__llvm_profile_is_continuous_mode_enabled()) { + *PaddingBytesBeforeCounters = 0; + *PaddingBytesAfterCounters = 0; + *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); + return; + } + + // In continuous mode, the file offsets for headers and for the start of + // counter sections need to be page-aligned. + unsigned PageSize = getpagesize(); + uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data); + uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t); + *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign( + sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize); + *PaddingBytesAfterCounters = + calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize); + *PaddingBytesAfterNames = + calculateBytesNeededToPageAlign(NamesSize, PageSize); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_size_for_buffer_internal( + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, const uint64_t *CountersEnd, + const char *NamesBegin, const char *NamesEnd) { + /* Match logic in __llvm_profile_write_buffer(). */ + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = CountersEnd - CountersBegin; + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + + return sizeof(__llvm_profile_header) + + (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters + + (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters + + NamesSize + PaddingBytesAfterNames; +} + +COMPILER_RT_VISIBILITY +void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) { + BufferWriter->Write = lprofBufferWriter; + BufferWriter->WriterCtx = Buffer; +} + +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, Buffer); + return lprofWriteData(&BufferWriter, 0, 0); +} + +COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( + char *Buffer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, + const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, Buffer); + return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, + CountersEnd, 0, NamesBegin, NamesEnd, 0); +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c new file mode 100644 index 0000000000000..7f3727eed92db --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -0,0 +1,1029 @@ +/*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include +#include +#include +#include +#ifdef _MSC_VER +/* For _alloca. */ +#include +#endif +#if defined(_WIN32) +#include "WindowsMMap.h" +/* For _chsize_s */ +#include +#include +#else +#include +#include +#include +#if defined(__linux__) +#include +#endif +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" +#include "InstrProfilingUtil.h" + +/* From where is profile name specified. + * The order the enumerators define their + * precedence. Re-order them may lead to + * runtime behavior change. */ +typedef enum ProfileNameSpecifier { + PNS_unknown = 0, + PNS_default, + PNS_command_line, + PNS_environment, + PNS_runtime_api +} ProfileNameSpecifier; + +static const char *getPNSStr(ProfileNameSpecifier PNS) { + switch (PNS) { + case PNS_default: + return "default setting"; + case PNS_command_line: + return "command line"; + case PNS_environment: + return "environment variable"; + case PNS_runtime_api: + return "runtime API"; + default: + return "Unknown"; + } +} + +#define MAX_PID_SIZE 16 +/* Data structure holding the result of parsed filename pattern. */ +typedef struct lprofFilename { + /* File name string possibly with %p or %h specifiers. */ + const char *FilenamePat; + /* A flag indicating if FilenamePat's memory is allocated + * by runtime. */ + unsigned OwnsFilenamePat; + const char *ProfilePathPrefix; + char PidChars[MAX_PID_SIZE]; + char Hostname[COMPILER_RT_MAX_HOSTLEN]; + unsigned NumPids; + unsigned NumHosts; + /* When in-process merging is enabled, this parameter specifies + * the total number of profile data files shared by all the processes + * spawned from the same binary. By default the value is 1. If merging + * is not enabled, its value should be 0. This parameter is specified + * by the %[0-9]m specifier. For instance %2m enables merging using + * 2 profile data files. %1m is equivalent to %m. Also %m specifier + * can only appear once at the end of the name pattern. */ + unsigned MergePoolSize; + ProfileNameSpecifier PNS; +} lprofFilename; + +static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, + 0, 0, 0, PNS_unknown}; + +static int ProfileMergeRequested = 0; +static int isProfileMergeRequested() { return ProfileMergeRequested; } +static void setProfileMergeRequested(int EnableMerge) { + ProfileMergeRequested = EnableMerge; +} + +static FILE *ProfileFile = NULL; +static FILE *getProfileFile() { return ProfileFile; } +static void setProfileFile(FILE *File) { ProfileFile = File; } + +COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because " + "continuous sync mode (%%c) is enabled", + fileno(File)); + return; + } + setProfileFile(File); + setProfileMergeRequested(EnableMerge); +} + +static int getCurFilenameLength(); +static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); +static unsigned doMerging() { + return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); +} + +/* Return 1 if there is an error, otherwise return 0. */ +static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + uint32_t I; + FILE *File = (FILE *)This->WriterCtx; + char Zeroes[sizeof(uint64_t)] = {0}; + for (I = 0; I < NumIOVecs; I++) { + if (IOVecs[I].Data) { + if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) != + IOVecs[I].NumElm) + return 1; + } else if (IOVecs[I].UseZeroPadding) { + size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm; + while (BytesToWrite > 0) { + size_t PartialWriteLen = + (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t); + if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) != + PartialWriteLen) { + return 1; + } + BytesToWrite -= PartialWriteLen; + } + } else { + if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1) + return 1; + } + } + return 0; +} + +/* TODO: make buffer size controllable by an internal option, and compiler can pass the size + to runtime via a variable. */ +static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { + if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != + INSTR_ORDER_FILE_BUFFER_SIZE) + return 1; + return 0; +} + +static void initFileWriter(ProfDataWriter *This, FILE *File) { + This->Write = fileWriter; + This->WriterCtx = File; +} + +COMPILER_RT_VISIBILITY ProfBufferIO * +lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) { + FreeHook = &free; + DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1); + VPBufferSize = BufferSz; + ProfDataWriter *fileWriter = + (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1); + initFileWriter(fileWriter, File); + ProfBufferIO *IO = lprofCreateBufferIO(fileWriter); + IO->OwnFileWriter = 1; + return IO; +} + +static void setupIOBuffer() { + const char *BufferSzStr = 0; + BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); + if (BufferSzStr && BufferSzStr[0]) { + VPBufferSize = atoi(BufferSzStr); + DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1); + } +} + +/* Get the size of the profile file. If there are any errors, print the + * message under the assumption that the profile is being read for merging + * purposes, and return -1. Otherwise return the file size in the inout param + * \p ProfileFileSize. */ +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize) { + if (fseek(ProfileFile, 0L, SEEK_END) == -1) { + PROF_ERR("Unable to merge profile data, unable to get size: %s\n", + strerror(errno)); + return -1; + } + *ProfileFileSize = ftell(ProfileFile); + + /* Restore file offset. */ + if (fseek(ProfileFile, 0L, SEEK_SET) == -1) { + PROF_ERR("Unable to merge profile data, unable to rewind: %s\n", + strerror(errno)); + return -1; + } + + if (*ProfileFileSize > 0 && + *ProfileFileSize < sizeof(__llvm_profile_header)) { + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is too small."); + return -1; + } + return 0; +} + +/* mmap() \p ProfileFile for profile merging purposes, assuming that an + * exclusive lock is held on the file and that \p ProfileFileSize is the + * length of the file. Return the mmap'd buffer in the inout variable + * \p ProfileBuffer. Returns -1 on failure. On success, the caller is + * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */ +static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize, + char **ProfileBuffer) { + *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE, + fileno(ProfileFile), 0); + if (*ProfileBuffer == MAP_FAILED) { + PROF_ERR("Unable to merge profile data, mmap failed: %s\n", + strerror(errno)); + return -1; + } + + if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) { + (void)munmap(*ProfileBuffer, ProfileFileSize); + PROF_WARN("Unable to merge profile data: %s\n", + "source profile file is not compatible."); + return -1; + } + return 0; +} + +/* Read profile data in \c ProfileFile and merge with in-memory + profile counters. Returns -1 if there is fatal error, otheriwse + 0 is returned. Returning 0 does not mean merge is actually + performed. If merge is actually done, *MergeDone is set to 1. +*/ +static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { + uint64_t ProfileFileSize; + char *ProfileBuffer; + + /* Get the size of the profile on disk. */ + if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1) + return -1; + + /* Nothing to merge. */ + if (!ProfileFileSize) + return 0; + + /* mmap() the profile and check that it is compatible with the data in + * the current image. */ + if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1) + return -1; + + /* Now start merging */ + __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); + + // Truncate the file in case merging of value profile did not happend to + // prevent from leaving garbage data at the end of the profile file. + COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer()); + + (void)munmap(ProfileBuffer, ProfileFileSize); + *MergeDone = 1; + + return 0; +} + +/* Create the directory holding the file, if needed. */ +static void createProfileDir(const char *Filename) { + size_t Length = strlen(Filename); + if (lprofFindFirstDirSeparator(Filename)) { + char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1); + strncpy(Copy, Filename, Length + 1); + __llvm_profile_recursive_mkdir(Copy); + } +} + +/* Open the profile data for merging. It opens the file in r+b mode with + * file locking. If the file has content which is compatible with the + * current process, it also reads in the profile data in the file and merge + * it with in-memory counters. After the profile data is merged in memory, + * the original profile data is truncated and gets ready for the profile + * dumper. With profile merging enabled, each executable as well as any of + * its instrumented shared libraries dump profile data into their own data file. +*/ +static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { + FILE *ProfileFile = NULL; + int rc; + + ProfileFile = getProfileFile(); + if (ProfileFile) { + lprofLockFileHandle(ProfileFile); + } else { + createProfileDir(ProfileFileName); + ProfileFile = lprofOpenFileEx(ProfileFileName); + } + if (!ProfileFile) + return NULL; + + rc = doProfileMerging(ProfileFile, MergeDone); + if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) || + fseek(ProfileFile, 0L, SEEK_SET) == -1) { + PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName, + strerror(errno)); + fclose(ProfileFile); + return NULL; + } + return ProfileFile; +} + +static FILE *getFileObject(const char *OutputName) { + FILE *File; + File = getProfileFile(); + if (File != NULL) { + return File; + } + + return fopen(OutputName, "ab"); +} + +/* Write profile data to file \c OutputName. */ +static int writeFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + int MergeDone = 0; + VPMergeHook = &lprofMergeValueProfData; + if (doMerging()) + OutputFile = openFileForMerging(OutputName, &MergeDone); + else + OutputFile = getFileObject(OutputName); + + if (!OutputFile) + return -1; + + FreeHook = &free; + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, OutputFile); + RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone); + + if (OutputFile == getProfileFile()) { + fflush(OutputFile); + if (doMerging()) { + lprofUnlockFileHandle(OutputFile); + } + } else { + fclose(OutputFile); + } + + return RetVal; +} + +/* Write order data to file \c OutputName. */ +static int writeOrderFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + OutputFile = fopen(OutputName, "w"); + + if (!OutputFile) { + PROF_WARN("can't open file with mode ab: %s\n", OutputName); + return -1; + } + + FreeHook = &free; + setupIOBuffer(); + const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); + RetVal = orderFileWriter(OutputFile, DataBegin); + + fclose(OutputFile); + return RetVal; +} + +#define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE" + +static void truncateCurrentFile(void) { + const char *Filename; + char *FilenameBuf; + FILE *File; + int Length; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + /* Only create the profile directory and truncate an existing profile once. + * In continuous mode, this is necessary, as the profile is written-to by the + * runtime initializer. */ + int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL; + if (initialized) + return; +#if defined(_WIN32) + _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV); +#else + setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1); +#endif + + /* Create the profile dir (even if online merging is enabled), so that + * the profile file can be set up if continuous mode is enabled. */ + createProfileDir(Filename); + + /* By pass file truncation to allow online raw profile merging. */ + if (lprofCurFilename.MergePoolSize) + return; + + /* Truncate the file. Later we'll reopen and append. */ + File = fopen(Filename, "w"); + if (!File) + return; + fclose(File); +} + +#ifndef _MSC_VER +static void assertIsZero(int *i) { + if (*i) + PROF_WARN("Expected flag to be 0, but got: %d\n", *i); +} +#endif + +#if !defined(__Fuchsia__) && !defined(_WIN32) +/* Write a partial profile to \p Filename, which is required to be backed by + * the open file object \p File. */ +static int writeProfileWithFileObject(const char *Filename, FILE *File) { + setProfileFile(File); + int rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + setProfileFile(NULL); + return rc; +} + +/* Unlock the profile \p File and clear the unlock flag. */ +static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) { + if (!*ProfileRequiresUnlock) { + PROF_WARN("%s", "Expected to require profile unlock\n"); + } + lprofUnlockFileHandle(File); + *ProfileRequiresUnlock = 0; +} +#endif // !defined(__Fuchsia__) && !defined(_WIN32) + +static void initializeProfileForContinuousMode(void) { + if (!__llvm_profile_is_continuous_mode_enabled()) + return; + +#if defined(__Fuchsia__) || defined(_WIN32) + PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows."); +#else // defined(__Fuchsia__) || defined(_WIN32) + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = CountersEnd - CountersBegin; + + /* Check that the counter and data sections in this image are page-aligned. */ + unsigned PageSize = getpagesize(); + if ((intptr_t)CountersBegin % PageSize != 0) { + PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", + CountersBegin, PageSize); + return; + } + if ((intptr_t)DataBegin % PageSize != 0) { + PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", + DataBegin, PageSize); + return; + } + + int Length = getCurFilenameLength(); + char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + const char *Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return; + + FILE *File = NULL; + off_t CurrentFileOffset = 0; + off_t OffsetModPage = 0; + + /* Whether an exclusive lock on the profile must be dropped after init. + * Use a cleanup to warn if the unlock does not occur. */ + COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0; + + if (!doMerging()) { + /* We are not merging profiles, so open the raw profile in append mode. */ + File = fopen(Filename, "a+b"); + if (!File) + return; + + /* Check that the offset within the file is page-aligned. */ + CurrentFileOffset = ftello(File); + OffsetModPage = CurrentFileOffset % PageSize; + if (OffsetModPage != 0) { + PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" + "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n", + (uint64_t)CurrentFileOffset, PageSize); + return; + } + + /* Grow the profile so that mmap() can succeed. Leak the file handle, as + * the file should stay open. */ + if (writeProfileWithFileObject(Filename, File) != 0) + return; + } else { + /* We are merging profiles. Map the counter section as shared memory into + * the profile, i.e. into each participating process. An increment in one + * process should be visible to every other process with the same counter + * section mapped. */ + File = lprofOpenFileEx(Filename); + if (!File) + return; + + ProfileRequiresUnlock = 1; + + uint64_t ProfileFileSize; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) + return unlockProfile(&ProfileRequiresUnlock, File); + + if (ProfileFileSize == 0) { + /* Grow the profile so that mmap() can succeed. Leak the file handle, as + * the file should stay open. */ + if (writeProfileWithFileObject(Filename, File) != 0) + return unlockProfile(&ProfileRequiresUnlock, File); + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + char *ProfileBuffer; + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 || + munmap(ProfileBuffer, ProfileFileSize) == -1) + return unlockProfile(&ProfileRequiresUnlock, File); + } + } + + int Fileno = fileno(File); + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + + uint64_t PageAlignedCountersLength = + (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = + CurrentFileOffset + sizeof(__llvm_profile_header) + + (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; + + uint64_t *CounterMmap = (uint64_t *)mmap( + (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); + if (CounterMmap != CountersBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - CountersBegin: %p\n" + " - PageAlignedCountersLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToCounters: %" PRIu64 "\n", + strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, + FileOffsetToCounters); + } + + unlockProfile(&ProfileRequiresUnlock, File); +#endif // defined(__Fuchsia__) || defined(_WIN32) +} + +static const char *DefaultProfileName = "default.profraw"; +static void resetFilenameToDefault(void) { + if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { + free((void *)lprofCurFilename.FilenamePat); + } + memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); + lprofCurFilename.FilenamePat = DefaultProfileName; + lprofCurFilename.PNS = PNS_default; +} + +static unsigned getMergePoolSize(const char *FilenamePat, int *I) { + unsigned J = 0, Num = 0; + for (;; ++J) { + char C = FilenamePat[*I + J]; + if (C == 'm') { + *I += J; + return Num ? Num : 1; + } + if (C < '0' || C > '9') + break; + Num = Num * 10 + C - '0'; + + /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed + * to be in-bound as the string is null terminated. */ + } + return 0; +} + +/* Parses the pattern string \p FilenamePat and stores the result to + * lprofcurFilename structure. */ +static int parseFilenamePattern(const char *FilenamePat, + unsigned CopyFilenamePat) { + int NumPids = 0, NumHosts = 0, I; + char *PidChars = &lprofCurFilename.PidChars[0]; + char *Hostname = &lprofCurFilename.Hostname[0]; + int MergingEnabled = 0; + + /* Clean up cached prefix and filename. */ + if (lprofCurFilename.ProfilePathPrefix) + free((void *)lprofCurFilename.ProfilePathPrefix); + + if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { + free((void *)lprofCurFilename.FilenamePat); + } + + memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); + + if (!CopyFilenamePat) + lprofCurFilename.FilenamePat = FilenamePat; + else { + lprofCurFilename.FilenamePat = strdup(FilenamePat); + lprofCurFilename.OwnsFilenamePat = 1; + } + /* Check the filename for "%p", which indicates a pid-substitution. */ + for (I = 0; FilenamePat[I]; ++I) + if (FilenamePat[I] == '%') { + if (FilenamePat[++I] == 'p') { + if (!NumPids++) { + if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { + PROF_WARN("Unable to get pid for filename pattern %s. Using the " + "default name.", + FilenamePat); + return -1; + } + } + } else if (FilenamePat[I] == 'h') { + if (!NumHosts++) + if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) { + PROF_WARN("Unable to get hostname for filename pattern %s. Using " + "the default name.", + FilenamePat); + return -1; + } + } else if (FilenamePat[I] == 'c') { + if (__llvm_profile_is_continuous_mode_enabled()) { + PROF_WARN("%%c specifier can only be specified once in %s.\n", + FilenamePat); + return -1; + } + + __llvm_profile_enable_continuous_mode(); + I++; /* advance to 'c' */ + } else { + unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I); + if (!MergePoolSize) + continue; + if (MergingEnabled) { + PROF_WARN("%%m specifier can only be specified once in %s.\n", + FilenamePat); + return -1; + } + MergingEnabled = 1; + lprofCurFilename.MergePoolSize = MergePoolSize; + } + } + + lprofCurFilename.NumPids = NumPids; + lprofCurFilename.NumHosts = NumHosts; + return 0; +} + +static void parseAndSetFilename(const char *FilenamePat, + ProfileNameSpecifier PNS, + unsigned CopyFilenamePat) { + + const char *OldFilenamePat = lprofCurFilename.FilenamePat; + ProfileNameSpecifier OldPNS = lprofCurFilename.PNS; + + /* The old profile name specifier takes precedence over the old one. */ + if (PNS < OldPNS) + return; + + if (!FilenamePat) + FilenamePat = DefaultProfileName; + + if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) { + lprofCurFilename.PNS = PNS; + return; + } + + /* When PNS >= OldPNS, the last one wins. */ + if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat)) + resetFilenameToDefault(); + lprofCurFilename.PNS = PNS; + + if (!OldFilenamePat) { + if (getenv("LLVM_PROFILE_VERBOSE")) + PROF_NOTE("Set profile file path to \"%s\" via %s.\n", + lprofCurFilename.FilenamePat, getPNSStr(PNS)); + } else { + if (getenv("LLVM_PROFILE_VERBOSE")) + PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n", + OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat, + getPNSStr(PNS)); + } + + truncateCurrentFile(); + initializeProfileForContinuousMode(); +} + +/* Return buffer length that is required to store the current profile + * filename with PID and hostname substitutions. */ +/* The length to hold uint64_t followed by 3 digits pool id including '_' */ +#define SIGLEN 24 +static int getCurFilenameLength() { + int Len; + if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) + return 0; + + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.MergePoolSize)) + return strlen(lprofCurFilename.FilenamePat); + + Len = strlen(lprofCurFilename.FilenamePat) + + lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + + lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2); + if (lprofCurFilename.MergePoolSize) + Len += SIGLEN; + return Len; +} + +/* Return the pointer to the current profile file name (after substituting + * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer + * to store the resulting filename. If no substitution is needed, the + * current filename pattern string is directly returned, unless ForceUseBuf + * is enabled. */ +static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { + int I, J, PidLength, HostNameLength, FilenamePatLength; + const char *FilenamePat = lprofCurFilename.FilenamePat; + + if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) + return 0; + + if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || + lprofCurFilename.MergePoolSize || + __llvm_profile_is_continuous_mode_enabled())) { + if (!ForceUseBuf) + return lprofCurFilename.FilenamePat; + + FilenamePatLength = strlen(lprofCurFilename.FilenamePat); + memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength); + FilenameBuf[FilenamePatLength] = '\0'; + return FilenameBuf; + } + + PidLength = strlen(lprofCurFilename.PidChars); + HostNameLength = strlen(lprofCurFilename.Hostname); + /* Construct the new filename. */ + for (I = 0, J = 0; FilenamePat[I]; ++I) + if (FilenamePat[I] == '%') { + if (FilenamePat[++I] == 'p') { + memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength); + J += PidLength; + } else if (FilenamePat[I] == 'h') { + memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength); + J += HostNameLength; + } else { + if (!getMergePoolSize(FilenamePat, &I)) + continue; + char LoadModuleSignature[SIGLEN + 1]; + int S; + int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize; + S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d", + lprofGetLoadModuleSignature(), ProfilePoolId); + if (S == -1 || S > SIGLEN) + S = SIGLEN; + memcpy(FilenameBuf + J, LoadModuleSignature, S); + J += S; + } + /* Drop any unknown substitutions. */ + } else + FilenameBuf[J++] = FilenamePat[I]; + FilenameBuf[J] = 0; + + return FilenameBuf; +} + +/* Returns the pointer to the environment variable + * string. Returns null if the env var is not set. */ +static const char *getFilenamePatFromEnv(void) { + const char *Filename = getenv("LLVM_PROFILE_FILE"); + if (!Filename || !Filename[0]) + return 0; + return Filename; +} + +COMPILER_RT_VISIBILITY +const char *__llvm_profile_get_path_prefix(void) { + int Length; + char *FilenameBuf, *Prefix; + const char *Filename, *PrefixEnd; + + if (lprofCurFilename.ProfilePathPrefix) + return lprofCurFilename.ProfilePathPrefix; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + if (!Filename) + return "\0"; + + PrefixEnd = lprofFindLastDirSeparator(Filename); + if (!PrefixEnd) + return "\0"; + + Length = PrefixEnd - Filename + 1; + Prefix = (char *)malloc(Length + 1); + if (!Prefix) { + PROF_ERR("Failed to %s\n", "allocate memory."); + return "\0"; + } + memcpy(Prefix, Filename, Length); + Prefix[Length] = '\0'; + lprofCurFilename.ProfilePathPrefix = Prefix; + return Prefix; +} + +COMPILER_RT_VISIBILITY +const char *__llvm_profile_get_filename(void) { + int Length; + char *FilenameBuf; + const char *Filename; + + Length = getCurFilenameLength(); + FilenameBuf = (char *)malloc(Length + 1); + if (!FilenameBuf) { + PROF_ERR("Failed to %s\n", "allocate memory."); + return "\0"; + } + Filename = getCurFilename(FilenameBuf, 1); + if (!Filename) + return "\0"; + + return FilenameBuf; +} + +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. Both user specified + * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE + * environment variable can override this default value. */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize_file(void) { + const char *EnvFilenamePat; + const char *SelectedPat = NULL; + ProfileNameSpecifier PNS = PNS_unknown; + int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0); + + EnvFilenamePat = getFilenamePatFromEnv(); + if (EnvFilenamePat) { + /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid + at the moment when __llvm_profile_write_file() gets executed. */ + parseAndSetFilename(EnvFilenamePat, PNS_environment, 1); + return; + } else if (hasCommandLineOverrider) { + SelectedPat = INSTR_PROF_PROFILE_NAME_VAR; + PNS = PNS_command_line; + } else { + SelectedPat = NULL; + PNS = PNS_default; + } + + parseAndSetFilename(SelectedPat, PNS, 0); +} + +/* This API is directly called by the user application code. It has the + * highest precedence compared with LLVM_PROFILE_FILE environment variable + * and command line option -fprofile-instr-generate=. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_set_filename(const char *FilenamePat) { + if (__llvm_profile_is_continuous_mode_enabled()) + return; + parseAndSetFilename(FilenamePat, PNS_runtime_api, 1); +} + +/* The public API for writing profile data into the file with name + * set by previous calls to __llvm_profile_set_filename or + * __llvm_profile_override_default_filename or + * __llvm_profile_initialize_file. */ +COMPILER_RT_VISIBILITY +int __llvm_profile_write_file(void) { + int rc, Length; + const char *Filename; + char *FilenameBuf; + int PDeathSig = 0; + + if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) { + PROF_NOTE("Profile data not written to file: %s.\n", "already written"); + return 0; + } + + Length = getCurFilenameLength(); + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 0); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + return -1; + } + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + // Temporarily suspend getting SIGKILL when the parent exits. + PDeathSig = lprofSuspendSigKill(); + + /* Write profile data to the file. */ + rc = writeFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_profile_dump(void) { + if (!doMerging()) + PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering " + " of previously dumped profile data : %s. Either use %%m " + "in profile name or change profile name before dumping.\n", + "online profile merging is not on"); + int rc = __llvm_profile_write_file(); + lprofSetProfileDumped(); + return rc; +} + +/* Order file data will be saved in a file with suffx .order. */ +static const char *OrderFileSuffix = ".order"; + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_write_file(void) { + int rc, Length, LengthBeforeAppend, SuffixLength; + const char *Filename; + char *FilenameBuf; + int PDeathSig = 0; + + SuffixLength = strlen(OrderFileSuffix); + Length = getCurFilenameLength() + SuffixLength; + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 1); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + return -1; + } + + /* Append order file suffix */ + LengthBeforeAppend = strlen(Filename); + memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); + FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + // Temporarily suspend getting SIGKILL when the parent exits. + PDeathSig = lprofSuspendSigKill(); + + /* Write order data to the file. */ + rc = writeOrderFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_dump(void) { + int rc = __llvm_orderfile_write_file(); + return rc; +} + +static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } + +COMPILER_RT_VISIBILITY +int __llvm_profile_register_write_file_atexit(void) { + static int HasBeenRegistered = 0; + + if (HasBeenRegistered) + return 0; + + lprofSetupValueProfiler(); + + HasBeenRegistered = 1; + return atexit(writeFileWithoutReturn); +} + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h new file mode 100644 index 0000000000000..0cea4876f0ae6 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -0,0 +1,196 @@ +/*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_INTERNALH_ +#define PROFILE_INSTRPROFILING_INTERNALH_ + +#include + +#include "InstrProfiling.h" + +/*! + * \brief Write instrumentation data to the given buffer, given explicit + * pointers to the live data in memory. This function is probably not what you + * want. Use __llvm_profile_get_size_for_buffer instead. Use this function if + * your program has a custom memory layout. + */ +uint64_t __llvm_profile_get_size_for_buffer_internal( + const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, const uint64_t *CountersEnd, + const char *NamesBegin, const char *NamesEnd); + +/*! + * \brief Write instrumentation data to the given buffer, given explicit + * pointers to the live data in memory. This function is probably not what you + * want. Use __llvm_profile_write_buffer instead. Use this function if your + * program has a custom memory layout. + * + * \pre \c Buffer is the start of a buffer at least as big as \a + * __llvm_profile_get_size_for_buffer_internal(). + */ +int __llvm_profile_write_buffer_internal( + char *Buffer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, + const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd); + +/*! + * The data structure describing the data to be written by the + * low level writer callback function. + * + * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is + * 0, the write is skipped (the writer simply advances ElmSize*NumElm bytes). + * + * If \ref ProfDataIOVec.Data is null, and \ref ProfDataIOVec.UseZeroPadding is + * nonzero, ElmSize*NumElm zero bytes are written. + */ +typedef struct ProfDataIOVec { + const void *Data; + size_t ElmSize; + size_t NumElm; + int UseZeroPadding; +} ProfDataIOVec; + +struct ProfDataWriter; +typedef uint32_t (*WriterCallback)(struct ProfDataWriter *This, ProfDataIOVec *, + uint32_t NumIOVecs); + +typedef struct ProfDataWriter { + WriterCallback Write; + void *WriterCtx; +} ProfDataWriter; + +/*! + * The data structure for buffered IO of profile data. + */ +typedef struct ProfBufferIO { + ProfDataWriter *FileWriter; + uint32_t OwnFileWriter; + /* The start of the buffer. */ + uint8_t *BufferStart; + /* Total size of the buffer. */ + uint32_t BufferSz; + /* Current byte offset from the start of the buffer. */ + uint32_t CurOffset; +} ProfBufferIO; + +/* The creator interface used by testing. */ +ProfBufferIO *lprofCreateBufferIOInternal(void *File, uint32_t BufferSz); + +/*! + * This is the interface to create a handle for buffered IO. + */ +ProfBufferIO *lprofCreateBufferIO(ProfDataWriter *FileWriter); + +/*! + * The interface to destroy the bufferIO handle and reclaim + * the memory. + */ +void lprofDeleteBufferIO(ProfBufferIO *BufferIO); + +/*! + * This is the interface to write \c Data of \c Size bytes through + * \c BufferIO. Returns 0 if successful, otherwise return -1. + */ +int lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, + uint32_t Size); +/*! + * The interface to flush the remaining data in the buffer. + * through the low level writer callback. + */ +int lprofBufferIOFlush(ProfBufferIO *BufferIO); + +/* The low level interface to write data into a buffer. It is used as the + * callback by other high level writer methods such as buffered IO writer + * and profile data writer. */ +uint32_t lprofBufferWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs); +void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer); + +struct ValueProfData; +struct ValueProfRecord; +struct InstrProfValueData; +struct ValueProfNode; + +/*! + * The class that defines a set of methods to read value profile + * data for streaming/serialization from the instrumentation runtime. + */ +typedef struct VPDataReaderType { + uint32_t (*InitRTRecord)(const __llvm_profile_data *Data, + uint8_t *SiteCountArray[]); + /* Function pointer to getValueProfRecordHeader method. */ + uint32_t (*GetValueProfRecordHeaderSize)(uint32_t NumSites); + /* Function pointer to getFristValueProfRecord method. */ + struct ValueProfRecord *(*GetFirstValueProfRecord)(struct ValueProfData *); + /* Return the number of value data for site \p Site. */ + uint32_t (*GetNumValueDataForSite)(uint32_t VK, uint32_t Site); + /* Return the total size of the value profile data of the + * current function. */ + uint32_t (*GetValueProfDataSize)(void); + /*! + * Read the next \p N value data for site \p Site and store the data + * in \p Dst. \p StartNode is the first value node to start with if + * it is not null. The function returns the pointer to the value + * node pointer to be used as the \p StartNode of the next batch reading. + * If there is nothing left, it returns NULL. + */ + struct ValueProfNode *(*GetValueData)(uint32_t ValueKind, uint32_t Site, + struct InstrProfValueData *Dst, + struct ValueProfNode *StartNode, + uint32_t N); +} VPDataReaderType; + +/* Write profile data to destinitation. If SkipNameDataWrite is set to 1, + the name data is already in destintation, we just skip over it. */ +int lprofWriteData(ProfDataWriter *Writer, VPDataReaderType *VPDataReader, + int SkipNameDataWrite); +int lprofWriteDataImpl(ProfDataWriter *Writer, + const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, + const uint64_t *CountersEnd, + VPDataReaderType *VPDataReader, const char *NamesBegin, + const char *NamesEnd, int SkipNameDataWrite); + +/* Merge value profile data pointed to by SrcValueProfData into + * in-memory profile counters pointed by to DstData. */ +void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData, + __llvm_profile_data *DstData); + +VPDataReaderType *lprofGetVPDataReader(); + +/* Internal interface used by test to reset the max number of + * tracked values per value site to be \p MaxVals. + */ +void lprofSetMaxValsPerSite(uint32_t MaxVals); +void lprofSetupValueProfiler(); + +/* Return the profile header 'signature' value associated with the current + * executable or shared library. The signature value can be used to for + * a profile name that is unique to this load module so that it does not + * collide with profiles from other binaries. It also allows shared libraries + * to dump merged profile data into its own profile file. */ +uint64_t lprofGetLoadModuleSignature(); + +/* + * Return non zero value if the profile data has already been + * dumped to the file. + */ +unsigned lprofProfileDumped(); +void lprofSetProfileDumped(); + +COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *); +COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer; +COMPILER_RT_VISIBILITY extern uint32_t VPBufferSize; +COMPILER_RT_VISIBILITY extern uint32_t VPMaxNumValsPerSite; +/* Pointer to the start of static value counters to be allocted. */ +COMPILER_RT_VISIBILITY extern ValueProfNode *CurrentVNode; +COMPILER_RT_VISIBILITY extern ValueProfNode *EndVNode; +extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *); + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c new file mode 100644 index 0000000000000..0fd9b2bcd41f8 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -0,0 +1,132 @@ +/*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------===* +|* This file defines the API needed for in-process merging of profile data +|* stored in memory buffer. +\*===---------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +COMPILER_RT_VISIBILITY +void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *); + +COMPILER_RT_VISIBILITY +uint64_t lprofGetLoadModuleSignature() { + /* A very fast way to compute a module signature. */ + uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() - + __llvm_profile_begin_counters()); + uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(), + __llvm_profile_end_data()); + uint64_t NamesSize = + (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); + uint64_t NumVnodes = + (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); + const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); + + return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) + + (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0); +} + +/* Returns 1 if profile is not structurally compatible. */ +COMPILER_RT_VISIBILITY +int __llvm_profile_check_compatibility(const char *ProfileData, + uint64_t ProfileSize) { + /* Check profile header only for now */ + __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; + SrcDataStart = + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header)); + SrcDataEnd = SrcDataStart + Header->DataSize; + + if (ProfileSize < sizeof(__llvm_profile_header)) + return 1; + + /* Check the header first. */ + if (Header->Magic != __llvm_profile_get_magic() || + Header->Version != __llvm_profile_get_version() || + Header->DataSize != + __llvm_profile_get_data_size(__llvm_profile_begin_data(), + __llvm_profile_end_data()) || + Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() - + __llvm_profile_begin_counters()) || + Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - + __llvm_profile_begin_names()) || + Header->ValueKindLast != IPVK_Last) + return 1; + + if (ProfileSize < sizeof(__llvm_profile_header) + + Header->DataSize * sizeof(__llvm_profile_data) + + Header->NamesSize + Header->CountersSize) + return 1; + + for (SrcData = SrcDataStart, + DstData = (__llvm_profile_data *)__llvm_profile_begin_data(); + SrcData < SrcDataEnd; ++SrcData, ++DstData) { + if (SrcData->NameRef != DstData->NameRef || + SrcData->FuncHash != DstData->FuncHash || + SrcData->NumCounters != DstData->NumCounters) + return 1; + } + + /* Matched! */ + return 0; +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_merge_from_buffer(const char *ProfileData, + uint64_t ProfileSize) { + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; + __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; + uint64_t *SrcCountersStart; + const char *SrcNameStart; + ValueProfData *SrcValueProfDataStart, *SrcValueProfData; + + SrcDataStart = + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header)); + SrcDataEnd = SrcDataStart + Header->DataSize; + SrcCountersStart = (uint64_t *)SrcDataEnd; + SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize); + SrcValueProfDataStart = + (ValueProfData *)(SrcNameStart + Header->NamesSize + + __llvm_profile_get_num_padding_bytes( + Header->NamesSize)); + + for (SrcData = SrcDataStart, + DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), + SrcValueProfData = SrcValueProfDataStart; + SrcData < SrcDataEnd; ++SrcData, ++DstData) { + uint64_t *SrcCounters; + uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr; + unsigned I, NC, NVK = 0; + + NC = SrcData->NumCounters; + SrcCounters = SrcCountersStart + + ((size_t)SrcData->CounterPtr - Header->CountersDelta) / + sizeof(uint64_t); + for (I = 0; I < NC; I++) + DstCounters[I] += SrcCounters[I]; + + /* Now merge value profile data. */ + if (!VPMergeHook) + continue; + + for (I = 0; I <= IPVK_Last; I++) + NVK += (SrcData->NumValueSites[I] != 0); + + if (!NVK) + continue; + + VPMergeHook(SrcValueProfData, DstData); + SrcValueProfData = (ValueProfData *)((char *)SrcValueProfData + + SrcValueProfData->TotalSize); + } +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c new file mode 100644 index 0000000000000..8923ba21cc580 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMergeFile.c @@ -0,0 +1,45 @@ +/*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +|*===----------------------------------------------------------------------=== +|* This file defines APIs needed to support in-process merging for profile data +|* stored in files. +\*===----------------------------------------------------------------------===*/ + +#if !defined(__Fuchsia__) + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +/* Merge value profile data pointed to by SrcValueProfData into + * in-memory profile counters pointed by to DstData. */ +COMPILER_RT_VISIBILITY +void lprofMergeValueProfData(ValueProfData *SrcValueProfData, + __llvm_profile_data *DstData) { + unsigned I, S, V, DstIndex = 0; + InstrProfValueData *VData; + ValueProfRecord *VR = getFirstValueProfRecord(SrcValueProfData); + for (I = 0; I < SrcValueProfData->NumValueKinds; I++) { + VData = getValueProfRecordValueData(VR); + unsigned SrcIndex = 0; + for (S = 0; S < VR->NumValueSites; S++) { + uint8_t NV = VR->SiteCountArray[S]; + for (V = 0; V < NV; V++) { + __llvm_profile_instrument_target_value(VData[SrcIndex].Value, DstData, + DstIndex, VData[SrcIndex].Count); + ++SrcIndex; + } + ++DstIndex; + } + VR = getValueProfRecordNext(VR); + } +} + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c new file mode 100644 index 0000000000000..2d67a55b985cf --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c @@ -0,0 +1,17 @@ +/*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +/* char __llvm_profile_filename[1] + * + * The runtime should only provide its own definition of this symbol when the + * user has not specified one. Set this up by moving the runtime's copy of this + * symbol to an object file within the archive. + */ +COMPILER_RT_WEAK char INSTR_PROF_PROFILE_NAME_VAR[1] = {0}; diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c new file mode 100644 index 0000000000000..23bdb7f37179d --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -0,0 +1,67 @@ +/*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(__APPLE__) +/* Use linker magic to find the bounds of the Data section. */ +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME); +COMPILER_RT_VISIBILITY +extern __llvm_profile_data + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint64_t + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint64_t + CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint32_t + OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); + +COMPILER_RT_VISIBILITY +extern ValueProfNode + VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME); +COMPILER_RT_VISIBILITY +extern ValueProfNode + VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME); + +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart; +} +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_names(void) { return &NamesStart; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_names(void) { return &NamesEnd; } +COMPILER_RT_VISIBILITY +uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } +COMPILER_RT_VISIBILITY +uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_begin_vnodes(void) { + return &VNodesStart; +} +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd; +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c new file mode 100644 index 0000000000000..23b7efbe672b9 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -0,0 +1,197 @@ +/*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This file implements the profiling runtime for Fuchsia and defines the + * shared profile runtime interface. Each module (executable or DSO) statically + * links in the whole profile runtime to satisfy the calls from its + * instrumented code. Several modules in the same program might be separately + * compiled and even use different versions of the instrumentation ABI and data + * format. All they share in common is the VMO and the offset, which live in + * exported globals so that exactly one definition will be shared across all + * modules. Each module has its own independent runtime that registers its own + * atexit hook to append its own data into the shared VMO which is published + * via the data sink hook provided by Fuchsia's dynamic linker. + */ + +#if defined(__Fuchsia__) + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +/* VMO that contains the coverage data shared across all modules. This symbol + * has default visibility and is exported in each module (executable or DSO) + * that statically links in the profiling runtime. + */ +zx_handle_t __llvm_profile_vmo; +/* Current offset within the VMO where data should be written next. This symbol + * has default visibility and is exported in each module (executable or DSO) + * that statically links in the profiling runtime. + */ +uint64_t __llvm_profile_offset; + +static const char ProfileSinkName[] = "llvm-profile"; + +static inline void lprofWrite(const char *fmt, ...) { + char s[256]; + + va_list ap; + va_start(ap, fmt); + int ret = vsnprintf(s, sizeof(s), fmt, ap); + va_end(ap); + + __sanitizer_log_write(s, ret + 1); +} + +static void createVMO() { + /* Don't create VMO if it has been alread created. */ + if (__llvm_profile_vmo != ZX_HANDLE_INVALID) + return; + + /* Get information about the current process. */ + zx_info_handle_basic_t Info; + zx_status_t Status = + _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, + sizeof(Info), NULL, NULL); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: cannot get info about current process: %s\n", + _zx_status_get_string(Status)); + return; + } + + /* Create VMO to hold the profile data. */ + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &__llvm_profile_vmo); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: cannot create VMO: %s\n", + _zx_status_get_string(Status)); + return; + } + + /* Give the VMO a name including our process KOID so it's easy to spot. */ + char VmoName[ZX_MAX_NAME_LEN]; + snprintf(VmoName, sizeof(VmoName), "%s.%" PRIu64, ProfileSinkName, Info.koid); + _zx_object_set_property(__llvm_profile_vmo, ZX_PROP_NAME, VmoName, + strlen(VmoName)); + + /* Duplicate the handle since __sanitizer_publish_data consumes it. */ + zx_handle_t Handle; + Status = + _zx_handle_duplicate(__llvm_profile_vmo, ZX_RIGHT_SAME_RIGHTS, &Handle); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: cannot duplicate VMO handle: %s\n", + _zx_status_get_string(Status)); + _zx_handle_close(__llvm_profile_vmo); + __llvm_profile_vmo = ZX_HANDLE_INVALID; + return; + } + + /* Publish the VMO which contains profile data to the system. */ + __sanitizer_publish_data(ProfileSinkName, Handle); + + /* Use the dumpfile symbolizer markup element to write the name of VMO. */ + lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName); +} + +static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + /* Compute the total length of data to be written. */ + size_t Length = 0; + for (uint32_t I = 0; I < NumIOVecs; I++) + Length += IOVecs[I].ElmSize * IOVecs[I].NumElm; + + /* Resize the VMO to ensure there's sufficient space for the data. */ + zx_status_t Status = + _zx_vmo_set_size(__llvm_profile_vmo, __llvm_profile_offset + Length); + if (Status != ZX_OK) + return -1; + + /* Copy the data into VMO. */ + for (uint32_t I = 0; I < NumIOVecs; I++) { + size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; + if (IOVecs[I].Data) { + Status = _zx_vmo_write(__llvm_profile_vmo, IOVecs[I].Data, + __llvm_profile_offset, Length); + if (Status != ZX_OK) + return -1; + } else if (IOVecs[I].UseZeroPadding) { + /* Resizing the VMO should zero fill. */ + } + __llvm_profile_offset += Length; + } + + return 0; +} + +static void initVMOWriter(ProfDataWriter *This) { + This->Write = lprofVMOWriter; + This->WriterCtx = NULL; +} + +static int dump(void) { + if (lprofProfileDumped()) { + lprofWrite("LLVM Profile: data not published: already written.\n"); + return 0; + } + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + lprofWrite("LLVM Profile: runtime and instrumentation version mismatch: " + "expected %d, but got %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + /* Write the profile data into the mapped region. */ + ProfDataWriter VMOWriter; + initVMOWriter(&VMOWriter); + if (lprofWriteData(&VMOWriter, lprofGetVPDataReader(), 0) != 0) + return -1; + + return 0; +} + +COMPILER_RT_VISIBILITY +int __llvm_profile_dump(void) { + int rc = dump(); + lprofSetProfileDumped(); + return rc; +} + +static void dumpWithoutReturn(void) { dump(); } + +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize_file(void) { createVMO(); } + +COMPILER_RT_VISIBILITY +int __llvm_profile_register_write_file_atexit(void) { + static bool HasBeenRegistered = false; + + if (HasBeenRegistered) + return 0; + + lprofSetupValueProfiler(); + + HasBeenRegistered = true; + return atexit(dumpWithoutReturn); +} + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c new file mode 100644 index 0000000000000..becfe1fd9f5ad --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -0,0 +1,83 @@ +/*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) + +#include + +#include "InstrProfiling.h" + +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) +#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) +#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) + +/* Declare section start and stop symbols for various sections + * generated by compiler instrumentation. + */ +extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; +extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; +extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; +extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY; +extern char PROF_NAME_START COMPILER_RT_VISIBILITY; +extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; +extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY; +extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY; + +/* Add dummy data to ensure the section is always created. */ +__llvm_profile_data + __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME); +uint64_t + __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME); +uint32_t + __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME); +char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME); +ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME); + +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_begin_data(void) { + return &PROF_DATA_START; +} +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_end_data(void) { + return &PROF_DATA_STOP; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { + return &PROF_NAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { + return &PROF_NAME_STOP; +} +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { + return &PROF_CNTS_START; +} +COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { + return &PROF_CNTS_STOP; +} +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { + return &PROF_ORDERFILE_START; +} + +COMPILER_RT_VISIBILITY ValueProfNode * +__llvm_profile_begin_vnodes(void) { + return &PROF_VNODES_START; +} +COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) { + return &PROF_VNODES_STOP; +} +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP; + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c new file mode 100644 index 0000000000000..56c5d8378c29d --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -0,0 +1,100 @@ +/*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \ + !defined(_WIN32) + +#include +#include + +#include "InstrProfiling.h" + +static const __llvm_profile_data *DataFirst = NULL; +static const __llvm_profile_data *DataLast = NULL; +static const char *NamesFirst = NULL; +static const char *NamesLast = NULL; +static uint64_t *CountersFirst = NULL; +static uint64_t *CountersLast = NULL; +static uint32_t *OrderFileFirst = NULL; + +static const void *getMinAddr(const void *A1, const void *A2) { + return A1 < A2 ? A1 : A2; +} + +static const void *getMaxAddr(const void *A1, const void *A2) { + return A1 > A2 ? A1 : A2; +} + +/*! + * \brief Register an instrumented function. + * + * Calls to this are emitted by clang with -fprofile-instr-generate. Such + * calls are only required (and only emitted) on targets where we haven't + * implemented linker magic to find the bounds of the sections. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_register_function(void *Data_) { + /* TODO: Only emit this function if we can't use linker magic. */ + const __llvm_profile_data *Data = (__llvm_profile_data *)Data_; + if (!DataFirst) { + DataFirst = Data; + DataLast = Data + 1; + CountersFirst = Data->CounterPtr; + CountersLast = (uint64_t *)Data->CounterPtr + Data->NumCounters; + return; + } + + DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); + CountersFirst = (uint64_t *)getMinAddr(CountersFirst, Data->CounterPtr); + + DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); + CountersLast = (uint64_t *)getMaxAddr( + CountersLast, (uint64_t *)Data->CounterPtr + Data->NumCounters); +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_register_names_function(void *NamesStart, + uint64_t NamesSize) { + if (!NamesFirst) { + NamesFirst = (const char *)NamesStart; + NamesLast = (const char *)NamesStart + NamesSize; + return; + } + NamesFirst = (const char *)getMinAddr(NamesFirst, NamesStart); + NamesLast = + (const char *)getMaxAddr(NamesLast, (const char *)NamesStart + NamesSize); +} + +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; } +COMPILER_RT_VISIBILITY +const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_names(void) { return NamesFirst; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_names(void) { return NamesLast; } +COMPILER_RT_VISIBILITY +uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } +COMPILER_RT_VISIBILITY +uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +/* TODO: correctly set up OrderFileFirst. */ +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } + +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_begin_vnodes(void) { + return 0; +} +COMPILER_RT_VISIBILITY +ValueProfNode *__llvm_profile_end_vnodes(void) { return 0; } + +COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = 0; +COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = 0; + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c new file mode 100644 index 0000000000000..81b708bb2a38f --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -0,0 +1,68 @@ +/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(_WIN32) + +#if defined(_MSC_VER) +/* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") +#pragma comment(linker, "/MERGE:.lprfv=.data") +#pragma comment(linker, "/MERGE:.lprfnd=.data") +/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find + * after the fact. + */ + +/* Allocate read-only section bounds. */ +#pragma section(".lprfn$A", read) +#pragma section(".lprfn$Z", read) + +/* Allocate read-write section bounds. */ +#pragma section(".lprfd$A", read, write) +#pragma section(".lprfd$Z", read, write) +#pragma section(".lprfc$A", read, write) +#pragma section(".lprfc$Z", read, write) +#pragma section(".lorderfile$A", read, write) +#pragma section(".lprfnd$A", read, write) +#pragma section(".lprfnd$Z", read, write) +#endif + +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0}; +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; + +const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; +const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; + +uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; +uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; + +ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; +ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd; + +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart + 1; +} +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } + +const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } +const char *__llvm_profile_end_names(void) { return &NamesEnd; } + +uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +ValueProfNode *CurrentVNode = &VNodesStart + 1; +ValueProfNode *EndVNode = &VNodesEnd; + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h new file mode 100644 index 0000000000000..20cf5d660c6ae --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h @@ -0,0 +1,144 @@ +/*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +/* This header must be included after all others so it can provide fallback + definitions for stuff missing in system headers. */ + +#ifndef PROFILE_INSTRPROFILING_PORT_H_ +#define PROFILE_INSTRPROFILING_PORT_H_ + +#ifdef _MSC_VER +#define COMPILER_RT_ALIGNAS(x) __declspec(align(x)) +#define COMPILER_RT_VISIBILITY +/* FIXME: selectany does not have the same semantics as weak. */ +#define COMPILER_RT_WEAK __declspec(selectany) +/* Need to include */ +#define COMPILER_RT_ALLOCA _alloca +/* Need to include and */ +#define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l) +#define COMPILER_RT_ALWAYS_INLINE __forceinline +#define COMPILER_RT_CLEANUP(x) +#elif __GNUC__ +#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) +#define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden"))) +#define COMPILER_RT_WEAK __attribute__((weak)) +#define COMPILER_RT_ALLOCA __builtin_alloca +#define COMPILER_RT_FTRUNCATE(f,l) ftruncate(fileno(f),l) +#define COMPILER_RT_ALWAYS_INLINE inline __attribute((always_inline)) +#define COMPILER_RT_CLEANUP(x) __attribute__((cleanup(x))) +#endif + +#if defined(__APPLE__) +#define COMPILER_RT_SEG "__DATA," +#else +#define COMPILER_RT_SEG "" +#endif + +#ifdef _MSC_VER +#define COMPILER_RT_SECTION(Sect) __declspec(allocate(Sect)) +#else +#define COMPILER_RT_SECTION(Sect) __attribute__((section(Sect))) +#endif + +#define COMPILER_RT_MAX_HOSTLEN 128 +#ifdef __ORBIS__ +#define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1)) +#else +#define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len) +#endif + +#if COMPILER_RT_HAS_ATOMICS == 1 +#ifdef _MSC_VER +#include +#if _MSC_VER < 1900 +#define snprintf _snprintf +#endif +#if defined(_WIN64) +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange64((LONGLONG volatile *)Ptr, (LONGLONG)NewV, \ + (LONGLONG)OldV) == (LONGLONG)OldV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)InterlockedExchangeAdd64((LONGLONG volatile *)&PtrVar, \ + (LONGLONG)sizeof(DomType) * PtrIncr) +#else /* !defined(_WIN64) */ +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + (InterlockedCompareExchange((LONG volatile *)Ptr, (LONG)NewV, (LONG)OldV) == \ + (LONG)OldV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)InterlockedExchangeAdd((LONG volatile *)&PtrVar, \ + (LONG)sizeof(DomType) * PtrIncr) +#endif +#else /* !defined(_MSC_VER) */ +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + __sync_bool_compare_and_swap(Ptr, OldV, NewV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)__sync_fetch_and_add((long *)&PtrVar, sizeof(DomType) * PtrIncr) +#endif +#else /* COMPILER_RT_HAS_ATOMICS != 1 */ +#include "InstrProfilingUtil.h" +#define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ + lprofBoolCmpXchg((void **)Ptr, OldV, NewV) +#define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ + (DomType *)lprofPtrFetchAdd((void **)&PtrVar, sizeof(DomType) * PtrIncr) +#endif + +#if defined(_WIN32) +#define DIR_SEPARATOR '\\' +#define DIR_SEPARATOR_2 '/' +#else +#define DIR_SEPARATOR '/' +#endif + +#ifndef DIR_SEPARATOR_2 +#define IS_DIR_SEPARATOR(ch) ((ch) == DIR_SEPARATOR) +#else /* DIR_SEPARATOR_2 */ +#define IS_DIR_SEPARATOR(ch) \ + (((ch) == DIR_SEPARATOR) || ((ch) == DIR_SEPARATOR_2)) +#endif /* DIR_SEPARATOR_2 */ + +#if defined(_WIN32) +#include +static inline size_t getpagesize() { + SYSTEM_INFO S; + GetNativeSystemInfo(&S); + return S.dwPageSize; +} +#else /* defined(_WIN32) */ +#include +#endif /* defined(_WIN32) */ + +#define PROF_ERR(Format, ...) \ + fprintf(stderr, "LLVM Profile Error: " Format, __VA_ARGS__); + +#define PROF_WARN(Format, ...) \ + fprintf(stderr, "LLVM Profile Warning: " Format, __VA_ARGS__); + +#define PROF_NOTE(Format, ...) \ + fprintf(stderr, "LLVM Profile Note: " Format, __VA_ARGS__); + +#ifndef MAP_FILE +#define MAP_FILE 0 +#endif + +#ifndef O_BINARY +#define O_BINARY 0 +#endif + +#if defined(__FreeBSD__) + +#include +#include + +#else /* defined(__FreeBSD__) */ + +#include +#include + +#endif /* defined(__FreeBSD__) && defined(__i386__) */ + +#endif /* PROFILE_INSTRPROFILING_PORT_H_ */ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp new file mode 100644 index 0000000000000..5dff09d706329 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp @@ -0,0 +1,30 @@ +//===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +extern "C" { + +#include "InstrProfiling.h" + +/* int __llvm_profile_runtime */ +COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR; +} + +namespace { + +class RegisterRuntime { +public: + RegisterRuntime() { + __llvm_profile_initialize_file(); + if (!__llvm_profile_is_continuous_mode_enabled()) + __llvm_profile_register_write_file_atexit(); + } +}; + +RegisterRuntime Registration; + +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c new file mode 100644 index 0000000000000..bf5a9670fe18c --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -0,0 +1,332 @@ +/*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifdef _WIN32 +#include +#include +#include +#include "WindowsMMap.h" +#else +#include +#include +#include +#include +#include +#include +#endif + +#ifdef COMPILER_RT_HAS_UNAME +#include +#endif + +#include +#include + +#if defined(__linux__) +#include +#include +#endif + +#include "InstrProfiling.h" +#include "InstrProfilingUtil.h" + +COMPILER_RT_WEAK unsigned lprofDirMode = 0755; + +COMPILER_RT_VISIBILITY +void __llvm_profile_recursive_mkdir(char *path) { + int i; + int start = 1; + +#if defined(__ANDROID__) && defined(__ANDROID_API__) && \ + defined(__ANDROID_API_FUTURE__) && \ + __ANDROID_API__ == __ANDROID_API_FUTURE__ + // Avoid spammy selinux denial messages in Android by not attempting to + // create directories in GCOV_PREFIX. These denials occur when creating (or + // even attempting to stat()) top-level directories like "/data". + // + // Do so by ignoring ${GCOV_PREFIX} when invoking mkdir(). + const char *gcov_prefix = getenv("GCOV_PREFIX"); + if (gcov_prefix != NULL) { + const int gcov_prefix_len = strlen(gcov_prefix); + if (strncmp(path, gcov_prefix, gcov_prefix_len) == 0) + start = gcov_prefix_len; + } +#endif + + for (i = start; path[i] != '\0'; ++i) { + char save = path[i]; + if (!IS_DIR_SEPARATOR(path[i])) + continue; + path[i] = '\0'; +#ifdef _WIN32 + _mkdir(path); +#else + /* Some of these will fail, ignore it. */ + mkdir(path, __llvm_profile_get_dir_mode()); +#endif + path[i] = save; + } +} + +COMPILER_RT_VISIBILITY +void __llvm_profile_set_dir_mode(unsigned Mode) { lprofDirMode = Mode; } + +COMPILER_RT_VISIBILITY +unsigned __llvm_profile_get_dir_mode(void) { return lprofDirMode; } + +#if COMPILER_RT_HAS_ATOMICS != 1 +COMPILER_RT_VISIBILITY +uint32_t lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV) { + void *R = *Ptr; + if (R == OldV) { + *Ptr = NewV; + return 1; + } + return 0; +} +COMPILER_RT_VISIBILITY +void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { + void *Old = *Mem; + *((char **)Mem) += ByteIncr; + return Old; +} + +#endif + +#ifdef _WIN32 +COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { + WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; + DWORD BufferSize = sizeof(Buffer); + BOOL Result = + GetComputerNameExW(ComputerNameDnsFullyQualified, Buffer, &BufferSize); + if (!Result) + return -1; + if (WideCharToMultiByte(CP_UTF8, 0, Buffer, -1, Name, Len, NULL, NULL) == 0) + return -1; + return 0; +} +#elif defined(COMPILER_RT_HAS_UNAME) +COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { + struct utsname N; + int R = uname(&N); + if (R >= 0) { + strncpy(Name, N.nodename, Len); + return 0; + } + return R; +} +#endif + +COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_WRLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_EX); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { +#ifdef COMPILER_RT_HAS_FCNTL_LCK + struct flock s_flock; + + s_flock.l_whence = SEEK_SET; + s_flock.l_start = 0; + s_flock.l_len = 0; /* Until EOF. */ + s_flock.l_pid = getpid(); + s_flock.l_type = F_UNLCK; + + while (fcntl(fd, F_SETLKW, &s_flock) == -1) { + if (errno != EINTR) { + if (errno == ENOLCK) { + return -1; + } + break; + } + } + return 0; +#else + flock(fd, LOCK_UN); + return 0; +#endif +} + +COMPILER_RT_VISIBILITY int lprofLockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofLockFd(fd); +} + +COMPILER_RT_VISIBILITY int lprofUnlockFileHandle(FILE *F) { + int fd; +#if defined(_WIN32) + fd = _fileno(F); +#else + fd = fileno(F); +#endif + return lprofUnlockFd(fd); +} + +COMPILER_RT_VISIBILITY FILE *lprofOpenFileEx(const char *ProfileName) { + FILE *f; + int fd; +#ifdef COMPILER_RT_HAS_FCNTL_LCK + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + + f = fdopen(fd, "r+b"); +#elif defined(_WIN32) + // FIXME: Use the wide variants to handle Unicode filenames. + HANDLE h = CreateFileA(ProfileName, GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_ALWAYS, + FILE_ATTRIBUTE_NORMAL, 0); + if (h == INVALID_HANDLE_VALUE) + return NULL; + + fd = _open_osfhandle((intptr_t)h, 0); + if (fd == -1) { + CloseHandle(h); + return NULL; + } + + if (lprofLockFd(fd) != 0) + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + + f = _fdopen(fd, "r+b"); + if (f == 0) { + CloseHandle(h); + return NULL; + } +#else + /* Worst case no locking applied. */ + PROF_WARN("Concurrent file access is not supported : %s\n", + "lack file locking"); + fd = open(ProfileName, O_RDWR | O_CREAT, 0666); + if (fd < 0) + return NULL; + f = fdopen(fd, "r+b"); +#endif + + return f; +} + +COMPILER_RT_VISIBILITY const char *lprofGetPathPrefix(int *PrefixStrip, + size_t *PrefixLen) { + const char *Prefix = getenv("GCOV_PREFIX"); + const char *PrefixStripStr = getenv("GCOV_PREFIX_STRIP"); + + *PrefixLen = 0; + *PrefixStrip = 0; + if (Prefix == NULL || Prefix[0] == '\0') + return NULL; + + if (PrefixStripStr) { + *PrefixStrip = atoi(PrefixStripStr); + + /* Negative GCOV_PREFIX_STRIP values are ignored */ + if (*PrefixStrip < 0) + *PrefixStrip = 0; + } else { + *PrefixStrip = 0; + } + *PrefixLen = strlen(Prefix); + + return Prefix; +} + +COMPILER_RT_VISIBILITY void +lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix, + size_t PrefixLen, int PrefixStrip) { + + const char *Ptr; + int Level; + const char *StrippedPathStr = PathStr; + + for (Level = 0, Ptr = PathStr + 1; Level < PrefixStrip; ++Ptr) { + if (*Ptr == '\0') + break; + + if (!IS_DIR_SEPARATOR(*Ptr)) + continue; + + StrippedPathStr = Ptr; + ++Level; + } + + memcpy(Dest, Prefix, PrefixLen); + + if (!IS_DIR_SEPARATOR(Prefix[PrefixLen - 1])) + Dest[PrefixLen++] = DIR_SEPARATOR; + + memcpy(Dest + PrefixLen, StrippedPathStr, strlen(StrippedPathStr) + 1); +} + +COMPILER_RT_VISIBILITY const char * +lprofFindFirstDirSeparator(const char *Path) { + const char *Sep = strchr(Path, DIR_SEPARATOR); +#if defined(DIR_SEPARATOR_2) + const char *Sep2 = strchr(Path, DIR_SEPARATOR_2); + if (Sep2 && (!Sep || Sep2 < Sep)) + Sep = Sep2; +#endif + return Sep; +} + +COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) { + const char *Sep = strrchr(Path, DIR_SEPARATOR); +#if defined(DIR_SEPARATOR_2) + const char *Sep2 = strrchr(Path, DIR_SEPARATOR_2); + if (Sep2 && (!Sep || Sep2 > Sep)) + Sep = Sep2; +#endif + return Sep; +} + +COMPILER_RT_VISIBILITY int lprofSuspendSigKill() { +#if defined(__linux__) + int PDeachSig = 0; + /* Temporarily suspend getting SIGKILL upon exit of the parent process. */ + if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL) + prctl(PR_SET_PDEATHSIG, 0); + return (PDeachSig == SIGKILL); +#else + return 0; +#endif +} + +COMPILER_RT_VISIBILITY void lprofRestoreSigKill() { +#if defined(__linux__) + prctl(PR_SET_PDEATHSIG, SIGKILL); +#endif +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h new file mode 100644 index 0000000000000..f0e29a8803a05 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -0,0 +1,74 @@ +/*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILINGUTIL_H +#define PROFILE_INSTRPROFILINGUTIL_H + +#include +#include + +/*! \brief Create a directory tree. */ +void __llvm_profile_recursive_mkdir(char *Pathname); + +/*! Set the mode used when creating profile directories. */ +void __llvm_profile_set_dir_mode(unsigned Mode); + +/*! Return the directory creation mode. */ +unsigned __llvm_profile_get_dir_mode(void); + +int lprofLockFd(int fd); +int lprofUnlockFd(int fd); +int lprofLockFileHandle(FILE *F); +int lprofUnlockFileHandle(FILE *F); + +/*! Open file \c Filename for read+write with write + * lock for exclusive access. The caller will block + * if the lock is already held by another process. */ +FILE *lprofOpenFileEx(const char *Filename); +/* PS4 doesn't have setenv/getenv. Define a shim. */ +#if __ORBIS__ +static inline char *getenv(const char *name) { return NULL; } +static inline int setenv(const char *name, const char *value, int overwrite) +{ return 0; } +#endif /* #if __ORBIS__ */ + +/* GCOV_PREFIX and GCOV_PREFIX_STRIP support */ +/* Return the path prefix specified by GCOV_PREFIX environment variable. + * If GCOV_PREFIX_STRIP is also specified, the strip level (integer value) + * is returned via \c *PrefixStrip. The prefix length is stored in *PrefixLen. + */ +const char *lprofGetPathPrefix(int *PrefixStrip, size_t *PrefixLen); +/* Apply the path prefix specified in \c Prefix to path string in \c PathStr, + * and store the result to buffer pointed to by \c Buffer. If \c PrefixStrip + * is not zero, path prefixes are stripped from \c PathStr (the level of + * stripping is specified by \c PrefixStrip) before \c Prefix is added. + */ +void lprofApplyPathPrefix(char *Dest, const char *PathStr, const char *Prefix, + size_t PrefixLen, int PrefixStrip); + +/* Returns a pointer to the first occurrence of \c DIR_SEPARATOR char in + * the string \c Path, or NULL if the char is not found. */ +const char *lprofFindFirstDirSeparator(const char *Path); +/* Returns a pointer to the last occurrence of \c DIR_SEPARATOR char in + * the string \c Path, or NULL if the char is not found. */ +const char *lprofFindLastDirSeparator(const char *Path); + +int lprofGetHostName(char *Name, int Len); + +unsigned lprofBoolCmpXchg(void **Ptr, void *OldV, void *NewV); +void *lprofPtrFetchAdd(void **Mem, long ByteIncr); + +/* Temporarily suspend SIGKILL. Return value of 1 means a restore is needed. + * Other return values mean no restore is needed. + */ +int lprofSuspendSigKill(); + +/* Restore previously suspended SIGKILL. */ +void lprofRestoreSigKill(); + +#endif /* PROFILE_INSTRPROFILINGUTIL_H */ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c new file mode 100644 index 0000000000000..fd53cac3dff31 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -0,0 +1,371 @@ +/*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include +#include +#include +#include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingUtil.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#define INSTR_PROF_COMMON_API_IMPL +#include "profile/InstrProfData.inc" + +static int hasStaticCounters = 1; +static int OutOfNodesWarnings = 0; +static int hasNonDefaultValsPerSite = 0; +#define INSTR_PROF_MAX_VP_WARNS 10 +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16 +#define INSTR_PROF_VNODE_POOL_SIZE 1024 + +#ifndef _MSC_VER +/* A shared static pool in addition to the vnodes statically + * allocated by the compiler. */ +COMPILER_RT_VISIBILITY ValueProfNode + lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); +#endif + +COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = + INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE; + +COMPILER_RT_VISIBILITY void lprofSetupValueProfiler() { + const char *Str = 0; + Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE"); + if (Str && Str[0]) { + VPMaxNumValsPerSite = atoi(Str); + hasNonDefaultValsPerSite = 1; + } + if (VPMaxNumValsPerSite > INSTR_PROF_MAX_NUM_VAL_PER_SITE) + VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; +} + +COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) { + VPMaxNumValsPerSite = MaxVals; + hasNonDefaultValsPerSite = 1; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void +__llvm_profile_set_num_value_sites(__llvm_profile_data *Data, + uint32_t ValueKind, uint16_t NumValueSites) { + *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY const __llvm_profile_data * +__llvm_profile_iterate_data(const __llvm_profile_data *Data) { + return Data + 1; +} + +/* This method is only used in value profiler mock testing. */ +COMPILER_RT_VISIBILITY void * +__llvm_get_function_addr(const __llvm_profile_data *Data) { + return Data->FunctionPointer; +} + +/* Allocate an array that holds the pointers to the linked lists of + * value profile counter nodes. The number of element of the array + * is the total number of value profile sites instrumented. Returns + * 0 if allocation fails. + */ + +static int allocateValueProfileCounters(__llvm_profile_data *Data) { + uint64_t NumVSites = 0; + uint32_t VKI; + + /* This function will never be called when value site array is allocated + statically at compile time. */ + hasStaticCounters = 0; + /* When dynamic allocation is enabled, allow tracking the max number of + * values allowd. */ + if (!hasNonDefaultValsPerSite) + VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE; + + for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) + NumVSites += Data->NumValueSites[VKI]; + + ValueProfNode **Mem = + (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); + if (!Mem) + return 0; + if (!COMPILER_RT_BOOL_CMPXCHG(&Data->Values, 0, Mem)) { + free(Mem); + return 0; + } + return 1; +} + +static ValueProfNode *allocateOneNode(void) { + ValueProfNode *Node; + + if (!hasStaticCounters) + return (ValueProfNode *)calloc(1, sizeof(ValueProfNode)); + + /* Early check to avoid value wrapping around. */ + if (CurrentVNode + 1 > EndVNode) { + if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) { + PROF_WARN("Unable to track new values: %s. " + " Consider using option -mllvm -vp-counters-per-site= to " + "allocate more" + " value profile counters at compile time. \n", + "Running out of static counters"); + } + return 0; + } + Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1); + /* Due to section padding, EndVNode point to a byte which is one pass + * an incomplete VNode, so we need to skip the last incomplete node. */ + if (Node + 1 > EndVNode) + return 0; + + return Node; +} + +static COMPILER_RT_ALWAYS_INLINE void +instrumentTargetValueImpl(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, uint64_t CountValue) { + __llvm_profile_data *PData = (__llvm_profile_data *)Data; + if (!PData) + return; + if (!CountValue) + return; + if (!PData->Values) { + if (!allocateValueProfileCounters(PData)) + return; + } + + ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values; + ValueProfNode *PrevVNode = NULL; + ValueProfNode *MinCountVNode = NULL; + ValueProfNode *CurVNode = ValueCounters[CounterIndex]; + uint64_t MinCount = UINT64_MAX; + + uint8_t VDataCount = 0; + while (CurVNode) { + if (TargetValue == CurVNode->Value) { + CurVNode->Count += CountValue; + return; + } + if (CurVNode->Count < MinCount) { + MinCount = CurVNode->Count; + MinCountVNode = CurVNode; + } + PrevVNode = CurVNode; + CurVNode = CurVNode->Next; + ++VDataCount; + } + + if (VDataCount >= VPMaxNumValsPerSite) { + /* Bump down the min count node's count. If it reaches 0, + * evict it. This eviction/replacement policy makes hot + * targets more sticky while cold targets less so. In other + * words, it makes it less likely for the hot targets to be + * prematurally evicted during warmup/establishment period, + * when their counts are still low. In a special case when + * the number of values tracked is reduced to only one, this + * policy will guarantee that the dominating target with >50% + * total count will survive in the end. Note that this scheme + * allows the runtime to track the min count node in an adaptive + * manner. It can correct previous mistakes and eventually + * lock on a cold target that is alread in stable state. + * + * In very rare cases, this replacement scheme may still lead + * to target loss. For instance, out of \c N value slots, \c N-1 + * slots are occupied by luke warm targets during the warmup + * period and the remaining one slot is competed by two or more + * very hot targets. If those hot targets occur in an interleaved + * way, none of them will survive (gain enough weight to throw out + * other established entries) due to the ping-pong effect. + * To handle this situation, user can choose to increase the max + * number of tracked values per value site. Alternatively, a more + * expensive eviction mechanism can be implemented. It requires + * the runtime to track the total number of evictions per-site. + * When the total number of evictions reaches certain threshold, + * the runtime can wipe out more than one lowest count entries + * to give space for hot targets. + */ + if (MinCountVNode->Count <= CountValue) { + CurVNode = MinCountVNode; + CurVNode->Value = TargetValue; + CurVNode->Count = CountValue; + } else + MinCountVNode->Count -= CountValue; + + return; + } + + CurVNode = allocateOneNode(); + if (!CurVNode) + return; + CurVNode->Value = TargetValue; + CurVNode->Count += CountValue; + + uint32_t Success = 0; + if (!ValueCounters[CounterIndex]) + Success = + COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurVNode); + else if (PrevVNode && !PrevVNode->Next) + Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurVNode); + + if (!Success && !hasStaticCounters) { + free(CurVNode); + return; + } +} + +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_target(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + instrumentTargetValueImpl(TargetValue, Data, CounterIndex, 1); +} +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, + uint32_t CounterIndex, + uint64_t CountValue) { + instrumentTargetValueImpl(TargetValue, Data, CounterIndex, CountValue); +} + +/* + * The target values are partitioned into multiple regions/ranges. There is one + * contiguous region which is precise -- every value in the range is tracked + * individually. A value outside the precise region will be collapsed into one + * value depending on the region it falls in. + * + * There are three regions: + * 1. (-inf, PreciseRangeStart) and (PreciseRangeLast, LargeRangeValue) belong + * to one region -- all values here should be mapped to one value of + * "PreciseRangeLast + 1". + * 2. [PreciseRangeStart, PreciseRangeLast] + * 3. Large values: [LargeValue, +inf) maps to one value of LargeValue. + * + * The range for large values is optional. The default value of INT64_MIN + * indicates it is not specified. + */ +COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range( + uint64_t TargetValue, void *Data, uint32_t CounterIndex, + int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) { + + if (LargeValue != INT64_MIN && (int64_t)TargetValue >= LargeValue) + TargetValue = LargeValue; + else if ((int64_t)TargetValue < PreciseRangeStart || + (int64_t)TargetValue > PreciseRangeLast) + TargetValue = PreciseRangeLast + 1; + + __llvm_profile_instrument_target(TargetValue, Data, CounterIndex); +} + +/* + * A wrapper struct that represents value profile runtime data. + * Like InstrProfRecord class which is used by profiling host tools, + * ValueProfRuntimeRecord also implements the abstract intefaces defined in + * ValueProfRecordClosure so that the runtime data can be serialized using + * shared C implementation. + */ +typedef struct ValueProfRuntimeRecord { + const __llvm_profile_data *Data; + ValueProfNode **NodesKind[IPVK_Last + 1]; + uint8_t **SiteCountArray; +} ValueProfRuntimeRecord; + +/* ValueProfRecordClosure Interface implementation. */ + +static uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { + return ((const ValueProfRuntimeRecord *)R)->Data->NumValueSites[VK]; +} + +static uint32_t getNumValueDataRT(const void *R, uint32_t VK) { + uint32_t S = 0, I; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR) + return 0; + for (I = 0; I < Record->Data->NumValueSites[VK]; I++) + S += Record->SiteCountArray[VK][I]; + return S; +} + +static uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, + uint32_t S) { + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + return Record->SiteCountArray[VK][S]; +} + +static ValueProfRuntimeRecord RTRecord; +static ValueProfRecordClosure RTRecordClosure = { + &RTRecord, INSTR_PROF_NULLPTR, /* GetNumValueKinds */ + getNumValueSitesRT, getNumValueDataRT, getNumValueDataForSiteRT, + INSTR_PROF_NULLPTR, /* RemapValueData */ + INSTR_PROF_NULLPTR, /* GetValueForSite, */ + INSTR_PROF_NULLPTR /* AllocValueProfData */ +}; + +static uint32_t +initializeValueProfRuntimeRecord(const __llvm_profile_data *Data, + uint8_t *SiteCountArray[]) { + unsigned I, J, S = 0, NumValueKinds = 0; + ValueProfNode **Nodes = (ValueProfNode **)Data->Values; + RTRecord.Data = Data; + RTRecord.SiteCountArray = SiteCountArray; + for (I = 0; I <= IPVK_Last; I++) { + uint16_t N = Data->NumValueSites[I]; + if (!N) + continue; + + NumValueKinds++; + + RTRecord.NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR; + for (J = 0; J < N; J++) { + /* Compute value count for each site. */ + uint32_t C = 0; + ValueProfNode *Site = + Nodes ? RTRecord.NodesKind[I][J] : INSTR_PROF_NULLPTR; + while (Site) { + C++; + Site = Site->Next; + } + if (C > UCHAR_MAX) + C = UCHAR_MAX; + RTRecord.SiteCountArray[I][J] = C; + } + S += N; + } + return NumValueKinds; +} + +static ValueProfNode *getNextNValueData(uint32_t VK, uint32_t Site, + InstrProfValueData *Dst, + ValueProfNode *StartNode, uint32_t N) { + unsigned I; + ValueProfNode *VNode = StartNode ? StartNode : RTRecord.NodesKind[VK][Site]; + for (I = 0; I < N; I++) { + Dst[I].Value = VNode->Value; + Dst[I].Count = VNode->Count; + VNode = VNode->Next; + } + return VNode; +} + +static uint32_t getValueProfDataSizeWrapper(void) { + return getValueProfDataSize(&RTRecordClosure); +} + +static uint32_t getNumValueDataForSiteWrapper(uint32_t VK, uint32_t S) { + return getNumValueDataForSiteRT(&RTRecord, VK, S); +} + +static VPDataReaderType TheVPDataReader = { + initializeValueProfRuntimeRecord, getValueProfRecordHeaderSize, + getFirstValueProfRecord, getNumValueDataForSiteWrapper, + getValueProfDataSizeWrapper, getNextNValueData}; + +COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader() { + return &TheVPDataReader; +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c new file mode 100644 index 0000000000000..c34e110a69590 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -0,0 +1,300 @@ +/*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifdef _MSC_VER +/* For _alloca */ +#include +#endif +#include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" +#include "InstrProfilingPort.h" + +#define INSTR_PROF_VALUE_PROF_DATA +#include "profile/InstrProfData.inc" + +COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL; +static ProfBufferIO TheBufferIO; +#define VP_BUFFER_SIZE 8 * 1024 +static uint8_t BufferIOBuffer[VP_BUFFER_SIZE]; +static InstrProfValueData VPDataArray[16]; +static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); + +COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; +COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; + +/* The buffer writer is reponsponsible in keeping writer state + * across the call. + */ +COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, + ProfDataIOVec *IOVecs, + uint32_t NumIOVecs) { + uint32_t I; + char **Buffer = (char **)&This->WriterCtx; + for (I = 0; I < NumIOVecs; I++) { + size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; + if (IOVecs[I].Data) + memcpy(*Buffer, IOVecs[I].Data, Length); + else if (IOVecs[I].UseZeroPadding) { + /* Allocating the buffer should zero fill. */ + } + *Buffer += Length; + } + return 0; +} + +static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter, + uint8_t *Buffer, uint32_t BufferSz) { + BufferIO->FileWriter = FileWriter; + BufferIO->OwnFileWriter = 0; + BufferIO->BufferStart = Buffer; + BufferIO->BufferSz = BufferSz; + BufferIO->CurOffset = 0; +} + +COMPILER_RT_VISIBILITY ProfBufferIO * +lprofCreateBufferIO(ProfDataWriter *FileWriter) { + uint8_t *Buffer = DynamicBufferIOBuffer; + uint32_t BufferSize = VPBufferSize; + if (!Buffer) { + Buffer = &BufferIOBuffer[0]; + BufferSize = sizeof(BufferIOBuffer); + } + llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize); + return &TheBufferIO; +} + +COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) { + if (BufferIO->OwnFileWriter) + FreeHook(BufferIO->FileWriter); + if (DynamicBufferIOBuffer) { + FreeHook(DynamicBufferIOBuffer); + DynamicBufferIOBuffer = 0; + VPBufferSize = 0; + } +} + +COMPILER_RT_VISIBILITY int +lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) { + /* Buffer is not large enough, it is time to flush. */ + if (Size + BufferIO->CurOffset > BufferIO->BufferSz) { + if (lprofBufferIOFlush(BufferIO) != 0) + return -1; + } + /* Special case, bypass the buffer completely. */ + ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}}; + if (Size > BufferIO->BufferSz) { + if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) + return -1; + } else { + /* Write the data to buffer */ + uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset; + ProfDataWriter BufferWriter; + initBufferWriter(&BufferWriter, (char *)Buffer); + lprofBufferWriter(&BufferWriter, IO, 1); + BufferIO->CurOffset = + (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart; + } + return 0; +} + +COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) { + if (BufferIO->CurOffset) { + ProfDataIOVec IO[] = { + {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}}; + if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1)) + return -1; + BufferIO->CurOffset = 0; + } + return 0; +} + +/* Write out value profile data for function specified with \c Data. + * The implementation does not use the method \c serializeValueProfData + * which depends on dynamic memory allocation. In this implementation, + * value profile data is written out to \c BufferIO piecemeal. + */ +static int writeOneValueProfData(ProfBufferIO *BufferIO, + VPDataReaderType *VPDataReader, + const __llvm_profile_data *Data) { + unsigned I, NumValueKinds = 0; + ValueProfData VPHeader; + uint8_t *SiteCountArray[IPVK_Last + 1]; + + for (I = 0; I <= IPVK_Last; I++) { + if (!Data->NumValueSites[I]) + SiteCountArray[I] = 0; + else { + uint32_t Sz = + VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - + offsetof(ValueProfRecord, SiteCountArray); + /* Only use alloca for this small byte array to avoid excessive + * stack growth. */ + SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz); + memset(SiteCountArray[I], 0, Sz); + } + } + + /* If NumValueKinds returned is 0, there is nothing to write, report + success and return. This should match the raw profile reader's behavior. */ + if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray))) + return 0; + + /* First write the header structure. */ + VPHeader.TotalSize = VPDataReader->GetValueProfDataSize(); + VPHeader.NumValueKinds = NumValueKinds; + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader, + sizeof(ValueProfData))) + return -1; + + /* Make sure nothing else needs to be written before value profile + * records. */ + if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) != + (void *)(&VPHeader + 1)) + return -1; + + /* Write out the value profile record for each value kind + * one by one. */ + for (I = 0; I <= IPVK_Last; I++) { + uint32_t J; + ValueProfRecord RecordHeader; + /* The size of the value prof record header without counting the + * site count array .*/ + uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray); + uint32_t SiteCountArraySize; + + if (!Data->NumValueSites[I]) + continue; + + /* Write out the record header. */ + RecordHeader.Kind = I; + RecordHeader.NumValueSites = Data->NumValueSites[I]; + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader, + RecordHeaderSize)) + return -1; + + /* Write out the site value count array including padding space. */ + SiteCountArraySize = + VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) - + RecordHeaderSize; + if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize)) + return -1; + + /* Write out the value profile data for each value site. */ + for (J = 0; J < Data->NumValueSites[I]; J++) { + uint32_t NRead, NRemain; + ValueProfNode *NextStartNode = 0; + NRemain = VPDataReader->GetNumValueDataForSite(I, J); + if (!NRemain) + continue; + /* Read and write out value data in small chunks till it is done. */ + do { + NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain); + NextStartNode = + VPDataReader->GetValueData(I, /* ValueKind */ + J, /* Site */ + &VPDataArray[0], NextStartNode, NRead); + if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0], + NRead * sizeof(InstrProfValueData))) + return -1; + NRemain -= NRead; + } while (NRemain != 0); + } + } + /* All done report success. */ + return 0; +} + +static int writeValueProfData(ProfDataWriter *Writer, + VPDataReaderType *VPDataReader, + const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd) { + ProfBufferIO *BufferIO; + const __llvm_profile_data *DI = 0; + + if (!VPDataReader) + return 0; + + BufferIO = lprofCreateBufferIO(Writer); + + for (DI = DataBegin; DI < DataEnd; DI++) { + if (writeOneValueProfData(BufferIO, VPDataReader, DI)) + return -1; + } + + if (lprofBufferIOFlush(BufferIO) != 0) + return -1; + lprofDeleteBufferIO(BufferIO); + + return 0; +} + +COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, + VPDataReaderType *VPDataReader, + int SkipNameDataWrite) { + /* Match logic in __llvm_profile_write_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const uint64_t *CountersBegin = __llvm_profile_begin_counters(); + const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, + CountersEnd, VPDataReader, NamesBegin, NamesEnd, + SkipNameDataWrite); +} + +COMPILER_RT_VISIBILITY int +lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, + const __llvm_profile_data *DataEnd, + const uint64_t *CountersBegin, const uint64_t *CountersEnd, + VPDataReaderType *VPDataReader, const char *NamesBegin, + const char *NamesEnd, int SkipNameDataWrite) { + + /* Calculate size of sections. */ + const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t CountersSize = CountersEnd - CountersBegin; + const uint64_t NamesSize = NamesEnd - NamesBegin; + + /* Create the header. */ + __llvm_profile_header Header; + + if (!DataSize) + return 0; + + /* Determine how much padding is needed before/after the counters and after + * the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + +/* Initialize header structure. */ +#define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; +#include "profile/InstrProfData.inc" + + /* Write the data. */ + ProfDataIOVec IOVec[] = { + {&Header, sizeof(__llvm_profile_header), 1, 0}, + {DataBegin, sizeof(__llvm_profile_data), DataSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, + {CountersBegin, sizeof(uint64_t), CountersSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; + if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) + return -1; + + /* Value profiling is not yet supported in continuous mode. */ + if (__llvm_profile_is_continuous_mode_enabled()) + return 0; + + return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); +} diff --git a/system/lib/update_compiler_rt.py b/system/lib/update_compiler_rt.py index ee5fd64e34dba..dde32a059854c 100755 --- a/system/lib/update_compiler_rt.py +++ b/system/lib/update_compiler_rt.py @@ -21,6 +21,7 @@ ('lib', 'lsan'), ('lib', 'ubsan'), ('lib', 'ubsan_minimal'), + ('lib', 'coverage'), ] def clear(dirname): diff --git a/tests/test_core.py b/tests/test_core.py index ece09250aaa1d..447c9f66f74cc 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -28,6 +28,9 @@ from runner import NON_ZERO import clang_native +LLVM_PROFDATA = os.path.expanduser(shared.build_llvm_tool_path(shared.exe_suffix('llvm-profdata'))) +LLVM_COV = os.path.expanduser(shared.build_llvm_tool_path(shared.exe_suffix('llvm-cov'))) + # decorators for limiting which modes a test can run in @@ -3015,7 +3018,7 @@ def get_data_export_count(wasm): export_count = get_data_export_count('test_dlfcn_self.wasm') # ensure there aren't too many globals; we don't want unnamed_addr self.assertGreater(export_count, 20) - self.assertLess(export_count, 56) + self.assertLess(export_count, 64) @needs_dlfcn def test_dlfcn_unique_sig(self): @@ -8225,6 +8228,15 @@ def test_abort_on_exceptions(self): self.emcc_args += ['--bind', '--post-js', path_from_root('tests', 'core', 'test_abort_on_exception_post.js')] self.do_run_in_out_file_test('tests', 'core', 'test_abort_on_exception.cpp') + def test_fcoverage_mapping(self): + self.emcc_args += ['-fprofile-instr-generate', '-fcoverage-mapping', '-sNODERAWFS', '-sEXIT_RUNTIME'] + self.do_run_in_out_file_test('tests', 'core', 'test_hello_world.c') + self.assertExists('default.profraw') + self.run_process([LLVM_PROFDATA, 'merge', '-sparse', 'default.profraw', '-o', 'out.profdata']) + self.assertExists('out.profdata') + # TODO(sbc): fix llvm-cov so it can find the data sections in needs in the wasm file + # self.run_process([LLVM_COV, 'show', 'test_hello_world.wasm', '-instr-profile=out.profdata']) + # Generate tests for everything def make_run(name, emcc_args, settings=None, env=None): diff --git a/tools/shared.py b/tools/shared.py index aab5f2c42f429..9d4a0e964ca10 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -426,6 +426,9 @@ def emsdk_ldflags(user_args): ] ldflags = ['-L' + l for l in library_paths] + if '-fprofile-instr-generate' in user_args: + ldflags += ['-u', '__llvm_runtime_variable'] + if '-nostdlib' in user_args: return ldflags diff --git a/tools/system_libs.py b/tools/system_libs.py index f9ba2bd1df88c..7f263eddc8ea5 100755 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -645,9 +645,14 @@ class libcompiler_rt(MTLibrary): # restriction soon: https://reviews.llvm.org/D71738 force_object_files = True - cflags = ['-O2', '-fno-builtin'] + cflags = ['-O2', '-fno-builtin', + '-DCOMPILER_RT_HAS_ATOMICS=1', + '-DCOMPILER_RT_HAS_UNAME=1'] src_dir = ['system', 'lib', 'compiler-rt', 'lib', 'builtins'] src_files = glob_in_path(src_dir, '*.c') + profile = ['system', 'lib', 'compiler-rt', 'lib', 'profile'] + src_files += glob_in_path(profile, '*.c') + src_files += glob_in_path(profile, '*.cpp') src_files.append(shared.path_from_root('system', 'lib', 'compiler-rt', 'stack_ops.S')) src_files.append(shared.path_from_root('system', 'lib', 'compiler-rt', 'stack_limits.S')) src_files.append(shared.path_from_root('system', 'lib', 'compiler-rt', 'emscripten_setjmp.c')) From cf07e7e561fddf3d7d51d00a8bd24645ece6858d Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Wed, 16 Apr 2025 08:48:43 -0400 Subject: [PATCH 02/11] update --- system/lib/update_compiler_rt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/update_compiler_rt.py b/system/lib/update_compiler_rt.py index 29535ec9a68b0..0dabc5068154c 100755 --- a/system/lib/update_compiler_rt.py +++ b/system/lib/update_compiler_rt.py @@ -23,7 +23,7 @@ ('lib', 'lsan'), ('lib', 'ubsan'), ('lib', 'ubsan_minimal'), - ('lib', 'coverage'), + ('lib', 'profile'), ] preserve_files = ('readme.txt',) From 69756f84e1a58dece806410a9091fcad15ea712d Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Wed, 16 Apr 2025 10:09:52 -0400 Subject: [PATCH 03/11] updates --- .../include/profile/InstrProfData.inc | 368 +++++++++-- .../include/profile/MIBEntryDef.inc | 55 ++ .../include/profile/MemProfData.inc | 235 +++++++ .../include/profile/instr_prof_interface.h | 92 +++ .../compiler-rt/lib/profile/GCDAProfiling.c | 337 +++++----- .../compiler-rt/lib/profile/InstrProfiling.c | 51 +- .../compiler-rt/lib/profile/InstrProfiling.h | 189 +++--- .../lib/profile/InstrProfilingBuffer.c | 201 ++++-- .../lib/profile/InstrProfilingFile.c | 577 +++++++++++++----- .../lib/profile/InstrProfilingInternal.c | 26 + .../lib/profile/InstrProfilingInternal.h | 41 +- .../lib/profile/InstrProfilingMerge.c | 225 +++++-- .../lib/profile/InstrProfilingNameVar.c | 2 +- .../lib/profile/InstrProfilingPlatformAIX.c | 221 +++++++ .../profile/InstrProfilingPlatformDarwin.c | 50 +- .../profile/InstrProfilingPlatformFuchsia.c | 193 +++--- .../lib/profile/InstrProfilingPlatformLinux.c | 197 +++++- .../lib/profile/InstrProfilingPlatformOther.c | 50 +- .../profile/InstrProfilingPlatformWindows.c | 53 +- .../lib/profile/InstrProfilingPort.h | 18 +- .../lib/profile/InstrProfilingRuntime.cpp | 24 +- .../lib/profile/InstrProfilingUtil.c | 53 +- .../lib/profile/InstrProfilingUtil.h | 15 +- .../lib/profile/InstrProfilingValue.c | 57 +- .../lib/profile/InstrProfilingVersionVar.c | 18 + .../lib/profile/InstrProfilingWriter.c | 133 +++- .../lib/compiler-rt/lib/profile/WindowsMMap.c | 187 ++++++ .../lib/compiler-rt/lib/profile/WindowsMMap.h | 82 +++ 28 files changed, 2915 insertions(+), 835 deletions(-) create mode 100644 system/lib/compiler-rt/include/profile/MIBEntryDef.inc create mode 100644 system/lib/compiler-rt/include/profile/MemProfData.inc create mode 100644 system/lib/compiler-rt/include/profile/instr_prof_interface.h create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c create mode 100644 system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c create mode 100644 system/lib/compiler-rt/lib/profile/WindowsMMap.c create mode 100644 system/lib/compiler-rt/lib/profile/WindowsMMap.h diff --git a/system/lib/compiler-rt/include/profile/InstrProfData.inc b/system/lib/compiler-rt/include/profile/InstrProfData.inc index 99f41d8fef077..847e53cfa7432 100644 --- a/system/lib/compiler-rt/include/profile/InstrProfData.inc +++ b/system/lib/compiler-rt/include/profile/InstrProfData.inc @@ -6,15 +6,15 @@ |* \*===----------------------------------------------------------------------===*/ /* - * This is the master file that defines all the data structure, signature, + * This is the main file that defines all the data structure, signature, * constant literals that are shared across profiling runtime library, * compiler (instrumentation), and host tools (reader/writer). The entities * defined in this file affect the profile runtime ABI, the raw profile format, * or both. * - * The file has two identical copies. The master copy lives in LLVM and + * The file has two identical copies. The primary copy lives in LLVM and * the other one sits in compiler-rt/lib/profile directory. To make changes - * in this file, first modify the master copy and copy it over to compiler-rt. + * in this file, first modify the primary copy and copy it over to compiler-rt. * Testing of any change in this file can start only after the two copies are * synced up. * @@ -62,6 +62,8 @@ #define INSTR_PROF_VISIBILITY #endif +// clang-format off:consider re-enabling clang-format if auto-formatted C macros +// are readable (e.g., after `issue #82426` is fixed) /* INSTR_PROF_DATA start. */ /* Definition of member fields of the per-function control structure. */ #ifndef INSTR_PROF_DATA @@ -75,24 +77,44 @@ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ INSTR_PROF_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ Inc->getHash()->getZExtValue())) -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt64PtrTy(Ctx), CounterPtr, \ - ConstantExpr::getBitCast(CounterPtr, \ - llvm::Type::getInt64PtrTy(Ctx))) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, CounterPtr, RelativeCounterPtr) +INSTR_PROF_DATA(const IntPtrT, IntPtrTy, BitmapPtr, RelativeBitmapPtr) /* This is used to map function pointers for the indirect call targets to * function name hashes during the conversion from raw to merged profile * data. */ -INSTR_PROF_DATA(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), FunctionPointer, \ +INSTR_PROF_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), FunctionPointer, \ FunctionAddr) -INSTR_PROF_DATA(IntPtrT, llvm::Type::getInt8PtrTy(Ctx), Values, \ +INSTR_PROF_DATA(IntPtrT, llvm::PointerType::getUnqual(Ctx), Values, \ ValuesPtrExpr) INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumCounters, \ ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumCounters)) INSTR_PROF_DATA(const uint16_t, Int16ArrayTy, NumValueSites[IPVK_Last+1], \ - ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) + ConstantArray::get(Int16ArrayTy, Int16ArrayVals)) \ +INSTR_PROF_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), NumBitmapBytes, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), NumBitmapBytes)) #undef INSTR_PROF_DATA /* INSTR_PROF_DATA end. */ +/* For a virtual table object, record the name hash to associate profiled + * addresses with global variables, and record {starting address, size in bytes} + * to map the profiled virtual table (which usually have an offset from the + * starting address) back to a virtual table object. */ +#ifndef INSTR_PROF_VTABLE_DATA +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_VTABLE_DATA_DEFINED +#endif +INSTR_PROF_VTABLE_DATA(const uint64_t, llvm::Type::getInt64Ty(Ctx), \ + VTableNameHash, ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ + IndexedInstrProf::ComputeHash(PGOVTableName))) +INSTR_PROF_VTABLE_DATA(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ + VTablePointer, VTableAddr) +INSTR_PROF_VTABLE_DATA(const uint32_t, llvm::Type::getInt32Ty(Ctx), VTableSize, \ + ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ + VTableSizeVal)) +#undef INSTR_PROF_VTABLE_DATA +/* INSTR_PROF_VTABLE_DATA end. */ /* This is an internal data structure used by value profiler. It * is defined here to allow serialization code sharing by LLVM @@ -115,13 +137,15 @@ INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Value, \ ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) INSTR_PROF_VALUE_NODE(uint64_t, llvm::Type::getInt64Ty(Ctx), Count, \ ConstantInt::get(llvm::Type::GetInt64Ty(Ctx), 0)) -INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ - ConstantInt::get(llvm::Type::GetInt8PtrTy(Ctx), 0)) +INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::PointerType::getUnqual(Ctx), Next, \ + ConstantInt::get(llvm::PointerType::getUnqual(Ctx), 0)) #undef INSTR_PROF_VALUE_NODE /* INSTR_PROF_VALUE_NODE end. */ /* INSTR_PROF_RAW_HEADER start */ /* Definition of member fields of the raw profile header data structure. */ +/* Please update llvm/docs/InstrProfileFormat.rst as appropriate when updating + raw profile format. */ #ifndef INSTR_PROF_RAW_HEADER #define INSTR_PROF_RAW_HEADER(Type, Name, Initializer) #else @@ -129,13 +153,21 @@ INSTR_PROF_VALUE_NODE(PtrToNodeT, llvm::Type::getInt8PtrTy(Ctx), Next, \ #endif INSTR_PROF_RAW_HEADER(uint64_t, Magic, __llvm_profile_get_magic()) INSTR_PROF_RAW_HEADER(uint64_t, Version, __llvm_profile_get_version()) -INSTR_PROF_RAW_HEADER(uint64_t, DataSize, DataSize) +INSTR_PROF_RAW_HEADER(uint64_t, BinaryIdsSize, __llvm_write_binary_ids(NULL)) +INSTR_PROF_RAW_HEADER(uint64_t, NumData, NumData) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesBeforeCounters, PaddingBytesBeforeCounters) -INSTR_PROF_RAW_HEADER(uint64_t, CountersSize, CountersSize) +INSTR_PROF_RAW_HEADER(uint64_t, NumCounters, NumCounters) INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterCounters, PaddingBytesAfterCounters) +INSTR_PROF_RAW_HEADER(uint64_t, NumBitmapBytes, NumBitmapBytes) +INSTR_PROF_RAW_HEADER(uint64_t, PaddingBytesAfterBitmapBytes, PaddingBytesAfterBitmapBytes) INSTR_PROF_RAW_HEADER(uint64_t, NamesSize, NamesSize) -INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, (uintptr_t)CountersBegin) +INSTR_PROF_RAW_HEADER(uint64_t, CountersDelta, + (uintptr_t)CountersBegin - (uintptr_t)DataBegin) +INSTR_PROF_RAW_HEADER(uint64_t, BitmapDelta, + (uintptr_t)BitmapBegin - (uintptr_t)DataBegin) INSTR_PROF_RAW_HEADER(uint64_t, NamesDelta, (uintptr_t)NamesBegin) +INSTR_PROF_RAW_HEADER(uint64_t, NumVTables, NumVTables) +INSTR_PROF_RAW_HEADER(uint64_t, VNamesSize, VNamesSize) INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #undef INSTR_PROF_RAW_HEADER /* INSTR_PROF_RAW_HEADER end */ @@ -153,18 +185,8 @@ INSTR_PROF_RAW_HEADER(uint64_t, ValueKindLast, IPVK_Last) #endif VALUE_PROF_FUNC_PARAM(uint64_t, TargetValue, Type::getInt64Ty(Ctx)) \ INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(void *, Data, Type::getInt8PtrTy(Ctx)) INSTR_PROF_COMMA -#ifndef VALUE_RANGE_PROF +VALUE_PROF_FUNC_PARAM(void *, Data, PointerType::getUnqual(Ctx)) INSTR_PROF_COMMA VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) -#else /* VALUE_RANGE_PROF */ -VALUE_PROF_FUNC_PARAM(uint32_t, CounterIndex, Type::getInt32Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeStart, Type::getInt64Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, PreciseRangeLast, Type::getInt64Ty(Ctx)) \ - INSTR_PROF_COMMA -VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) -#endif /*VALUE_RANGE_PROF */ #undef VALUE_PROF_FUNC_PARAM #undef INSTR_PROF_COMMA /* VALUE_PROF_FUNC_PARAM end */ @@ -187,17 +209,38 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") +/* For virtual table address profiling, the address point of the virtual table + * (i.e., the address contained in objects pointing to a virtual table) are + * profiled. Note this may not be the address of the per C++ class virtual table + * object (e.g., there might be an offset). + * + * The profiled addresses are stored in raw profile, together with the following + * two types of information. + * 1. The (starting and ending) addresses of per C++ class virtual table objects. + * 2. The (compressed) virtual table object names. + * RawInstrProfReader converts profiled virtual table addresses to virtual table + * objects' MD5 hash. + */ +VALUE_PROF_KIND(IPVK_VTableTarget, 2, "The profiled address point of the vtable") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") +VALUE_PROF_KIND(IPVK_Last, IPVK_VTableTarget, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ +#undef COVMAP_V2_OR_V3 +#ifdef COVMAP_V2 +#define COVMAP_V2_OR_V3 +#endif +#ifdef COVMAP_V3 +#define COVMAP_V2_OR_V3 +#endif + /* COVMAP_FUNC_RECORD start */ /* Definition of member fields of the function record structure in coverage * map. @@ -208,22 +251,36 @@ VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #define INSTR_PROF_DATA_DEFINED #endif #ifdef COVMAP_V1 -COVMAP_FUNC_RECORD(const IntPtrT, llvm::Type::getInt8PtrTy(Ctx), \ +COVMAP_FUNC_RECORD(const IntPtrT, llvm::PointerType::getUnqual(Ctx), \ NamePtr, llvm::ConstantExpr::getBitCast(NamePtr, \ - llvm::Type::getInt8PtrTy(Ctx))) + llvm::PointerType::getUnqual(Ctx))) COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), NameSize, \ llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx), \ NameValue.size())) -#else +#endif +#ifdef COVMAP_V2_OR_V3 COVMAP_FUNC_RECORD(const int64_t, llvm::Type::getInt64Ty(Ctx), NameRef, \ - llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), \ - llvm::IndexedInstrProf::ComputeHash(NameValue))) + llvm::ConstantInt::get( \ + llvm::Type::getInt64Ty(Ctx), NameHash)) #endif COVMAP_FUNC_RECORD(const uint32_t, llvm::Type::getInt32Ty(Ctx), DataSize, \ - llvm::ConstantInt::get(llvm::Type::getInt32Ty(Ctx),\ - CoverageMapping.size())) + llvm::ConstantInt::get( \ + llvm::Type::getInt32Ty(Ctx), CoverageMapping.size())) COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ - llvm::ConstantInt::get(llvm::Type::getInt64Ty(Ctx), FuncHash)) + llvm::ConstantInt::get( \ + llvm::Type::getInt64Ty(Ctx), FuncHash)) +#ifdef COVMAP_V3 +COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FilenamesRef, \ + llvm::ConstantInt::get( \ + llvm::Type::getInt64Ty(Ctx), FilenamesRef)) +COVMAP_FUNC_RECORD(const char, \ + llvm::ArrayType::get(llvm::Type::getInt8Ty(Ctx), \ + CoverageMapping.size()), \ + CoverageMapping, + llvm::ConstantDataArray::getRaw( \ + CoverageMapping, CoverageMapping.size(), \ + llvm::Type::getInt8Ty(Ctx))) +#endif #undef COVMAP_FUNC_RECORD /* COVMAP_FUNC_RECORD end. */ @@ -236,7 +293,7 @@ COVMAP_FUNC_RECORD(const uint64_t, llvm::Type::getInt64Ty(Ctx), FuncHash, \ #define INSTR_PROF_DATA_DEFINED #endif COVMAP_HEADER(uint32_t, Int32Ty, NRecords, \ - llvm::ConstantInt::get(Int32Ty, FunctionRecords.size())) + llvm::ConstantInt::get(Int32Ty, NRecords)) COVMAP_HEADER(uint32_t, Int32Ty, FilenamesSize, \ llvm::ConstantInt::get(Int32Ty, FilenamesSize)) COVMAP_HEADER(uint32_t, Int32Ty, CoverageSize, \ @@ -255,21 +312,39 @@ INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ INSTR_PROF_CNTS_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_bitmap, \ + INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON), \ + INSTR_PROF_BITS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ INSTR_PROF_NAME_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vname, \ + INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON), \ + INSTR_PROF_VNAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ INSTR_PROF_VNODES_COFF, "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_vtab, \ + INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON), \ + INSTR_PROF_VTAB_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covfun, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON), \ + INSTR_PROF_COVFUN_COFF, "__LLVM_COV,") INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") +INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON), \ + INSTR_PROF_COVDATA_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covname, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ + INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -325,7 +400,7 @@ typedef struct ValueProfRecord { * Do byte swap for this instance. \c Old is the original order before * the swap, and \c New is the New byte order. */ - void swapBytes(support::endianness Old, support::endianness New); + void swapBytes(llvm::endianness Old, llvm::endianness New); #endif } ValueProfRecord; @@ -380,15 +455,15 @@ typedef struct ValueProfData { static Expected> getValueProfData(const unsigned char *SrcBuffer, const unsigned char *const SrcBufferEnd, - support::endianness SrcDataEndianness); + llvm::endianness SrcDataEndianness); /*! * Swap byte order from \c Endianness order to host byte order. */ - void swapBytesToHost(support::endianness Endianness); + void swapBytesToHost(llvm::endianness Endianness); /*! * Swap byte order from host byte order to \c Endianness order. */ - void swapBytesFromHost(support::endianness Endianness); + void swapBytesFromHost(llvm::endianness Endianness); /*! * Return the total size of \c ValueProfileData. */ @@ -461,12 +536,14 @@ getValueProfRecordHeaderSize(uint32_t NumValueSites); #define offsetof(TYPE, MEMBER) ((size_t) &((TYPE *)0)->MEMBER) #endif +// clang-format on + /*! * Return the \c ValueProfRecord header size including the * padding bytes. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t +getValueProfRecordHeaderSize(uint32_t NumValueSites) { uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + sizeof(uint8_t) * NumValueSites; /* Round the size to multiple of 8 bytes. */ @@ -478,9 +555,8 @@ uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { * Return the total size of the value profile record including the * header and the value data. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordSize(uint32_t NumValueSites, - uint32_t NumValueData) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t +getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) { return getValueProfRecordHeaderSize(NumValueSites) + sizeof(InstrProfValueData) * NumValueData; } @@ -488,8 +564,8 @@ uint32_t getValueProfRecordSize(uint32_t NumValueSites, /*! * Return the pointer to the start of value data array. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE InstrProfValueData * +getValueProfRecordValueData(ValueProfRecord *This) { return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( This->NumValueSites)); } @@ -497,8 +573,8 @@ InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { /*! * Return the total number of value data for \c This record. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint32_t +getValueProfRecordNumValueData(ValueProfRecord *This) { uint32_t NumValueData = 0; uint32_t I; for (I = 0; I < This->NumValueSites; I++) @@ -509,8 +585,8 @@ uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { /*! * Use this method to advance to the next \c This \c ValueProfRecord. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord * +getValueProfRecordNext(ValueProfRecord *This) { uint32_t NumValueData = getValueProfRecordNumValueData(This); return (ValueProfRecord *)((char *)This + getValueProfRecordSize(This->NumValueSites, @@ -520,8 +596,8 @@ ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { /*! * Return the first \c ValueProfRecord instance. */ -INSTR_PROF_VISIBILITY INSTR_PROF_INLINE -ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE ValueProfRecord * +getFirstValueProfRecord(ValueProfData *This) { return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); } @@ -604,6 +680,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /*============================================================================*/ +// clang-format off:consider re-enabling clang-format if auto-formatted C macros +// are readable (e.g., after `issue #82426` is fixed) #ifndef INSTR_PROF_DATA_DEFINED #ifndef INSTR_PROF_DATA_INC @@ -630,24 +708,38 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, (uint64_t)'f' << 16 | (uint64_t)'R' << 8 | (uint64_t)129 /* Raw profile format version (start from 1). */ -#define INSTR_PROF_RAW_VERSION 5 +#define INSTR_PROF_RAW_VERSION 10 /* Indexed profile format version (start from 1). */ -#define INSTR_PROF_INDEX_VERSION 5 -/* Coverage mapping format vresion (start from 0). */ -#define INSTR_PROF_COVMAP_VERSION 2 +#define INSTR_PROF_INDEX_VERSION 12 +/* Coverage mapping format version (start from 0). */ +#define INSTR_PROF_COVMAP_VERSION 6 -/* Profile version is always of type uint64_t. Reserve the upper 8 bits in the - * version for other variants of profile. We set the lowest bit of the upper 8 - * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton +/* Profile version is always of type uint64_t. Reserve the upper 32 bits in the + * version for other variants of profile. We set the 8th most significant bit + * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. + * The 59th bit indicates whether to use debug info to correlate profiles. + * The 60th bit indicates single byte coverage instrumentation. + * The 61st bit indicates function entry instrumentation only. + * The 62nd bit indicates whether memory profile information is present. + * The 63rd bit indicates if this is a temporal profile. */ -#define VARIANT_MASKS_ALL 0xff00000000000000ULL +#define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) +#define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) +#define VARIANT_MASK_DBG_CORRELATE (0x1ULL << 59) +#define VARIANT_MASK_BYTE_COVERAGE (0x1ULL << 60) +#define VARIANT_MASK_FUNCTION_ENTRY_ONLY (0x1ULL << 61) +#define VARIANT_MASK_MEMPROF (0x1ULL << 62) +#define VARIANT_MASK_TEMPORAL_PROF (0x1ULL << 63) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime +#define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp +#define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling /* The variable that holds the name of the profile data * specified via command line. */ @@ -657,20 +749,35 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, than WIN32 */ #define INSTR_PROF_DATA_COMMON __llvm_prf_data #define INSTR_PROF_NAME_COMMON __llvm_prf_names +#define INSTR_PROF_VNAME_COMMON __llvm_prf_vns #define INSTR_PROF_CNTS_COMMON __llvm_prf_cnts +#define INSTR_PROF_BITS_COMMON __llvm_prf_bits #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds +#define INSTR_PROF_VTAB_COMMON __llvm_prf_vtab #define INSTR_PROF_COVMAP_COMMON __llvm_covmap +#define INSTR_PROF_COVFUN_COMMON __llvm_covfun +#define INSTR_PROF_COVDATA_COMMON __llvm_covdata +#define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ #define INSTR_PROF_DATA_COFF ".lprfd$M" #define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_VNAME_COFF ".lprfvn$M" #define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_BITS_COFF ".lprfb$M" #define INSTR_PROF_VALS_COFF ".lprfv$M" #define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_VTAB_COFF ".lprfvt$M" #define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_COVFUN_COFF ".lcovfun$M" +/* Since cov data and cov names sections are not allocated, we don't need to + * access them at runtime. + */ +#define INSTR_PROF_COVDATA_COFF ".lcovd" +#define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 @@ -678,6 +785,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COFF #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COFF +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_BITS_COFF +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_VTAB_COFF +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_VNAME_COFF /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -685,12 +795,18 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_COVFUN_COFF +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) #define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) #define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) +#define INSTR_PROF_BITS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_BITS_COMMON) +#define INSTR_PROF_VTAB_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VTAB_COMMON) +#define INSTR_PROF_VNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNAME_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ @@ -698,6 +814,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +#define INSTR_PROF_COVFUN_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVFUN_COMMON) +#define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVDATA_COMMON) +#define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON) /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) @@ -724,9 +843,9 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALUE_PROF_FUNC __llvm_profile_instrument_target #define INSTR_PROF_VALUE_PROF_FUNC_STR \ INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_FUNC) -#define INSTR_PROF_VALUE_RANGE_PROF_FUNC __llvm_profile_instrument_range -#define INSTR_PROF_VALUE_RANGE_PROF_FUNC_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALUE_RANGE_PROF_FUNC) +#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC __llvm_profile_instrument_memop +#define INSTR_PROF_VALUE_PROF_MEMOP_FUNC_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_VALUE_PROF_MEMOP_FUNC) /* InstrProfile per-function control data alignment. */ #define INSTR_PROF_DATA_ALIGNMENT 8 @@ -752,3 +871,126 @@ typedef struct InstrProfValueData { #else #undef INSTR_PROF_DATA_DEFINED #endif + +#undef COVMAP_V2_OR_V3 + +#ifdef INSTR_PROF_VALUE_PROF_MEMOP_API + +#ifdef __cplusplus +#define INSTR_PROF_INLINE inline +#else +#define INSTR_PROF_INLINE +#endif + +/* The value range buckets (22 buckets) for the memop size value profiling looks + * like: + * + * [0, 0] + * [1, 1] + * [2, 2] + * [3, 3] + * [4, 4] + * [5, 5] + * [6, 6] + * [7, 7] + * [8, 8] + * [9, 15] + * [16, 16] + * [17, 31] + * [32, 32] + * [33, 63] + * [64, 64] + * [65, 127] + * [128, 128] + * [129, 255] + * [256, 256] + * [257, 511] + * [512, 512] + * [513, UINT64_MAX] + * + * Each range has a 'representative value' which is the lower end value of the + * range and used to store in the runtime profile data records and the VP + * metadata. For example, it's 2 for [2, 2] and 64 for [65, 127]. + */ +#define INSTR_PROF_NUM_BUCKETS 22 + +/* + * Clz and Popcount. This code was copied from + * compiler-rt/lib/fuzzer/{FuzzerBuiltins.h,FuzzerBuiltinsMsvc.h} and + * llvm/include/llvm/Support/MathExtras.h. + */ +#if defined(_MSC_VER) && !defined(__clang__) + +#include +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfClzll(unsigned long long X) { + unsigned long LeadZeroIdx = 0; +#if !defined(_M_ARM64) && !defined(_M_X64) + // Scan the high 32 bits. + if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X >> 32))) + return (int)(63 - (LeadZeroIdx + 32)); // Create a bit offset + // from the MSB. + // Scan the low 32 bits. + if (_BitScanReverse(&LeadZeroIdx, (unsigned long)(X))) + return (int)(63 - LeadZeroIdx); +#else + if (_BitScanReverse64(&LeadZeroIdx, X)) return 63 - LeadZeroIdx; +#endif + return 64; +} +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfPopcountll(unsigned long long X) { + // This code originates from https://reviews.llvm.org/rG30626254510f. + unsigned long long v = X; + v = v - ((v >> 1) & 0x5555555555555555ULL); + v = (v & 0x3333333333333333ULL) + ((v >> 2) & 0x3333333333333333ULL); + v = (v + (v >> 4)) & 0x0F0F0F0F0F0F0F0FULL; + return (int)((unsigned long long)(v * 0x0101010101010101ULL) >> 56); +} + +#else + +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfClzll(unsigned long long X) { return __builtin_clzll(X); } +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE +int InstProfPopcountll(unsigned long long X) { return __builtin_popcountll(X); } + +#endif /* defined(_MSC_VER) && !defined(__clang__) */ + +// clang-format on + +/* Map an (observed) memop size value to the representative value of its range. + * For example, 5 -> 5, 22 -> 17, 99 -> 65, 256 -> 256, 1001 -> 513. */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE uint64_t +InstrProfGetRangeRepValue(uint64_t Value) { + if (Value <= 8) + // The first ranges are individually tracked. Use the value as is. + return Value; + else if (Value >= 513) + // The last range is mapped to its lowest value. + return 513; + else if (InstProfPopcountll(Value) == 1) + // If it's a power of two, use it as is. + return Value; + else + // Otherwise, take to the previous power of two + 1. + return (UINT64_C(1) << (64 - InstProfClzll(Value) - 1)) + 1; +} + +/* Return true if the range that an (observed) memop size value belongs to has + * only a single value in the range. For example, 0 -> true, 8 -> true, 10 -> + * false, 64 -> true, 100 -> false, 513 -> false. */ +INSTR_PROF_VISIBILITY INSTR_PROF_INLINE unsigned +InstrProfIsSingleValRange(uint64_t Value) { + if (Value <= 8) + // The first ranges are individually tracked. + return 1; + else if (InstProfPopcountll(Value) == 1) + // If it's a power of two, there's only one value. + return 1; + else + // Otherwise, there's more than one value in the range. + return 0; +} + +#endif /* INSTR_PROF_VALUE_PROF_MEMOP_API */ diff --git a/system/lib/compiler-rt/include/profile/MIBEntryDef.inc b/system/lib/compiler-rt/include/profile/MIBEntryDef.inc new file mode 100644 index 0000000000000..58c1fc4de4aba --- /dev/null +++ b/system/lib/compiler-rt/include/profile/MIBEntryDef.inc @@ -0,0 +1,55 @@ +/*===-- MemEntryDef.inc - MemProf profiling runtime macros -*- C++ -*-======== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This file defines the macros for memprof profiling data structures. + * Eg. usage to define the memprof meminfoblock struct: + * + * struct MemInfoBlock { + * #define MIBEntryDef(NameTag, Name, Type) Type Name; + * #include MIBEntryDef.inc + * #undef MIBEntryDef + * }; + * + * This file has two identical copies. The primary copy lives in LLVM and + * the other one sits in compiler-rt/include/profile directory. To make changes + * in this file, first modify the primary copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * +\*===----------------------------------------------------------------------===*/ +#ifndef MIBEntryDef +#define MIBEntryDef(NameTag, Name, Type) +#endif + +MIBEntryDef(AllocCount = 1, AllocCount, uint32_t) +MIBEntryDef(TotalAccessCount = 2, TotalAccessCount, uint64_t) +MIBEntryDef(MinAccessCount = 3, MinAccessCount, uint64_t) +MIBEntryDef(MaxAccessCount = 4, MaxAccessCount, uint64_t) +MIBEntryDef(TotalSize = 5, TotalSize, uint64_t) +MIBEntryDef(MinSize = 6, MinSize, uint32_t) +MIBEntryDef(MaxSize = 7, MaxSize, uint32_t) +MIBEntryDef(AllocTimestamp = 8, AllocTimestamp, uint32_t) +MIBEntryDef(DeallocTimestamp = 9, DeallocTimestamp, uint32_t) +MIBEntryDef(TotalLifetime = 10, TotalLifetime, uint64_t) +MIBEntryDef(MinLifetime = 11, MinLifetime, uint32_t) +MIBEntryDef(MaxLifetime = 12, MaxLifetime, uint32_t) +MIBEntryDef(AllocCpuId = 13, AllocCpuId, uint32_t) +MIBEntryDef(DeallocCpuId = 14, DeallocCpuId, uint32_t) +MIBEntryDef(NumMigratedCpu = 15, NumMigratedCpu, uint32_t) +MIBEntryDef(NumLifetimeOverlaps = 16, NumLifetimeOverlaps, uint32_t) +MIBEntryDef(NumSameAllocCpu = 17, NumSameAllocCpu, uint32_t) +MIBEntryDef(NumSameDeallocCpu = 18, NumSameDeallocCpu, uint32_t) +MIBEntryDef(DataTypeId = 19, DataTypeId, uint64_t) +MIBEntryDef(TotalAccessDensity = 20, TotalAccessDensity, uint64_t) +MIBEntryDef(MinAccessDensity = 21, MinAccessDensity, uint32_t) +MIBEntryDef(MaxAccessDensity = 22, MaxAccessDensity, uint32_t) +MIBEntryDef(TotalLifetimeAccessDensity = 23, TotalLifetimeAccessDensity, uint64_t) +MIBEntryDef(MinLifetimeAccessDensity = 24, MinLifetimeAccessDensity, uint32_t) +MIBEntryDef(MaxLifetimeAccessDensity = 25, MaxLifetimeAccessDensity, uint32_t) +MIBEntryDef(AccessHistogramSize = 26, AccessHistogramSize, uint32_t) +MIBEntryDef(AccessHistogram = 27, AccessHistogram, uintptr_t) \ No newline at end of file diff --git a/system/lib/compiler-rt/include/profile/MemProfData.inc b/system/lib/compiler-rt/include/profile/MemProfData.inc new file mode 100644 index 0000000000000..3f785bd23fce3 --- /dev/null +++ b/system/lib/compiler-rt/include/profile/MemProfData.inc @@ -0,0 +1,235 @@ +#ifndef MEMPROF_DATA_INC +#define MEMPROF_DATA_INC +/*===-- MemProfData.inc - MemProf profiling runtime structures -*- C++ -*-=== *\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ +/* + * This is the main file that defines all the data structure, signature, + * constant literals that are shared across profiling runtime library, + * and host tools (reader/writer). + * + * This file has two identical copies. The primary copy lives in LLVM and + * the other one sits in compiler-rt/include/profile directory. To make changes + * in this file, first modify the primary copy and copy it over to compiler-rt. + * Testing of any change in this file can start only after the two copies are + * synced up. + * +\*===----------------------------------------------------------------------===*/ +#include + +#ifdef _MSC_VER +#define PACKED(...) __pragma(pack(push,1)) __VA_ARGS__ __pragma(pack(pop)) +#else +#define PACKED(...) __VA_ARGS__ __attribute__((__packed__)) +#endif + +// A 64-bit magic number to uniquely identify the raw binary memprof profile file. +#define MEMPROF_RAW_MAGIC_64 \ + ((uint64_t)255 << 56 | (uint64_t)'m' << 48 | (uint64_t)'p' << 40 | (uint64_t)'r' << 32 | \ + (uint64_t)'o' << 24 | (uint64_t)'f' << 16 | (uint64_t)'r' << 8 | (uint64_t)129) + +// The version number of the raw binary format. +#define MEMPROF_RAW_VERSION 4ULL + +// Currently supported versions. +#define MEMPROF_RAW_SUPPORTED_VERSIONS \ + { 3ULL, 4ULL } + +#define MEMPROF_V3_MIB_SIZE 132ULL; + +#define MEMPROF_BUILDID_MAX_SIZE 32ULL + +namespace llvm { +namespace memprof { +// A struct describing the header used for the raw binary memprof profile format. +PACKED(struct Header { + uint64_t Magic; + uint64_t Version; + uint64_t TotalSize; + uint64_t SegmentOffset; + uint64_t MIBOffset; + uint64_t StackOffset; +}); + +// A struct describing the information necessary to describe a /proc/maps +// segment entry for a particular binary/library identified by its build id. +PACKED(struct SegmentEntry { + uint64_t Start; + uint64_t End; + uint64_t Offset; + uint64_t BuildIdSize; + uint8_t BuildId[MEMPROF_BUILDID_MAX_SIZE] = {0}; + + // This constructor is only used in tests so don't set the BuildId. + SegmentEntry(uint64_t S, uint64_t E, uint64_t O) + : Start(S), End(E), Offset(O), BuildIdSize(0) {} + + SegmentEntry(const SegmentEntry& S) { + Start = S.Start; + End = S.End; + Offset = S.Offset; + BuildIdSize = S.BuildIdSize; + memcpy(BuildId, S.BuildId, S.BuildIdSize); + } + + SegmentEntry& operator=(const SegmentEntry& S) { + Start = S.Start; + End = S.End; + Offset = S.Offset; + BuildIdSize = S.BuildIdSize; + memcpy(BuildId, S.BuildId, S.BuildIdSize); + return *this; + } + + bool operator==(const SegmentEntry& S) const { + return Start == S.Start && End == S.End && Offset == S.Offset && + BuildIdSize == S.BuildIdSize && + memcmp(BuildId, S.BuildId, S.BuildIdSize) == 0; + } +}); + +// Packed struct definition for MSVC. We can't use the PACKED macro defined in +// MemProfData.inc since it would mean we are embedding a directive (the +// #include for MIBEntryDef) into the macros which is undefined behaviour. +#ifdef _MSC_VER +__pragma(pack(push,1)) +#endif + +// A struct representing the heap allocation characteristics of a particular +// runtime context. This struct is shared between the compiler-rt runtime and +// the raw profile reader. The indexed format uses a separate, self-describing +// backwards compatible format. +struct MemInfoBlock{ + +#define MIBEntryDef(NameTag, Name, Type) Type Name; +#include "MIBEntryDef.inc" +#undef MIBEntryDef + +bool operator==(const MemInfoBlock& Other) const { + bool IsEqual = true; +#define MIBEntryDef(NameTag, Name, Type) \ + IsEqual = (IsEqual && Name == Other.Name); +#include "MIBEntryDef.inc" +#undef MIBEntryDef + return IsEqual; +} + +MemInfoBlock() { +#define MIBEntryDef(NameTag, Name, Type) Name = Type(); +#include "MIBEntryDef.inc" +#undef MIBEntryDef +} + +MemInfoBlock(uint32_t Size, uint64_t AccessCount, uint32_t AllocTs, + uint32_t DeallocTs, uint32_t AllocCpu, uint32_t DeallocCpu, + uintptr_t Histogram, uint32_t HistogramSize) + : MemInfoBlock() { + AllocCount = 1U; + TotalAccessCount = AccessCount; + MinAccessCount = AccessCount; + MaxAccessCount = AccessCount; + TotalSize = Size; + MinSize = Size; + MaxSize = Size; + AllocTimestamp = AllocTs; + DeallocTimestamp = DeallocTs; + TotalLifetime = DeallocTimestamp - AllocTimestamp; + MinLifetime = TotalLifetime; + MaxLifetime = TotalLifetime; + // Access density is accesses per byte. Multiply by 100 to include the + // fractional part. + TotalAccessDensity = AccessCount * 100 / Size; + MinAccessDensity = TotalAccessDensity; + MaxAccessDensity = TotalAccessDensity; + // Lifetime access density is the access density per second of lifetime. + // Multiply by 1000 to convert denominator lifetime to seconds (using a + // minimum lifetime of 1ms to avoid divide by 0. Do the multiplication first + // to reduce truncations to 0. + TotalLifetimeAccessDensity = + TotalAccessDensity * 1000 / (TotalLifetime ? TotalLifetime : 1); + MinLifetimeAccessDensity = TotalLifetimeAccessDensity; + MaxLifetimeAccessDensity = TotalLifetimeAccessDensity; + AllocCpuId = AllocCpu; + DeallocCpuId = DeallocCpu; + NumMigratedCpu = AllocCpuId != DeallocCpuId; + AccessHistogramSize = HistogramSize; + AccessHistogram = Histogram; +} + +void Merge(const MemInfoBlock &newMIB) { + AllocCount += newMIB.AllocCount; + + TotalAccessCount += newMIB.TotalAccessCount; + MinAccessCount = newMIB.MinAccessCount < MinAccessCount ? newMIB.MinAccessCount : MinAccessCount; + MaxAccessCount = newMIB.MaxAccessCount > MaxAccessCount ? newMIB.MaxAccessCount : MaxAccessCount; + + TotalSize += newMIB.TotalSize; + MinSize = newMIB.MinSize < MinSize ? newMIB.MinSize : MinSize; + MaxSize = newMIB.MaxSize > MaxSize ? newMIB.MaxSize : MaxSize; + + TotalLifetime += newMIB.TotalLifetime; + MinLifetime = newMIB.MinLifetime < MinLifetime ? newMIB.MinLifetime : MinLifetime; + MaxLifetime = newMIB.MaxLifetime > MaxLifetime ? newMIB.MaxLifetime : MaxLifetime; + + TotalAccessDensity += newMIB.TotalAccessDensity; + MinAccessDensity = newMIB.MinAccessDensity < MinAccessDensity + ? newMIB.MinAccessDensity + : MinAccessDensity; + MaxAccessDensity = newMIB.MaxAccessDensity > MaxAccessDensity + ? newMIB.MaxAccessDensity + : MaxAccessDensity; + + TotalLifetimeAccessDensity += newMIB.TotalLifetimeAccessDensity; + MinLifetimeAccessDensity = + newMIB.MinLifetimeAccessDensity < MinLifetimeAccessDensity + ? newMIB.MinLifetimeAccessDensity + : MinLifetimeAccessDensity; + MaxLifetimeAccessDensity = + newMIB.MaxLifetimeAccessDensity > MaxLifetimeAccessDensity + ? newMIB.MaxLifetimeAccessDensity + : MaxLifetimeAccessDensity; + + // We know newMIB was deallocated later, so just need to check if it was + // allocated before last one deallocated. + NumLifetimeOverlaps += newMIB.AllocTimestamp < DeallocTimestamp; + AllocTimestamp = newMIB.AllocTimestamp; + DeallocTimestamp = newMIB.DeallocTimestamp; + + NumSameAllocCpu += AllocCpuId == newMIB.AllocCpuId; + NumSameDeallocCpu += DeallocCpuId == newMIB.DeallocCpuId; + AllocCpuId = newMIB.AllocCpuId; + DeallocCpuId = newMIB.DeallocCpuId; + + // For merging histograms, we always keep the longer histogram, and add + // values of shorter histogram to larger one. + uintptr_t ShorterHistogram; + uint32_t ShorterHistogramSize; + if (newMIB.AccessHistogramSize > AccessHistogramSize) { + ShorterHistogram = AccessHistogram; + ShorterHistogramSize = AccessHistogramSize; + // Swap histogram of current to larger histogram + AccessHistogram = newMIB.AccessHistogram; + AccessHistogramSize = newMIB.AccessHistogramSize; + } else { + ShorterHistogram = newMIB.AccessHistogram; + ShorterHistogramSize = newMIB.AccessHistogramSize; + } + for (size_t i = 0; i < ShorterHistogramSize; ++i) { + ((uint64_t *)AccessHistogram)[i] += ((uint64_t *)ShorterHistogram)[i]; + } +} + +#ifdef _MSC_VER +} __pragma(pack(pop)); +#else +} __attribute__((__packed__)); +#endif + +} // namespace memprof +} // namespace llvm + +#endif diff --git a/system/lib/compiler-rt/include/profile/instr_prof_interface.h b/system/lib/compiler-rt/include/profile/instr_prof_interface.h new file mode 100644 index 0000000000000..be40f2685934b --- /dev/null +++ b/system/lib/compiler-rt/include/profile/instr_prof_interface.h @@ -0,0 +1,92 @@ +/*===---- instr_prof_interface.h - Instrumentation PGO User Program API ----=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + * + * This header provides a public interface for fine-grained control of counter + * reset and profile dumping. These interface functions can be directly called + * in user programs. + * +\*===---------------------------------------------------------------------===*/ + +#ifndef COMPILER_RT_INSTR_PROFILING +#define COMPILER_RT_INSTR_PROFILING + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __LLVM_INSTR_PROFILE_GENERATE +// Profile file reset and dump interfaces. +// When `-fprofile[-instr]-generate`/`-fcs-profile-generate` is in effect, +// clang defines __LLVM_INSTR_PROFILE_GENERATE to pick up the API calls. + +/*! + * \brief Set the filename for writing instrumentation data. + * + * Sets the filename to be used for subsequent calls to + * \a __llvm_profile_write_file(). + * + * \c Name is not copied, so it must remain valid. Passing NULL resets the + * filename logic to the default behaviour. + * + * Note: There may be multiple copies of the profile runtime (one for each + * instrumented image/DSO). This API only modifies the filename within the + * copy of the runtime available to the calling image. + * + * Warning: This is a no-op if continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is + * that in continuous mode, profile counters are mmap()'d to the profile at + * program initialization time. Support for transferring the mmap'd profile + * counts to a new file has not been implemented. + */ +void __llvm_profile_set_filename(const char *Name); + +/*! + * \brief Interface to set all PGO counters to zero for the current process. + * + */ +void __llvm_profile_reset_counters(void); + +/*! + * \brief this is a wrapper interface to \c __llvm_profile_write_file. + * After this interface is invoked, an already dumped flag will be set + * so that profile won't be dumped again during program exit. + * Invocation of interface __llvm_profile_reset_counters will clear + * the flag. This interface is designed to be used to collect profile + * data from user selected hot regions. The use model is + * __llvm_profile_reset_counters(); + * ... hot region 1 + * __llvm_profile_dump(); + * .. some other code + * __llvm_profile_reset_counters(); + * ... hot region 2 + * __llvm_profile_dump(); + * + * It is expected that on-line profile merging is on with \c %m specifier + * used in profile filename . If merging is not turned on, user is expected + * to invoke __llvm_profile_set_filename to specify different profile names + * for different regions before dumping to avoid profile write clobbering. + */ +int __llvm_profile_dump(void); + +// Interface to dump the current process' order file to disk. +int __llvm_orderfile_dump(void); + +#else + +#define __llvm_profile_set_filename(Name) +#define __llvm_profile_reset_counters() +#define __llvm_profile_dump() (0) +#define __llvm_orderfile_dump() (0) + +#endif + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif diff --git a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c index 498c05900bf27..4f46fd2839b90 100644 --- a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c @@ -3,9 +3,9 @@ |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |* See https://llvm.org/LICENSE.txt for license information. |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -|* +|* |*===----------------------------------------------------------------------===*| -|* +|* |* This file implements the call back routines for the gcov profiling |* instrumentation pass. Link against this library when running code through |* the -insert-gcov-profiling LLVM pass. @@ -23,6 +23,7 @@ #include #include +#include #include #include #include @@ -32,31 +33,10 @@ #include #include "WindowsMMap.h" #else -#include #include -#endif - -#if defined(__FreeBSD__) && defined(__i386__) -#define I386_FREEBSD 1 -#else -#define I386_FREEBSD 0 -#endif - -#if !defined(_MSC_VER) && !I386_FREEBSD -#include -#endif - -#if defined(_MSC_VER) -typedef unsigned char uint8_t; -typedef unsigned int uint32_t; -typedef unsigned long long uint64_t; -#elif I386_FREEBSD -/* System headers define 'size_t' incorrectly on x64 FreeBSD (prior to - * FreeBSD 10, r232261) when compiled in 32-bit mode. - */ -typedef unsigned char uint8_t; -typedef unsigned int uint32_t; -typedef unsigned long long uint64_t; +#include +#include +#include #endif #include "InstrProfiling.h" @@ -64,6 +44,16 @@ typedef unsigned long long uint64_t; /* #define DEBUG_GCDAPROFILING */ +enum { + GCOV_DATA_MAGIC = 0x67636461, // "gcda" + + GCOV_TAG_FUNCTION = 0x01000000, + GCOV_TAG_COUNTER_ARCS = 0x01a10000, + // GCOV_TAG_OBJECT_SUMMARY superseded GCOV_TAG_PROGRAM_SUMMARY in GCC 9. + GCOV_TAG_OBJECT_SUMMARY = 0xa1000000, + GCOV_TAG_PROGRAM_SUMMARY = 0xa3000000, +}; + /* * --- GCOV file format I/O primitives --- */ @@ -75,7 +65,7 @@ static char *filename = NULL; /* * The current file we're outputting. - */ + */ static FILE *output_file = NULL; /* @@ -87,12 +77,13 @@ static uint64_t cur_buffer_size = 0; static uint64_t cur_pos = 0; static uint64_t file_size = 0; static int new_file = 0; +static int gcov_version; #if defined(_WIN32) static HANDLE mmap_handle = NULL; #endif static int fd = -1; -typedef void (*fn_ptr)(); +typedef void (*fn_ptr)(void); typedef void* dynamic_object_id; // The address of this variable identifies a given dynamic object. @@ -115,9 +106,9 @@ struct fn_list { struct fn_list writeout_fn_list; /* - * A list of flush functions that our __gcov_flush() function should call, shared between all dynamic objects. + * A list of reset functions, shared between all dynamic objects. */ -struct fn_list flush_fn_list; +struct fn_list reset_fn_list; static void fn_list_insert(struct fn_list* list, fn_ptr fn) { struct fn_node* new_node = malloc(sizeof(struct fn_node)); @@ -192,18 +183,7 @@ static void write_64bit_value(uint64_t i) { write_32bit_value(hi); } -static uint32_t length_of_string(const char *s) { - return (strlen(s) / 4) + 1; -} - -static void write_string(const char *s) { - uint32_t len = length_of_string(s); - write_32bit_value(len); - write_bytes(s, strlen(s)); - write_bytes("\0\0\0\0", 4 - (strlen(s) % 4)); -} - -static uint32_t read_32bit_value() { +static uint32_t read_32bit_value(void) { uint32_t val; if (new_file) @@ -214,19 +194,7 @@ static uint32_t read_32bit_value() { return val; } -static uint32_t read_le_32bit_value() { - uint32_t val = 0; - int i; - - if (new_file) - return (uint32_t)-1; - - for (i = 0; i < 4; i++) - val |= write_buffer[cur_pos++] << (8*i); - return val; -} - -static uint64_t read_64bit_value() { +static uint64_t read_64bit_value(void) { // GCOV uses a lo-/hi-word format even on big-endian systems. // See also GCOVBuffer::readInt64 in LLVM. uint32_t lo = read_32bit_value(); @@ -250,12 +218,12 @@ static char *mangle_filename(const char *orig_filename) { return new_filename; } -static int map_file() { +static int map_file(void) { fseek(output_file, 0L, SEEK_END); file_size = ftell(output_file); - /* A size of 0 is invalid to `mmap'. Return a fail here, but don't issue an - * error message because it should "just work" for the user. */ + /* A size of 0 means the file has been created just now (possibly by another + * process in lock-after-open race condition). No need to mmap. */ if (file_size == 0) return -1; @@ -294,13 +262,8 @@ static int map_file() { return 0; } -static void unmap_file() { +static void unmap_file(void) { #if defined(_WIN32) - if (!FlushViewOfFile(write_buffer, file_size)) { - fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename, - GetLastError()); - } - if (!UnmapViewOfFile(write_buffer)) { fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, GetLastError()); @@ -313,16 +276,11 @@ static void unmap_file() { mmap_handle = NULL; #else - if (msync(write_buffer, file_size, MS_SYNC) == -1) { + if (munmap(write_buffer, file_size) == -1) { int errnum = errno; - fprintf(stderr, "profiling: %s: cannot msync: %s\n", filename, + fprintf(stderr, "profiling: %s: cannot munmap: %s\n", filename, strerror(errnum)); } - - /* We explicitly ignore errors from unmapping because at this point the data - * is written and we don't care. - */ - (void)munmap(write_buffer, file_size); #endif write_buffer = NULL; @@ -338,30 +296,36 @@ static void unmap_file() { * started at a time. */ COMPILER_RT_VISIBILITY -void llvm_gcda_start_file(const char *orig_filename, const char version[4], +void llvm_gcda_start_file(const char *orig_filename, uint32_t version, uint32_t checksum) { const char *mode = "r+b"; filename = mangle_filename(orig_filename); /* Try just opening the file. */ - new_file = 0; fd = open(filename, O_RDWR | O_BINARY); if (fd == -1) { - /* Try opening the file, creating it if necessary. */ - new_file = 1; - mode = "w+b"; - fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); - if (fd == -1) { + /* Try creating the file. */ + fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644); + if (fd != -1) { + mode = "w+b"; + } else { /* Try creating the directories first then opening the file. */ __llvm_profile_recursive_mkdir(filename); - fd = open(filename, O_RDWR | O_CREAT | O_BINARY, 0644); - if (fd == -1) { - /* Bah! It's hopeless. */ - int errnum = errno; - fprintf(stderr, "profiling: %s: cannot open: %s\n", filename, - strerror(errnum)); - return; + fd = open(filename, O_RDWR | O_CREAT | O_EXCL | O_BINARY, 0644); + if (fd != -1) { + mode = "w+b"; + } else { + /* Another process may have created the file just now. + * Try opening it without O_CREAT and O_EXCL. */ + fd = open(filename, O_RDWR | O_BINARY); + if (fd == -1) { + /* Bah! It's hopeless. */ + int errnum = errno; + fprintf(stderr, "profiling: %s: cannot open: %s\n", filename, + strerror(errnum)); + return; + } } } } @@ -374,27 +338,30 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], output_file = fdopen(fd, mode); /* Initialize the write buffer. */ + new_file = 0; write_buffer = NULL; cur_buffer_size = 0; cur_pos = 0; - if (new_file) { + if (map_file() == -1) { + /* The file has been created just now (file_size == 0) or mmap failed + * unexpectedly. In the latter case, try to recover by clobbering. */ + new_file = 1; + write_buffer = NULL; resize_write_buffer(WRITE_BUFFER_SIZE); memset(write_buffer, 0, WRITE_BUFFER_SIZE); - } else { - if (map_file() == -1) { - /* mmap failed, try to recover by clobbering */ - new_file = 1; - write_buffer = NULL; - cur_buffer_size = 0; - resize_write_buffer(WRITE_BUFFER_SIZE); - memset(write_buffer, 0, WRITE_BUFFER_SIZE); - } } /* gcda file, version, stamp checksum. */ - write_bytes("adcg", 4); - write_bytes(version, 4); + { + uint8_t c3 = version >> 24; + uint8_t c2 = (version >> 16) & 255; + uint8_t c1 = (version >> 8) & 255; + gcov_version = c3 >= 'A' ? (c3 - 'A') * 100 + (c2 - '0') * 10 + c1 - '0' + : (c3 - '0') * 10 + c1 - '0'; + } + write_32bit_value(GCOV_DATA_MAGIC); + write_32bit_value(version); write_32bit_value(checksum); #ifdef DEBUG_GCDAPROFILING @@ -402,57 +369,26 @@ void llvm_gcda_start_file(const char *orig_filename, const char version[4], #endif } -/* Given an array of pointers to counters (counters), increment the n-th one, - * where we're also given a pointer to n (predecessor). - */ -COMPILER_RT_VISIBILITY -void llvm_gcda_increment_indirect_counter(uint32_t *predecessor, - uint64_t **counters) { - uint64_t *counter; - uint32_t pred; - - pred = *predecessor; - if (pred == 0xffffffff) - return; - counter = counters[pred]; - - /* Don't crash if the pred# is out of sync. This can happen due to threads, - or because of a TODO in GCOVProfiling.cpp buildEdgeLookupTable(). */ - if (counter) - ++*counter; -#ifdef DEBUG_GCDAPROFILING - else - fprintf(stderr, - "llvmgcda: increment_indirect_counter counters=%08llx, pred=%u\n", - *counter, *predecessor); -#endif -} - COMPILER_RT_VISIBILITY -void llvm_gcda_emit_function(uint32_t ident, const char *function_name, - uint32_t func_checksum, uint8_t use_extra_checksum, +void llvm_gcda_emit_function(uint32_t ident, uint32_t func_checksum, uint32_t cfg_checksum) { uint32_t len = 2; + int use_extra_checksum = gcov_version >= 47; if (use_extra_checksum) len++; #ifdef DEBUG_GCDAPROFILING - fprintf(stderr, "llvmgcda: function id=0x%08x name=%s\n", ident, - function_name ? function_name : "NULL"); + fprintf(stderr, "llvmgcda: function id=0x%08x\n", ident); #endif if (!output_file) return; /* function tag */ - write_bytes("\0\0\0\1", 4); - if (function_name) - len += 1 + length_of_string(function_name); + write_32bit_value(GCOV_TAG_FUNCTION); write_32bit_value(len); write_32bit_value(ident); write_32bit_value(func_checksum); if (use_extra_checksum) write_32bit_value(cfg_checksum); - if (function_name) - write_string(function_name); } COMPILER_RT_VISIBILITY @@ -464,11 +400,11 @@ void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { if (!output_file) return; - val = read_le_32bit_value(); + val = read_32bit_value(); if (val != (uint32_t)-1) { /* There are counters present in the file. Merge them. */ - if (val != 0x01a10000) { + if (val != GCOV_TAG_COUNTER_ARCS) { fprintf(stderr, "profiling: %s: cannot merge previous GCDA file: " "corrupt arc tag (0x%08x)\n", filename, val); @@ -491,7 +427,7 @@ void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { cur_pos = save_cur_pos; /* Counter #1 (arcs) tag */ - write_bytes("\0\0\xa1\1", 4); + write_32bit_value(GCOV_TAG_COUNTER_ARCS); write_32bit_value(num_counters * 2); for (i = 0; i < num_counters; ++i) { counters[i] += (old_ctrs ? old_ctrs[i] : 0); @@ -508,9 +444,7 @@ void llvm_gcda_emit_arcs(uint32_t num_counters, uint64_t *counters) { } COMPILER_RT_VISIBILITY -void llvm_gcda_summary_info() { - const uint32_t obj_summary_len = 9; /* Length for gcov compatibility. */ - uint32_t i; +void llvm_gcda_summary_info(void) { uint32_t runs = 1; static uint32_t run_counted = 0; // We only want to increase the run count once. uint32_t val = 0; @@ -518,46 +452,53 @@ void llvm_gcda_summary_info() { if (!output_file) return; - val = read_le_32bit_value(); + val = read_32bit_value(); if (val != (uint32_t)-1) { /* There are counters present in the file. Merge them. */ - if (val != 0xa1000000) { - fprintf(stderr, "profiling: %s: cannot merge previous run count: " - "corrupt object tag (0x%08x)\n", + uint32_t gcov_tag = + gcov_version >= 90 ? GCOV_TAG_OBJECT_SUMMARY : GCOV_TAG_PROGRAM_SUMMARY; + if (val != gcov_tag) { + fprintf(stderr, + "profiling: %s: cannot merge previous run count: " + "corrupt object tag (0x%08x)\n", filename, val); return; } val = read_32bit_value(); /* length */ - if (val != obj_summary_len) { - fprintf(stderr, "profiling: %s: cannot merge previous run count: " - "mismatched object length (%d)\n", - filename, val); - return; + uint32_t prev_runs; + if (gcov_version < 90) { + read_32bit_value(); + read_32bit_value(); + prev_runs = read_32bit_value(); + } else { + prev_runs = read_32bit_value(); + read_32bit_value(); } - - read_32bit_value(); /* checksum, unused */ - read_32bit_value(); /* num, unused */ - uint32_t prev_runs = read_32bit_value(); + for (uint32_t i = gcov_version < 90 ? 3 : 2; i < val; ++i) + read_32bit_value(); /* Add previous run count to new counter, if not already counted before. */ runs = run_counted ? prev_runs : prev_runs + 1; } cur_pos = save_cur_pos; - /* Object summary tag */ - write_bytes("\0\0\0\xa1", 4); - write_32bit_value(obj_summary_len); - write_32bit_value(0); /* checksum, unused */ - write_32bit_value(0); /* num, unused */ - write_32bit_value(runs); - for (i = 3; i < obj_summary_len; ++i) + if (gcov_version >= 90) { + write_32bit_value(GCOV_TAG_OBJECT_SUMMARY); + write_32bit_value(2); + write_32bit_value(runs); + write_32bit_value(0); // sum_max + } else { + // Before gcov 4.8 (r190952), GCOV_TAG_SUMMARY_LENGTH was 9. r190952 set + // GCOV_TAG_SUMMARY_LENGTH to 22. We simply use the smallest length which + // can make gcov read "Runs:". + write_32bit_value(GCOV_TAG_PROGRAM_SUMMARY); + write_32bit_value(3); write_32bit_value(0); - - /* Program summary tag */ - write_bytes("\0\0\0\xa3", 4); /* tag indicates 1 program */ - write_32bit_value(0); /* 0 length */ + write_32bit_value(0); + write_32bit_value(runs); + } run_counted = 1; @@ -567,7 +508,7 @@ void llvm_gcda_summary_info() { } COMPILER_RT_VISIBILITY -void llvm_gcda_end_file() { +void llvm_gcda_end_file(void) { /* Write out EOF record. */ if (output_file) { write_bytes("\0\0\0\0\0\0\0\0", 8); @@ -609,48 +550,88 @@ void llvm_writeout_files(void) { } } -COMPILER_RT_VISIBILITY -void llvm_delete_writeout_function_list(void) { +#ifndef _WIN32 +// __attribute__((destructor)) and destructors whose priorities are greater than +// 100 run before this function and can thus be tracked. The priority is +// compatible with GCC 7 onwards. +#if __GNUC__ >= 9 +#pragma GCC diagnostic ignored "-Wprio-ctor-dtor" +#endif +__attribute__((destructor(100))) +#endif +static void llvm_writeout_and_clear(void) { + llvm_writeout_files(); fn_list_remove(&writeout_fn_list); } COMPILER_RT_VISIBILITY -void llvm_register_flush_function(fn_ptr fn) { - fn_list_insert(&flush_fn_list, fn); +void llvm_register_reset_function(fn_ptr fn) { + fn_list_insert(&reset_fn_list, fn); } -void __gcov_flush() { - struct fn_node* curr = flush_fn_list.head; +COMPILER_RT_VISIBILITY +void llvm_delete_reset_function_list(void) { fn_list_remove(&reset_fn_list); } + +COMPILER_RT_VISIBILITY +void llvm_reset_counters(void) { + struct fn_node *curr = reset_fn_list.head; while (curr) { - curr->fn(); + if (curr->id == CURRENT_ID) { + curr->fn(); + } curr = curr->next; } } +#if !defined(_WIN32) COMPILER_RT_VISIBILITY -void llvm_delete_flush_function_list(void) { - fn_list_remove(&flush_fn_list); +pid_t __gcov_fork() { + pid_t parent_pid = getpid(); + pid_t pid = fork(); + + if (pid == 0) { + pid_t child_pid = getpid(); + if (child_pid != parent_pid) { + // The pid changed so we've a fork (one could have its own fork function) + // Just reset the counters for this child process + // threads. + llvm_reset_counters(); + } + } + return pid; } +#endif COMPILER_RT_VISIBILITY -void llvm_gcov_init(fn_ptr wfn, fn_ptr ffn) { +void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) { static int atexit_ran = 0; if (wfn) llvm_register_writeout_function(wfn); - if (ffn) - llvm_register_flush_function(ffn); + if (rfn) + llvm_register_reset_function(rfn); if (atexit_ran == 0) { atexit_ran = 1; /* Make sure we write out the data and delete the data structures. */ - atexit(llvm_delete_flush_function_list); - atexit(llvm_delete_writeout_function_list); - atexit(llvm_writeout_files); + atexit(llvm_delete_reset_function_list); +#ifdef _WIN32 + atexit(llvm_writeout_and_clear); +#endif } } +void __gcov_dump(void) { + for (struct fn_node *f = writeout_fn_list.head; f; f = f->next) + f->fn(); +} + +void __gcov_reset(void) { + for (struct fn_node *f = reset_fn_list.head; f; f = f->next) + f->fn(); +} + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.c b/system/lib/compiler-rt/lib/profile/InstrProfiling.c index 087d1cdd2efe1..da04d8ebdec95 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfiling.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.c @@ -6,6 +6,9 @@ |* \*===----------------------------------------------------------------------===*/ +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + #include #include #include @@ -17,26 +20,21 @@ #define INSTR_PROF_VALUE_PROF_DATA #include "profile/InstrProfData.inc" +static uint32_t __llvm_profile_global_timestamp = 1; -COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = INSTR_PROF_RAW_VERSION; +COMPILER_RT_VISIBILITY +void INSTR_PROF_PROFILE_SET_TIMESTAMP(uint64_t *Probe) { + if (*Probe == 0 || *Probe == (uint64_t)-1) + *Probe = __llvm_profile_global_timestamp++; +} COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_magic(void) { return sizeof(void *) == sizeof(uint64_t) ? (INSTR_PROF_RAW_MAGIC_64) : (INSTR_PROF_RAW_MAGIC_32); } -static unsigned ProfileDumped = 0; - -COMPILER_RT_VISIBILITY unsigned lprofProfileDumped() { - return ProfileDumped; -} - -COMPILER_RT_VISIBILITY void lprofSetProfileDumped() { - ProfileDumped = 1; -} - -COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped() { - lprofSetProfileDumped(); +COMPILER_RT_VISIBILITY void __llvm_profile_set_dumped(void) { + lprofSetProfileDumped(1); } /* Return the number of bytes needed to add to SizeInBytes to make it @@ -48,14 +46,23 @@ __llvm_profile_get_num_padding_bytes(uint64_t SizeInBytes) { } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_version(void) { - return __llvm_profile_raw_version; + return INSTR_PROF_RAW_VERSION_VAR; } COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { - uint64_t *I = __llvm_profile_begin_counters(); - uint64_t *E = __llvm_profile_end_counters(); + if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) + __llvm_profile_global_timestamp = 1; + + char *I = __llvm_profile_begin_counters(); + char *E = __llvm_profile_end_counters(); + + char ResetValue = + (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) ? 0xFF : 0; + memset(I, ResetValue, E - I); - memset(I, 0, sizeof(uint64_t) * (E - I)); + I = __llvm_profile_begin_bitmap(); + E = __llvm_profile_end_bitmap(); + memset(I, 0x0, E - I); const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); @@ -72,13 +79,13 @@ COMPILER_RT_VISIBILITY void __llvm_profile_reset_counters(void) { CurrentVSiteCount += DI->NumValueSites[VKI]; for (i = 0; i < CurrentVSiteCount; ++i) { - ValueProfNode *CurrentVNode = ValueCounters[i]; + ValueProfNode *CurrVNode = ValueCounters[i]; - while (CurrentVNode) { - CurrentVNode->Count = 0; - CurrentVNode = CurrentVNode->Next; + while (CurrVNode) { + CurrVNode->Count = 0; + CurrVNode = CurrVNode->Next; } } } - ProfileDumped = 0; + lprofSetProfileDumped(0); } diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.h b/system/lib/compiler-rt/lib/profile/InstrProfiling.h index 3a3bab3d0b4c8..6906d52eacaf1 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfiling.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.h @@ -12,6 +12,17 @@ #include "InstrProfilingPort.h" #include +// Make sure __LLVM_INSTR_PROFILE_GENERATE is always defined before +// including instr_prof_interface.h so the interface functions are +// declared correctly for the runtime. +// __LLVM_INSTR_PROFILE_GENERATE is always `#undef`ed after the header, +// because compiler-rt does not support profiling the profiling runtime itself. +#ifndef __LLVM_INSTR_PROFILE_GENERATE +#define __LLVM_INSTR_PROFILE_GENERATE +#endif +#include "profile/instr_prof_interface.h" +#undef __LLVM_INSTR_PROFILE_GENERATE + #define INSTR_PROF_VISIBILITY COMPILER_RT_VISIBILITY #include "profile/InstrProfData.inc" @@ -38,6 +49,11 @@ typedef struct ValueProfNode { #include "profile/InstrProfData.inc" } ValueProfNode; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData { +#define INSTR_PROF_VTABLE_DATA(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} VTableProfData; + /*! * \brief Return 1 if profile counters are continuously synced to the raw * profile via an mmap(). This is in contrast to the default mode, in which @@ -54,6 +70,21 @@ int __llvm_profile_is_continuous_mode_enabled(void); */ void __llvm_profile_enable_continuous_mode(void); +/*! + * \brief Disable continuous mode. + * + */ +void __llvm_profile_disable_continuous_mode(void); + +/*! + * \brief Set the page size. + * + * This is a pre-requisite for enabling continuous mode. The buffer size + * calculation code inside of libprofile cannot simply call getpagesize(), as + * it is not allowed to depend on libc. + */ +void __llvm_profile_set_page_size(unsigned PageSize); + /*! * \brief Get number of bytes necessary to pad the argument to eight * byte boundary. @@ -77,35 +108,35 @@ const __llvm_profile_data *__llvm_profile_begin_data(void); const __llvm_profile_data *__llvm_profile_end_data(void); const char *__llvm_profile_begin_names(void); const char *__llvm_profile_end_names(void); -uint64_t *__llvm_profile_begin_counters(void); -uint64_t *__llvm_profile_end_counters(void); +const char *__llvm_profile_begin_vtabnames(void); +const char *__llvm_profile_end_vtabnames(void); +char *__llvm_profile_begin_counters(void); +char *__llvm_profile_end_counters(void); +char *__llvm_profile_begin_bitmap(void); +char *__llvm_profile_end_bitmap(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +const VTableProfData *__llvm_profile_begin_vtables(); +const VTableProfData *__llvm_profile_end_vtables(); uint32_t *__llvm_profile_begin_orderfile(); -/*! - * \brief Clear profile counters to zero. - * - */ -void __llvm_profile_reset_counters(void); - /*! * \brief Merge profile data from buffer. * - * Read profile data form buffer \p Profile and merge with - * in-process profile counters. The client is expected to - * have checked or already knows the profile data in the - * buffer matches the in-process counter structure before - * calling it. + * Read profile data from buffer \p Profile and merge with in-process profile + * counters and bitmaps. The client is expected to have checked or already + * know the profile data in the buffer matches the in-process counter + * structure before calling it. Returns 0 (success) if the profile data is + * valid. Upon reading invalid/corrupted profile data, returns 1 (failure). */ -void __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); +int __llvm_profile_merge_from_buffer(const char *Profile, uint64_t Size); /*! \brief Check if profile in buffer matches the current binary. * * Returns 0 (success) if the profile data in buffer \p Profile with size * \p Size was generated by the same binary and therefore matches - * structurally the in-process counters. If the profile data in buffer is - * not compatible, the interface returns 1 (failure). + * structurally the in-process counters and bitmaps. If the profile data in + * buffer is not compatible, the interface returns 1 (failure). */ int __llvm_profile_check_compatibility(const char *Profile, uint64_t Size); @@ -139,53 +170,10 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, int __llvm_profile_write_file(void); int __llvm_orderfile_write_file(void); -/*! - * \brief this is a wrapper interface to \c __llvm_profile_write_file. - * After this interface is invoked, a arleady dumped flag will be set - * so that profile won't be dumped again during program exit. - * Invocation of interface __llvm_profile_reset_counters will clear - * the flag. This interface is designed to be used to collect profile - * data from user selected hot regions. The use model is - * __llvm_profile_reset_counters(); - * ... hot region 1 - * __llvm_profile_dump(); - * .. some other code - * __llvm_profile_reset_counters(); - * ... hot region 2 - * __llvm_profile_dump(); - * - * It is expected that on-line profile merging is on with \c %m specifier - * used in profile filename . If merging is not turned on, user is expected - * to invoke __llvm_profile_set_filename to specify different profile names - * for different regions before dumping to avoid profile write clobbering. - */ -int __llvm_profile_dump(void); - -int __llvm_orderfile_dump(void); /*! - * \brief Set the filename for writing instrumentation data. - * - * Sets the filename to be used for subsequent calls to - * \a __llvm_profile_write_file(). - * - * \c Name is not copied, so it must remain valid. Passing NULL resets the - * filename logic to the default behaviour. - * - * Note: There may be multiple copies of the profile runtime (one for each - * instrumented image/DSO). This API only modifies the filename within the - * copy of the runtime available to the calling image. - * - * Warning: This is a no-op if continuous mode (\ref - * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is - * that in continuous mode, profile counters are mmap()'d to the profile at - * program initialization time. Support for transferring the mmap'd profile - * counts to a new file has not been implemented. - */ -void __llvm_profile_set_filename(const char *Name); - -/*! - * \brief Set the FILE object for writing instrumentation data. + * \brief Set the FILE object for writing instrumentation data. Return 0 if set + * successfully or return 1 if failed. * * Sets the FILE object to be used for subsequent calls to * \a __llvm_profile_write_file(). The profile file name set by environment @@ -204,13 +192,12 @@ void __llvm_profile_set_filename(const char *Name); * instrumented image/DSO). This API only modifies the file object within the * copy of the runtime available to the calling image. * - * Warning: This is a no-op if continuous mode (\ref - * __llvm_profile_is_continuous_mode_enabled) is on. The reason for this is - * that in continuous mode, profile counters are mmap()'d to the profile at - * program initialization time. Support for transferring the mmap'd profile - * counts to a new file has not been implemented. + * Warning: This is a no-op if EnableMerge is 0 in continuous mode (\ref + * __llvm_profile_is_continuous_mode_enabled), because disable merging requires + * copying the old profile file to new profile file and this function is usually + * used when the proess doesn't have permission to open file. */ -void __llvm_profile_set_file_object(FILE *File, int EnableMerge); +int __llvm_profile_set_file_object(FILE *File, int EnableMerge); /*! \brief Register to write instrumentation data to file at exit. */ int __llvm_profile_register_write_file_atexit(void); @@ -218,6 +205,9 @@ int __llvm_profile_register_write_file_atexit(void); /*! \brief Initialize file handling. */ void __llvm_profile_initialize_file(void); +/*! \brief Initialize the profile runtime. */ +void __llvm_profile_initialize(void); + /*! * \brief Return path prefix (excluding the base filename) of the profile data. * This is useful for users using \c -fprofile-generate=./path_prefix who do @@ -248,26 +238,54 @@ uint64_t __llvm_profile_get_magic(void); uint64_t __llvm_profile_get_version(void); /*! \brief Get the number of entries in the profile data section. */ +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End); + +/*! \brief Get the size of the profile data section in bytes. */ uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, const __llvm_profile_data *End); -/* ! \brief Given the sizes of the data and counter information, return the - * number of padding bytes before and after the counters, and after the names, - * in the raw profile. - * - * Note: In this context, "size" means "number of entries", i.e. the first two - * arguments must be the result of __llvm_profile_get_data_size() and of - * (__llvm_profile_end_counters() - __llvm_profile_begin_counters()) resp. +/*! \brief Get the size in bytes of a single counter entry. */ +size_t __llvm_profile_counter_entry_size(void); + +/*! \brief Get the number of entries in the profile counters section. */ +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End); + +/*! \brief Get the size of the profile counters section in bytes. */ +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End); + +/*! \brief Get the number of bytes in the profile bitmap section. */ +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End); + +/*! \brief Get the size of the profile name section in bytes. */ +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End); + +/*! \brief Get the number of virtual table profile data entries */ +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End); + +/*! \brief Get the size of virtual table profile data in bytes. */ +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End); + +/* ! \brief Given the sizes of the data and counter information, computes the + * number of padding bytes before and after the counter section, as well as the + * number of padding bytes after other setions in the raw profile. + * Returns -1 upon errors and 0 upon success. Output parameters should be used + * iff return value is 0. * * Note: When mmap() mode is disabled, no padding bytes before/after counters * are needed. However, in mmap() mode, the counter section in the raw profile * must be page-aligned: this API computes the number of padding bytes * needed to achieve that. */ -void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, +int __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, - uint64_t *PaddingBytesAfterNames); + uint64_t *PaddingBytesAfterBitmap, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVNames); /*! * \brief Set the flag that profile data has been dumped to the file. @@ -289,16 +307,15 @@ void __llvm_profile_set_dumped(); COMPILER_RT_VISIBILITY extern int INSTR_PROF_PROFILE_RUNTIME_VAR; /*! - * This variable is defined in InstrProfiling.c. Its main purpose is to - * encode the raw profile version value and other format related information - * such as whether the profile is from IR based instrumentation. The variable - * is defined as weak so that compiler can emit an overriding definition - * depending on user option. Since we don't support mixing FE and IR based - * data in the same raw profile data file (in other words, shared libs and - * main program are expected to be instrumented in the same way), there is - * no need for this variable to be hidden. + * This variable is defined in InstrProfilingVersionVar.c as a hidden symbol + * (except on Apple platforms where this symbol is checked by TAPI). Its main + * purpose is to encode the raw profile version value and other format related + * information such as whether the profile is from IR based instrumentation. The + * variable is defined as weak so that compiler can emit an overriding + * definition depending on user option. */ -extern uint64_t INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */ +COMPILER_RT_VISIBILITY extern uint64_t + INSTR_PROF_RAW_VERSION_VAR; /* __llvm_profile_raw_version */ /*! * This variable is a weak symbol defined in InstrProfiling.c. It allows diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c index 174280fd4b528..1c451d7ec7563 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -6,6 +6,9 @@ |* \*===----------------------------------------------------------------------===*/ +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + #include "InstrProfiling.h" #include "InstrProfilingInternal.h" #include "InstrProfilingPort.h" @@ -18,92 +21,211 @@ * layering is violated. */ static int ContinuouslySyncProfile = 0; +/* The system page size. Only valid when non-zero. If 0, the page size is + * unavailable. */ +static unsigned PageSize = 0; + COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) { - return ContinuouslySyncProfile; + return ContinuouslySyncProfile && PageSize; } COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { ContinuouslySyncProfile = 1; } +COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { + ContinuouslySyncProfile = 0; +} + +COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { + PageSize = PS; +} + COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer(void) { const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return __llvm_profile_get_size_for_buffer_internal( - DataBegin, DataEnd, CountersBegin, CountersEnd, NamesBegin, NamesEnd); + DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd, + NamesBegin, NamesEnd, VTableBegin, VTableEnd, VNamesBegin, VNamesEnd); } +// NOTE: Caller should guarantee that `Begin` and `End` specifies a half-open +// interval [Begin, End). Namely, `End` is one-byte past the end of the array. COMPILER_RT_VISIBILITY -uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, - const __llvm_profile_data *End) { +uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) / sizeof(__llvm_profile_data); } +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin, + const __llvm_profile_data *End) { + return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data); +} + +// Counts the number of `VTableProfData` elements within the range of [Begin, +// End). Caller should guarantee that End points to one byte past the inclusive +// range. +// FIXME: Add a compiler-rt test to make sure the number of vtables in the +// raw profile is the same as the number of vtable elements in the instrumented +// binary. +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_vtable(const VTableProfData *Begin, + const VTableProfData *End) { + // Convert pointers to intptr_t to use integer arithmetic. + intptr_t EndI = (intptr_t)End, BeginI = (intptr_t)Begin; + return (EndI - BeginI) / sizeof(VTableProfData); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_vtable_section_size(const VTableProfData *Begin, + const VTableProfData *End) { + return (intptr_t)(End) - (intptr_t)(Begin); +} + +COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) + return sizeof(uint8_t); + return sizeof(uint64_t); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) { + intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End; + return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) / + __llvm_profile_counter_entry_size(); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) { + return __llvm_profile_get_num_counters(Begin, End) * + __llvm_profile_counter_entry_size(); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin, + const char *End) { + return (End - Begin); +} + +COMPILER_RT_VISIBILITY +uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) { + return End - Begin; +} + /// Calculate the number of padding bytes needed to add to \p Offset in order /// for (\p Offset + Padding) to be page-aligned. -static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset, - unsigned PageSize) { +static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) { uint64_t OffsetModPage = Offset % PageSize; if (OffsetModPage > 0) return PageSize - OffsetModPage; return 0; } +static int needsCounterPadding(void) { +#if defined(__APPLE__) + return __llvm_profile_is_continuous_mode_enabled(); +#else + return 0; +#endif +} + COMPILER_RT_VISIBILITY -void __llvm_profile_get_padding_sizes_for_counters( - uint64_t DataSize, uint64_t CountersSize, uint64_t NamesSize, +int __llvm_profile_get_padding_sizes_for_counters( + uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes, + uint64_t NamesSize, uint64_t VTableSize, uint64_t VNameSize, uint64_t *PaddingBytesBeforeCounters, uint64_t *PaddingBytesAfterCounters, - uint64_t *PaddingBytesAfterNames) { - if (!__llvm_profile_is_continuous_mode_enabled()) { + uint64_t *PaddingBytesAfterBitmapBytes, uint64_t *PaddingBytesAfterNames, + uint64_t *PaddingBytesAfterVTable, uint64_t *PaddingBytesAfterVName) { + // Counter padding is needed only if continuous mode is enabled. + if (!needsCounterPadding()) { *PaddingBytesBeforeCounters = 0; - *PaddingBytesAfterCounters = 0; + *PaddingBytesAfterCounters = + __llvm_profile_get_num_padding_bytes(CountersSize); + *PaddingBytesAfterBitmapBytes = + __llvm_profile_get_num_padding_bytes(NumBitmapBytes); *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize); - return; + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = + __llvm_profile_get_num_padding_bytes(VTableSize); + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = __llvm_profile_get_num_padding_bytes(VNameSize); + return 0; } + // Value profiling not supported in continuous mode at profile-write time. + // Return -1 to alert the incompatibility. + if (VTableSize != 0 || VNameSize != 0) + return -1; + // In continuous mode, the file offsets for headers and for the start of // counter sections need to be page-aligned. - unsigned PageSize = getpagesize(); - uint64_t DataSizeInBytes = DataSize * sizeof(__llvm_profile_data); - uint64_t CountersSizeInBytes = CountersSize * sizeof(uint64_t); - *PaddingBytesBeforeCounters = calculateBytesNeededToPageAlign( - sizeof(__llvm_profile_header) + DataSizeInBytes, PageSize); - *PaddingBytesAfterCounters = - calculateBytesNeededToPageAlign(CountersSizeInBytes, PageSize); - *PaddingBytesAfterNames = - calculateBytesNeededToPageAlign(NamesSize, PageSize); + *PaddingBytesBeforeCounters = + calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize); + *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize); + *PaddingBytesAfterBitmapBytes = + calculateBytesNeededToPageAlign(NumBitmapBytes); + *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize); + // Set these two variables to zero to avoid uninitialized variables + // even if VTableSize and VNameSize are known to be zero. + if (PaddingBytesAfterVTable != NULL) + *PaddingBytesAfterVTable = 0; + if (PaddingBytesAfterVName != NULL) + *PaddingBytesAfterVName = 0; + return 0; } COMPILER_RT_VISIBILITY uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const char *NamesBegin, const char *NamesEnd) { + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd) { /* Match logic in __llvm_profile_write_buffer(). */ const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - uint64_t CountersSize = CountersEnd - CountersBegin; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t VTableSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNameSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + DataSize, CountersSize, NumBitmapBytes, NamesSize, 0 /* VTableSize */, + 0 /* VNameSize */, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames); - return sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters + - (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters + - NamesSize + PaddingBytesAfterNames; + return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + + DataSize + PaddingBytesBeforeCounters + CountersSize + + PaddingBytesAfterCounters + NumBitmapBytes + + PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames + + VTableSize + PaddingBytesAfterVTable + VNameSize + + PaddingBytesAfterVNames; } COMPILER_RT_VISIBILITY @@ -120,10 +242,15 @@ COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) { COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, - const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd) { + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd) { ProfDataWriter BufferWriter; initBufferWriter(&BufferWriter, Buffer); - return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin, - CountersEnd, 0, NamesBegin, NamesEnd, 0); + // Set virtual table arguments to NULL since they are not supported yet. + return lprofWriteDataImpl( + &BufferWriter, DataBegin, DataEnd, CountersBegin, CountersEnd, + BitmapBegin, BitmapEnd, /*VPDataReader=*/0, NamesBegin, NamesEnd, + /*VTableBegin=*/NULL, /*VTableEnd=*/NULL, /*VNamesBegin=*/NULL, + /*VNamesEnd=*/NULL, /*SkipNameDataWrite=*/0); } diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c index 7f3727eed92db..1c58584d2d4f7 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -8,6 +8,7 @@ #if !defined(__Fuchsia__) +#include #include #include #include @@ -72,6 +73,7 @@ typedef struct lprofFilename { unsigned OwnsFilenamePat; const char *ProfilePathPrefix; char PidChars[MAX_PID_SIZE]; + char *TmpDir; char Hostname[COMPILER_RT_MAX_HOSTLEN]; unsigned NumPids; unsigned NumHosts; @@ -86,34 +88,207 @@ typedef struct lprofFilename { ProfileNameSpecifier PNS; } lprofFilename; -static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0}, - 0, 0, 0, PNS_unknown}; +static lprofFilename lprofCurFilename = {0, 0, 0, {0}, NULL, + {0}, 0, 0, 0, PNS_unknown}; static int ProfileMergeRequested = 0; -static int isProfileMergeRequested() { return ProfileMergeRequested; } +static int getProfileFileSizeForMerging(FILE *ProfileFile, + uint64_t *ProfileFileSize); + +#if defined(__APPLE__) +static const int ContinuousModeSupported = 1; +static const int UseBiasVar = 0; +static const char *FileOpenMode = "a+b"; +static void *BiasAddr = NULL; +static void *BiasDefaultAddr = NULL; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + const char *NamesBegin = __llvm_profile_begin_names(); + const char *NamesEnd = __llvm_profile_end_names(); + const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + + /* Check that the counter, bitmap, and data sections in this image are + * page-aligned. */ + unsigned PageSize = getpagesize(); + if ((intptr_t)CountersBegin % PageSize != 0) { + PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", + CountersBegin, PageSize); + return 1; + } + if ((intptr_t)BitmapBegin % PageSize != 0) { + PROF_ERR("Bitmap section not page-aligned (start = %p, pagesz = %u).\n", + BitmapBegin, PageSize); + return 1; + } + if ((intptr_t)DataBegin % PageSize != 0) { + PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", + DataBegin, PageSize); + return 1; + } + + int Fileno = fileno(File); + /* Determine how much padding is needed before/after the counters and + * after the names. */ + uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, + PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + __llvm_profile_get_padding_sizes_for_counters( + DataSize, CountersSize, NumBitmapBytes, NamesSize, /*VTableSize=*/0, + /*VNameSize=*/0, &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters, + &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames, + &PaddingBytesAfterVTable, &PaddingBytesAfterVNames); + + uint64_t PageAlignedCountersLength = CountersSize + PaddingBytesAfterCounters; + uint64_t FileOffsetToCounters = CurrentFileOffset + + sizeof(__llvm_profile_header) + DataSize + + PaddingBytesBeforeCounters; + void *CounterMmap = mmap((void *)CountersBegin, PageAlignedCountersLength, + PROT_READ | PROT_WRITE, MAP_FIXED | MAP_SHARED, + Fileno, FileOffsetToCounters); + if (CounterMmap != CountersBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - CountersBegin: %p\n" + " - PageAlignedCountersLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToCounters: %" PRIu64 "\n", + strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, + FileOffsetToCounters); + return 1; + } + + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + uint64_t PageAlignedBitmapLength = + NumBitmapBytes + PaddingBytesAfterBitmapBytes; + uint64_t FileOffsetToBitmap = + FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters; + void *BitmapMmap = + mmap((void *)BitmapBegin, PageAlignedBitmapLength, PROT_READ | PROT_WRITE, + MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToBitmap); + if (BitmapMmap != BitmapBegin) { + PROF_ERR( + "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" + " - BitmapBegin: %p\n" + " - PageAlignedBitmapLength: %" PRIu64 "\n" + " - Fileno: %d\n" + " - FileOffsetToBitmap: %" PRIu64 "\n", + strerror(errno), BitmapBegin, PageAlignedBitmapLength, Fileno, + FileOffsetToBitmap); + return 1; + } + return 0; +} +#elif defined(__ELF__) || defined(_WIN32) + +#define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ + INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default) +COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; + +/* This variable is a weak external reference which could be used to detect + * whether or not the compiler defined this symbol. */ +#if defined(_MSC_VER) +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +#if defined(_M_IX86) || defined(__i386__) +#define WIN_SYM_PREFIX "_" +#else +#define WIN_SYM_PREFIX +#endif +#pragma comment( \ + linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)) +#else +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR + __attribute__((weak, alias(INSTR_PROF_QUOTE( + INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)))); +#endif +static const int ContinuousModeSupported = 1; +static const int UseBiasVar = 1; +/* TODO: If there are two DSOs, the second DSO initilization will truncate the + * first profile file. */ +static const char *FileOpenMode = "w+b"; +/* This symbol is defined by the compiler when runtime counter relocation is + * used and runtime provides a weak alias so we can check if it's defined. */ +static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + /* Get the sizes of various profile data sections. Taken from + * __llvm_profile_get_size_for_buffer(). */ + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); + uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + /* Get the file size. */ + uint64_t FileSize = 0; + if (getProfileFileSizeForMerging(File, &FileSize)) + return 1; + + int Fileno = fileno(File); + uint64_t FileOffsetToCounters = + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; + + /* Map the profile. */ + char *Profile = (char *)mmap(NULL, FileSize, PROT_READ | PROT_WRITE, + MAP_SHARED, Fileno, 0); + if (Profile == MAP_FAILED) { + PROF_ERR("Unable to mmap profile: %s\n", strerror(errno)); + return 1; + } + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Profile - (uintptr_t)CountersBegin + FileOffsetToCounters; + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); + + /* BIAS MODE not supported yet for Bitmap (MCDC). */ + + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); + return 0; +} +#else +static const int ContinuousModeSupported = 0; +static const int UseBiasVar = 0; +static const char *FileOpenMode = "a+b"; +static void *BiasAddr = NULL; +static void *BiasDefaultAddr = NULL; +static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { + return 0; +} +#endif + +static int isProfileMergeRequested(void) { return ProfileMergeRequested; } static void setProfileMergeRequested(int EnableMerge) { ProfileMergeRequested = EnableMerge; } static FILE *ProfileFile = NULL; -static FILE *getProfileFile() { return ProfileFile; } +static FILE *getProfileFile(void) { return ProfileFile; } static void setProfileFile(FILE *File) { ProfileFile = File; } -COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File, - int EnableMerge) { - if (__llvm_profile_is_continuous_mode_enabled()) { - PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because " - "continuous sync mode (%%c) is enabled", - fileno(File)); - return; - } - setProfileFile(File); - setProfileMergeRequested(EnableMerge); -} - -static int getCurFilenameLength(); +static int getCurFilenameLength(void); static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf); -static unsigned doMerging() { +static unsigned doMerging(void) { return lprofCurFilename.MergePoolSize || isProfileMergeRequested(); } @@ -164,17 +339,17 @@ static void initFileWriter(ProfDataWriter *This, FILE *File) { COMPILER_RT_VISIBILITY ProfBufferIO * lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) { FreeHook = &free; - DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1); + DynamicBufferIOBuffer = (uint8_t *)calloc(1, BufferSz); VPBufferSize = BufferSz; ProfDataWriter *fileWriter = - (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1); + (ProfDataWriter *)calloc(1, sizeof(ProfDataWriter)); initFileWriter(fileWriter, File); ProfBufferIO *IO = lprofCreateBufferIO(fileWriter); IO->OwnFileWriter = 1; return IO; } -static void setupIOBuffer() { +static void setupIOBuffer(void) { const char *BufferSzStr = 0; BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE"); if (BufferSzStr && BufferSzStr[0]) { @@ -259,11 +434,16 @@ static int doProfileMerging(FILE *ProfileFile, int *MergeDone) { return -1; /* Now start merging */ - __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize); + if (__llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize)) { + PROF_ERR("%s\n", "Invalid profile data to merge"); + (void)munmap(ProfileBuffer, ProfileFileSize); + return -1; + } - // Truncate the file in case merging of value profile did not happend to + // Truncate the file in case merging of value profile did not happen to // prevent from leaving garbage data at the end of the profile file. - COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer()); + (void)COMPILER_RT_FTRUNCATE(ProfileFile, + __llvm_profile_get_size_for_buffer()); (void)munmap(ProfileBuffer, ProfileFileSize); *MergeDone = 1; @@ -290,13 +470,15 @@ static void createProfileDir(const char *Filename) { * its instrumented shared libraries dump profile data into their own data file. */ static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) { - FILE *ProfileFile = NULL; + FILE *ProfileFile = getProfileFile(); int rc; - - ProfileFile = getProfileFile(); - if (ProfileFile) { + // initializeProfileForContinuousMode will lock the profile, but if + // ProfileFile is set by user via __llvm_profile_set_file_object, it's assumed + // unlocked at this point. + if (ProfileFile && !__llvm_profile_is_continuous_mode_enabled()) { lprofLockFileHandle(ProfileFile); - } else { + } + if (!ProfileFile) { createProfileDir(ProfileFileName); ProfileFile = lprofOpenFileEx(ProfileFileName); } @@ -347,7 +529,7 @@ static int writeFile(const char *OutputName) { if (OutputFile == getProfileFile()) { fflush(OutputFile); - if (doMerging()) { + if (doMerging() && !__llvm_profile_is_continuous_mode_enabled()) { lprofUnlockFileHandle(OutputFile); } } else { @@ -419,14 +601,6 @@ static void truncateCurrentFile(void) { fclose(File); } -#ifndef _MSC_VER -static void assertIsZero(int *i) { - if (*i) - PROF_WARN("Expected flag to be 0, but got: %d\n", *i); -} -#endif - -#if !defined(__Fuchsia__) && !defined(_WIN32) /* Write a partial profile to \p Filename, which is required to be backed by * the open file object \p File. */ static int writeProfileWithFileObject(const char *Filename, FILE *File) { @@ -438,48 +612,22 @@ static int writeProfileWithFileObject(const char *Filename, FILE *File) { return rc; } -/* Unlock the profile \p File and clear the unlock flag. */ -static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) { - if (!*ProfileRequiresUnlock) { - PROF_WARN("%s", "Expected to require profile unlock\n"); - } - lprofUnlockFileHandle(File); - *ProfileRequiresUnlock = 0; -} -#endif // !defined(__Fuchsia__) && !defined(_WIN32) - static void initializeProfileForContinuousMode(void) { if (!__llvm_profile_is_continuous_mode_enabled()) return; - -#if defined(__Fuchsia__) || defined(_WIN32) - PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows."); -#else // defined(__Fuchsia__) || defined(_WIN32) - /* Get the sizes of various profile data sections. Taken from - * __llvm_profile_get_size_for_buffer(). */ - const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); - const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); - const char *NamesBegin = __llvm_profile_begin_names(); - const char *NamesEnd = __llvm_profile_end_names(); - const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char); - uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - uint64_t CountersSize = CountersEnd - CountersBegin; - - /* Check that the counter and data sections in this image are page-aligned. */ - unsigned PageSize = getpagesize(); - if ((intptr_t)CountersBegin % PageSize != 0) { - PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n", - CountersBegin, PageSize); + if (!ContinuousModeSupported) { + PROF_ERR("%s\n", "continuous mode is unsupported on this platform"); return; } - if ((intptr_t)DataBegin % PageSize != 0) { - PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n", - DataBegin, PageSize); + if (UseBiasVar && BiasAddr == BiasDefaultAddr) { + PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined"); return; } + /* Get the sizes of counter section. */ + uint64_t CountersSize = __llvm_profile_get_counters_size( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); + int Length = getCurFilenameLength(); char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); const char *Filename = getCurFilename(FilenameBuf, 0); @@ -487,34 +635,8 @@ static void initializeProfileForContinuousMode(void) { return; FILE *File = NULL; - off_t CurrentFileOffset = 0; - off_t OffsetModPage = 0; - - /* Whether an exclusive lock on the profile must be dropped after init. - * Use a cleanup to warn if the unlock does not occur. */ - COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0; - - if (!doMerging()) { - /* We are not merging profiles, so open the raw profile in append mode. */ - File = fopen(Filename, "a+b"); - if (!File) - return; - - /* Check that the offset within the file is page-aligned. */ - CurrentFileOffset = ftello(File); - OffsetModPage = CurrentFileOffset % PageSize; - if (OffsetModPage != 0) { - PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" - "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n", - (uint64_t)CurrentFileOffset, PageSize); - return; - } - - /* Grow the profile so that mmap() can succeed. Leak the file handle, as - * the file should stay open. */ - if (writeProfileWithFileObject(Filename, File) != 0) - return; - } else { + uint64_t CurrentFileOffset = 0; + if (doMerging()) { /* We are merging profiles. Map the counter section as shared memory into * the profile, i.e. into each participating process. An increment in one * process should be visible to every other process with the same counter @@ -523,65 +645,80 @@ static void initializeProfileForContinuousMode(void) { if (!File) return; - ProfileRequiresUnlock = 1; - - uint64_t ProfileFileSize; - if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) - return unlockProfile(&ProfileRequiresUnlock, File); - + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } if (ProfileFileSize == 0) { /* Grow the profile so that mmap() can succeed. Leak the file handle, as * the file should stay open. */ - if (writeProfileWithFileObject(Filename, File) != 0) - return unlockProfile(&ProfileRequiresUnlock, File); + if (writeProfileWithFileObject(Filename, File) != 0) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } } else { /* The merged profile has a non-zero length. Check that it is compatible * with the data in this process. */ char *ProfileBuffer; - if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 || - munmap(ProfileBuffer, ProfileFileSize) == -1) - return unlockProfile(&ProfileRequiresUnlock, File); + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) { + lprofUnlockFileHandle(File); + fclose(File); + return; + } + (void)munmap(ProfileBuffer, ProfileFileSize); + } + } else { + File = fopen(Filename, FileOpenMode); + if (!File) + return; + /* Check that the offset within the file is page-aligned. */ + CurrentFileOffset = ftell(File); + unsigned PageSize = getpagesize(); + if (CurrentFileOffset % PageSize != 0) { + PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not" + "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n", + (uint64_t)CurrentFileOffset, PageSize); + fclose(File); + return; + } + if (writeProfileWithFileObject(Filename, File) != 0) { + fclose(File); + return; } } - int Fileno = fileno(File); - - /* Determine how much padding is needed before/after the counters and after - * the names. */ - uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; - __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); - - uint64_t PageAlignedCountersLength = - (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters; - uint64_t FileOffsetToCounters = - CurrentFileOffset + sizeof(__llvm_profile_header) + - (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters; + /* mmap() the profile counters so long as there is at least one counter. + * If there aren't any counters, mmap() would fail with EINVAL. */ + if (CountersSize > 0) + mmapForContinuousMode(CurrentFileOffset, File); - uint64_t *CounterMmap = (uint64_t *)mmap( - (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE, - MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters); - if (CounterMmap != CountersBegin) { - PROF_ERR( - "Continuous counter sync mode is enabled, but mmap() failed (%s).\n" - " - CountersBegin: %p\n" - " - PageAlignedCountersLength: %" PRIu64 "\n" - " - Fileno: %d\n" - " - FileOffsetToCounters: %" PRIu64 "\n", - strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno, - FileOffsetToCounters); + if (doMerging()) { + lprofUnlockFileHandle(File); + } + if (File != NULL) { + fclose(File); } - - unlockProfile(&ProfileRequiresUnlock, File); -#endif // defined(__Fuchsia__) || defined(_WIN32) } static const char *DefaultProfileName = "default.profraw"; static void resetFilenameToDefault(void) { if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif free((void *)lprofCurFilename.FilenamePat); +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif } memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); lprofCurFilename.FilenamePat = DefaultProfileName; @@ -606,6 +743,13 @@ static unsigned getMergePoolSize(const char *FilenamePat, int *I) { return 0; } +/* Assert that Idx does index past a string null terminator. Return the + * result of the check. */ +static int checkBounds(int Idx, int Strlen) { + assert(Idx <= Strlen && "Indexing past string null terminator"); + return Idx <= Strlen; +} + /* Parses the pattern string \p FilenamePat and stores the result to * lprofcurFilename structure. */ static int parseFilenamePattern(const char *FilenamePat, @@ -614,7 +758,15 @@ static int parseFilenamePattern(const char *FilenamePat, char *PidChars = &lprofCurFilename.PidChars[0]; char *Hostname = &lprofCurFilename.Hostname[0]; int MergingEnabled = 0; - + int FilenamePatLen = strlen(FilenamePat); + +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif /* Clean up cached prefix and filename. */ if (lprofCurFilename.ProfilePathPrefix) free((void *)lprofCurFilename.ProfilePathPrefix); @@ -622,6 +774,11 @@ static int parseFilenamePattern(const char *FilenamePat, if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) { free((void *)lprofCurFilename.FilenamePat); } +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif memset(&lprofCurFilename, 0, sizeof(lprofCurFilename)); @@ -632,9 +789,12 @@ static int parseFilenamePattern(const char *FilenamePat, lprofCurFilename.OwnsFilenamePat = 1; } /* Check the filename for "%p", which indicates a pid-substitution. */ - for (I = 0; FilenamePat[I]; ++I) + for (I = 0; checkBounds(I, FilenamePatLen) && FilenamePat[I]; ++I) { if (FilenamePat[I] == '%') { - if (FilenamePat[++I] == 'p') { + ++I; /* Advance to the next character. */ + if (!checkBounds(I, FilenamePatLen)) + break; + if (FilenamePat[I] == 'p') { if (!NumPids++) { if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) { PROF_WARN("Unable to get pid for filename pattern %s. Using the " @@ -651,15 +811,29 @@ static int parseFilenamePattern(const char *FilenamePat, FilenamePat); return -1; } + } else if (FilenamePat[I] == 't') { + lprofCurFilename.TmpDir = getenv("TMPDIR"); + if (!lprofCurFilename.TmpDir) { + PROF_WARN("Unable to get the TMPDIR environment variable, referenced " + "in %s. Using the default path.", + FilenamePat); + return -1; + } } else if (FilenamePat[I] == 'c') { if (__llvm_profile_is_continuous_mode_enabled()) { PROF_WARN("%%c specifier can only be specified once in %s.\n", FilenamePat); + __llvm_profile_disable_continuous_mode(); return -1; } - +#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) + __llvm_profile_set_page_size(getpagesize()); __llvm_profile_enable_continuous_mode(); - I++; /* advance to 'c' */ +#else + PROF_WARN("%s", "Continous mode is currently only supported for Mach-O," + " ELF and COFF formats."); + return -1; +#endif } else { unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I); if (!MergePoolSize) @@ -673,6 +847,7 @@ static int parseFilenamePattern(const char *FilenamePat, lprofCurFilename.MergePoolSize = MergePoolSize; } } + } lprofCurFilename.NumPids = NumPids; lprofCurFilename.NumHosts = NumHosts; @@ -715,25 +890,27 @@ static void parseAndSetFilename(const char *FilenamePat, } truncateCurrentFile(); - initializeProfileForContinuousMode(); + if (__llvm_profile_is_continuous_mode_enabled()) + initializeProfileForContinuousMode(); } /* Return buffer length that is required to store the current profile * filename with PID and hostname substitutions. */ /* The length to hold uint64_t followed by 3 digits pool id including '_' */ #define SIGLEN 24 -static int getCurFilenameLength() { +static int getCurFilenameLength(void) { int Len; if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) return 0; if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || - lprofCurFilename.MergePoolSize)) + lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize)) return strlen(lprofCurFilename.FilenamePat); Len = strlen(lprofCurFilename.FilenamePat) + lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) + - lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2); + lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2) + + (lprofCurFilename.TmpDir ? (strlen(lprofCurFilename.TmpDir) - 1) : 0); if (lprofCurFilename.MergePoolSize) Len += SIGLEN; return Len; @@ -745,14 +922,14 @@ static int getCurFilenameLength() { * current filename pattern string is directly returned, unless ForceUseBuf * is enabled. */ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { - int I, J, PidLength, HostNameLength, FilenamePatLength; + int I, J, PidLength, HostNameLength, TmpDirLength, FilenamePatLength; const char *FilenamePat = lprofCurFilename.FilenamePat; if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0]) return 0; if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts || - lprofCurFilename.MergePoolSize || + lprofCurFilename.TmpDir || lprofCurFilename.MergePoolSize || __llvm_profile_is_continuous_mode_enabled())) { if (!ForceUseBuf) return lprofCurFilename.FilenamePat; @@ -765,6 +942,7 @@ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { PidLength = strlen(lprofCurFilename.PidChars); HostNameLength = strlen(lprofCurFilename.Hostname); + TmpDirLength = lprofCurFilename.TmpDir ? strlen(lprofCurFilename.TmpDir) : 0; /* Construct the new filename. */ for (I = 0, J = 0; FilenamePat[I]; ++I) if (FilenamePat[I] == '%') { @@ -774,6 +952,10 @@ static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) { } else if (FilenamePat[I] == 'h') { memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength); J += HostNameLength; + } else if (FilenamePat[I] == 't') { + memcpy(FilenameBuf + J, lprofCurFilename.TmpDir, TmpDirLength); + FilenameBuf[J + TmpDirLength] = DIR_SEPARATOR; + J += TmpDirLength + 1; } else { if (!getMergePoolSize(FilenamePat, &I)) continue; @@ -854,10 +1036,10 @@ const char *__llvm_profile_get_filename(void) { return FilenameBuf; } -/* This method is invoked by the runtime initialization hook - * InstrProfilingRuntime.o if it is linked in. Both user specified +/* This API initializes the file handling, both user specified * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE - * environment variable can override this default value. */ + * environment variable can override this default value. + */ COMPILER_RT_VISIBILITY void __llvm_profile_initialize_file(void) { const char *EnvFilenamePat; @@ -882,6 +1064,16 @@ void __llvm_profile_initialize_file(void) { parseAndSetFilename(SelectedPat, PNS, 0); } +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. + */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize(void) { + __llvm_profile_initialize_file(); + if (!__llvm_profile_is_continuous_mode_enabled()) + __llvm_profile_register_write_file_atexit(); +} + /* This API is directly called by the user application code. It has the * highest precedence compared with LLVM_PROFILE_FILE environment variable * and command line option -fprofile-instr-generate=. @@ -902,10 +1094,14 @@ int __llvm_profile_write_file(void) { int rc, Length; const char *Filename; char *FilenameBuf; - int PDeathSig = 0; + + // Temporarily suspend getting SIGKILL when the parent exits. + int PDeathSig = lprofSuspendSigKill(); if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) { PROF_NOTE("Profile data not written to file: %s.\n", "already written"); + if (PDeathSig == 1) + lprofRestoreSigKill(); return 0; } @@ -916,6 +1112,8 @@ int __llvm_profile_write_file(void) { /* Check the filename. */ if (!Filename) { PROF_ERR("Failed to write file : %s\n", "Filename not set"); + if (PDeathSig == 1) + lprofRestoreSigKill(); return -1; } @@ -925,12 +1123,11 @@ int __llvm_profile_write_file(void) { "expected %d, but get %d\n", INSTR_PROF_RAW_VERSION, (int)GET_VERSION(__llvm_profile_get_version())); + if (PDeathSig == 1) + lprofRestoreSigKill(); return -1; } - // Temporarily suspend getting SIGKILL when the parent exits. - PDeathSig = lprofSuspendSigKill(); - /* Write profile data to the file. */ rc = writeFile(Filename); if (rc) @@ -951,7 +1148,7 @@ int __llvm_profile_dump(void) { "in profile name or change profile name before dumping.\n", "online profile merging is not on"); int rc = __llvm_profile_write_file(); - lprofSetProfileDumped(); + lprofSetProfileDumped(1); return rc; } @@ -963,7 +1160,9 @@ int __llvm_orderfile_write_file(void) { int rc, Length, LengthBeforeAppend, SuffixLength; const char *Filename; char *FilenameBuf; - int PDeathSig = 0; + + // Temporarily suspend getting SIGKILL when the parent exits. + int PDeathSig = lprofSuspendSigKill(); SuffixLength = strlen(OrderFileSuffix); Length = getCurFilenameLength() + SuffixLength; @@ -973,6 +1172,8 @@ int __llvm_orderfile_write_file(void) { /* Check the filename. */ if (!Filename) { PROF_ERR("Failed to write file : %s\n", "Filename not set"); + if (PDeathSig == 1) + lprofRestoreSigKill(); return -1; } @@ -987,12 +1188,11 @@ int __llvm_orderfile_write_file(void) { "expected %d, but get %d\n", INSTR_PROF_RAW_VERSION, (int)GET_VERSION(__llvm_profile_get_version())); + if (PDeathSig == 1) + lprofRestoreSigKill(); return -1; } - // Temporarily suspend getting SIGKILL when the parent exits. - PDeathSig = lprofSuspendSigKill(); - /* Write order data to the file. */ rc = writeOrderFile(Filename); if (rc) @@ -1026,4 +1226,53 @@ int __llvm_profile_register_write_file_atexit(void) { return atexit(writeFileWithoutReturn); } +COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File, + int EnableMerge) { + if (__llvm_profile_is_continuous_mode_enabled()) { + if (!EnableMerge) { + PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported in " + "continuous sync mode when merging is disabled\n", + fileno(File)); + return 1; + } + if (lprofLockFileHandle(File) != 0) { + PROF_WARN("Data may be corrupted during profile merging : %s\n", + "Fail to obtain file lock due to system limit."); + } + uint64_t ProfileFileSize = 0; + if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) { + lprofUnlockFileHandle(File); + return 1; + } + if (ProfileFileSize == 0) { + FreeHook = &free; + setupIOBuffer(); + ProfDataWriter fileWriter; + initFileWriter(&fileWriter, File); + if (lprofWriteData(&fileWriter, 0, 0)) { + lprofUnlockFileHandle(File); + PROF_ERR("Failed to write file \"%d\": %s\n", fileno(File), + strerror(errno)); + return 1; + } + fflush(File); + } else { + /* The merged profile has a non-zero length. Check that it is compatible + * with the data in this process. */ + char *ProfileBuffer; + if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1) { + lprofUnlockFileHandle(File); + return 1; + } + (void)munmap(ProfileBuffer, ProfileFileSize); + } + mmapForContinuousMode(0, File); + lprofUnlockFileHandle(File); + } else { + setProfileFile(File); + setProfileMergeRequested(EnableMerge); + } + return 0; +} + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c new file mode 100644 index 0000000000000..3dd659f905102 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.c @@ -0,0 +1,26 @@ +/*===- InstrProfilingInternal.c - Support library for PGO instrumentation -===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + +#if !defined(__Fuchsia__) + +#include "InstrProfilingInternal.h" + +static unsigned ProfileDumped = 0; + +COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) { + return ProfileDumped; +} + +COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) { + ProfileDumped = Value; +} + +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h index 0cea4876f0ae6..d5bd0e41fb129 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -21,8 +21,10 @@ */ uint64_t __llvm_profile_get_size_for_buffer_internal( const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, - const char *NamesBegin, const char *NamesEnd); + const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin, + const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd, + const VTableProfData *VTableBegin, const VTableProfData *VTableEnd, + const char *VNamesBegin, const char *VNamesEnd); /*! * \brief Write instrumentation data to the given buffer, given explicit @@ -35,8 +37,9 @@ uint64_t __llvm_profile_get_size_for_buffer_internal( */ int __llvm_profile_write_buffer_internal( char *Buffer, const __llvm_profile_data *DataBegin, - const __llvm_profile_data *DataEnd, const uint64_t *CountersBegin, - const uint64_t *CountersEnd, const char *NamesBegin, const char *NamesEnd); + const __llvm_profile_data *DataEnd, const char *CountersBegin, + const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd, + const char *NamesBegin, const char *NamesEnd); /*! * The data structure describing the data to be written by the @@ -145,17 +148,19 @@ typedef struct VPDataReaderType { uint32_t N); } VPDataReaderType; -/* Write profile data to destinitation. If SkipNameDataWrite is set to 1, - the name data is already in destintation, we just skip over it. */ +/* Write profile data to destination. If SkipNameDataWrite is set to 1, + the name data is already in destination, we just skip over it. */ int lprofWriteData(ProfDataWriter *Writer, VPDataReaderType *VPDataReader, int SkipNameDataWrite); int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, - const uint64_t *CountersEnd, + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite); + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite); /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ @@ -181,8 +186,8 @@ uint64_t lprofGetLoadModuleSignature(); * Return non zero value if the profile data has already been * dumped to the file. */ -unsigned lprofProfileDumped(); -void lprofSetProfileDumped(); +unsigned lprofProfileDumped(void); +void lprofSetProfileDumped(unsigned); COMPILER_RT_VISIBILITY extern void (*FreeHook)(void *); COMPILER_RT_VISIBILITY extern uint8_t *DynamicBufferIOBuffer; @@ -193,4 +198,18 @@ COMPILER_RT_VISIBILITY extern ValueProfNode *CurrentVNode; COMPILER_RT_VISIBILITY extern ValueProfNode *EndVNode; extern void (*VPMergeHook)(struct ValueProfData *, __llvm_profile_data *); +/* + * Write binary ids into profiles if writer is given. + * Return -1 if an error occurs, otherwise, return total size of binary ids. + */ +int __llvm_write_binary_ids(ProfDataWriter *Writer); + +/* + * Write binary id length and then its data, because binary id does not + * have a fixed length. + */ +int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, + const uint8_t *BinaryIdData, + uint64_t BinaryIdPadding); + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c index 0fd9b2bcd41f8..c0706b73e1668 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -20,32 +20,42 @@ COMPILER_RT_VISIBILITY void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *); COMPILER_RT_VISIBILITY -uint64_t lprofGetLoadModuleSignature() { +uint64_t lprofGetLoadModuleSignature(void) { /* A very fast way to compute a module signature. */ - uint64_t CounterSize = (uint64_t)(__llvm_profile_end_counters() - - __llvm_profile_begin_counters()); - uint64_t DataSize = __llvm_profile_get_data_size(__llvm_profile_begin_data(), - __llvm_profile_end_data()); + uint64_t Version = __llvm_profile_get_version(); + uint64_t NumCounters = __llvm_profile_get_num_counters( + __llvm_profile_begin_counters(), __llvm_profile_end_counters()); + uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()); uint64_t NamesSize = (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names()); uint64_t NumVnodes = (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes()); const __llvm_profile_data *FirstD = __llvm_profile_begin_data(); - return (NamesSize << 40) + (CounterSize << 30) + (DataSize << 20) + - (NumVnodes << 10) + (DataSize > 0 ? FirstD->NameRef : 0); + return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) + + (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version + + __llvm_profile_get_magic(); } +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif + /* Returns 1 if profile is not structurally compatible. */ COMPILER_RT_VISIBILITY int __llvm_profile_check_compatibility(const char *ProfileData, uint64_t ProfileSize) { - /* Check profile header only for now */ __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; SrcDataStart = - (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header)); - SrcDataEnd = SrcDataStart + Header->DataSize; + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + + Header->BinaryIdsSize); + SrcDataEnd = SrcDataStart + Header->NumData; if (ProfileSize < sizeof(__llvm_profile_header)) return 1; @@ -53,19 +63,26 @@ int __llvm_profile_check_compatibility(const char *ProfileData, /* Check the header first. */ if (Header->Magic != __llvm_profile_get_magic() || Header->Version != __llvm_profile_get_version() || - Header->DataSize != - __llvm_profile_get_data_size(__llvm_profile_begin_data(), - __llvm_profile_end_data()) || - Header->CountersSize != (uint64_t)(__llvm_profile_end_counters() - - __llvm_profile_begin_counters()) || - Header->NamesSize != (uint64_t)(__llvm_profile_end_names() - - __llvm_profile_begin_names()) || + Header->NumData != + __llvm_profile_get_num_data(__llvm_profile_begin_data(), + __llvm_profile_end_data()) || + Header->NumCounters != + __llvm_profile_get_num_counters(__llvm_profile_begin_counters(), + __llvm_profile_end_counters()) || + Header->NumBitmapBytes != + __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(), + __llvm_profile_end_bitmap()) || + Header->NamesSize != + __llvm_profile_get_name_size(__llvm_profile_begin_names(), + __llvm_profile_end_names()) || Header->ValueKindLast != IPVK_Last) return 1; - if (ProfileSize < sizeof(__llvm_profile_header) + - Header->DataSize * sizeof(__llvm_profile_data) + - Header->NamesSize + Header->CountersSize) + if (ProfileSize < + sizeof(__llvm_profile_header) + Header->BinaryIdsSize + + Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize + + Header->NumCounters * __llvm_profile_counter_entry_size() + + Header->NumBitmapBytes) return 1; for (SrcData = SrcDataStart, @@ -73,7 +90,8 @@ int __llvm_profile_check_compatibility(const char *ProfileData, SrcData < SrcDataEnd; ++SrcData, ++DstData) { if (SrcData->NameRef != DstData->NameRef || SrcData->FuncHash != DstData->FuncHash || - SrcData->NumCounters != DstData->NumCounters) + SrcData->NumCounters != DstData->NumCounters || + SrcData->NumBitmapBytes != DstData->NumBitmapBytes) return 1; } @@ -81,52 +99,163 @@ int __llvm_profile_check_compatibility(const char *ProfileData, return 0; } +static uintptr_t signextIfWin64(void *V) { +#ifdef _WIN64 + return (uintptr_t)(int32_t)(uintptr_t)V; +#else + return (uintptr_t)V; +#endif +} + +// Skip names section, vtable profile data section and vtable names section +// for runtime profile merge. To merge runtime addresses from multiple +// profiles collected from the same instrumented binary, the binary should be +// loaded at fixed base address (e.g., build with -no-pie, or run with ASLR +// disabled). In this set-up these three sections remain unchanged. +static uint64_t +getDistanceFromCounterToValueProf(const __llvm_profile_header *const Header) { + const uint64_t VTableSectionSize = + Header->NumVTables * sizeof(VTableProfData); + const uint64_t PaddingBytesAfterVTableSection = + __llvm_profile_get_num_padding_bytes(VTableSectionSize); + const uint64_t VNamesSize = Header->VNamesSize; + const uint64_t PaddingBytesAfterVNamesSize = + __llvm_profile_get_num_padding_bytes(VNamesSize); + return Header->NamesSize + + __llvm_profile_get_num_padding_bytes(Header->NamesSize) + + VTableSectionSize + PaddingBytesAfterVTableSection + VNamesSize + + PaddingBytesAfterVNamesSize; +} + COMPILER_RT_VISIBILITY -void __llvm_profile_merge_from_buffer(const char *ProfileData, - uint64_t ProfileSize) { +int __llvm_profile_merge_from_buffer(const char *ProfileData, + uint64_t ProfileSize) { + if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) { + PROF_ERR("%s\n", + "Temporal profiles do not support profile merging at runtime. " + "Instead, merge raw profiles using the llvm-profdata tool."); + return 1; + } + __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; - uint64_t *SrcCountersStart; + char *SrcCountersStart, *DstCounter; + const char *SrcCountersEnd, *SrcCounter; + const char *SrcBitmapStart; const char *SrcNameStart; - ValueProfData *SrcValueProfDataStart, *SrcValueProfData; + const char *SrcValueProfDataStart, *SrcValueProfData; + uintptr_t CountersDelta = Header->CountersDelta; + uintptr_t BitmapDelta = Header->BitmapDelta; SrcDataStart = - (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header)); - SrcDataEnd = SrcDataStart + Header->DataSize; - SrcCountersStart = (uint64_t *)SrcDataEnd; - SrcNameStart = (const char *)(SrcCountersStart + Header->CountersSize); + (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + + Header->BinaryIdsSize); + SrcDataEnd = SrcDataStart + Header->NumData; + SrcCountersStart = (char *)SrcDataEnd; + SrcCountersEnd = SrcCountersStart + + Header->NumCounters * __llvm_profile_counter_entry_size(); + SrcBitmapStart = SrcCountersEnd; + SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; SrcValueProfDataStart = - (ValueProfData *)(SrcNameStart + Header->NamesSize + - __llvm_profile_get_num_padding_bytes( - Header->NamesSize)); + SrcNameStart + getDistanceFromCounterToValueProf(Header); + if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) + return 1; + + // Merge counters by iterating the entire counter section when data section is + // empty due to correlation. + if (Header->NumData == 0) { + for (SrcCounter = SrcCountersStart, + DstCounter = __llvm_profile_begin_counters(); + SrcCounter < SrcCountersEnd;) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { + *DstCounter &= *SrcCounter; + } else { + *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter; + } + SrcCounter += __llvm_profile_counter_entry_size(); + DstCounter += __llvm_profile_counter_entry_size(); + } + return 0; + } for (SrcData = SrcDataStart, DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), SrcValueProfData = SrcValueProfDataStart; SrcData < SrcDataEnd; ++SrcData, ++DstData) { - uint64_t *SrcCounters; - uint64_t *DstCounters = (uint64_t *)DstData->CounterPtr; - unsigned I, NC, NVK = 0; - - NC = SrcData->NumCounters; - SrcCounters = SrcCountersStart + - ((size_t)SrcData->CounterPtr - Header->CountersDelta) / - sizeof(uint64_t); - for (I = 0; I < NC; I++) - DstCounters[I] += SrcCounters[I]; - - /* Now merge value profile data. */ + // For the in-memory destination, CounterPtr is the distance from the start + // address of the data to the start address of the counter. On WIN64, + // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign + // extend CounterPtr to get the original value. + char *DstCounters = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); + char *DstBitmap = + (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); + unsigned NVK = 0; + + // SrcData is a serialized representation of the memory image. We need to + // compute the in-buffer counter offset from the in-memory address distance. + // The initial CountersDelta is the in-memory address difference + // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr - + // CountersDelta computes the offset into the in-buffer counter section. + // + // On WIN64, CountersDelta is truncated as well, so no need for signext. + char *SrcCounters = + SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta); + // CountersDelta needs to be decreased as we advance to the next data + // record. + CountersDelta -= sizeof(*SrcData); + unsigned NC = SrcData->NumCounters; + if (NC == 0) + return 1; + if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart || + (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart) + return 1; + for (unsigned I = 0; I < NC; I++) { + if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { + // A value of zero signifies the function is covered. + DstCounters[I] &= SrcCounters[I]; + } else { + ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I]; + } + } + + const char *SrcBitmap = + SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); + // BitmapDelta also needs to be decreased as we advance to the next data + // record. + BitmapDelta -= sizeof(*SrcData); + unsigned NB = SrcData->NumBitmapBytes; + // NumBitmapBytes may legitimately be 0. Just keep going. + if (NB != 0) { + if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart) + return 1; + // Merge Src and Dst Bitmap bytes by simply ORing them together. + for (unsigned I = 0; I < NB; I++) + DstBitmap[I] |= SrcBitmap[I]; + } + + /* Now merge value profile data. */ if (!VPMergeHook) continue; - for (I = 0; I <= IPVK_Last; I++) + for (unsigned I = 0; I <= IPVK_Last; I++) NVK += (SrcData->NumValueSites[I] != 0); if (!NVK) continue; - VPMergeHook(SrcValueProfData, DstData); - SrcValueProfData = (ValueProfData *)((char *)SrcValueProfData + - SrcValueProfData->TotalSize); + if (SrcValueProfData >= ProfileData + ProfileSize) + return 1; + VPMergeHook((ValueProfData *)SrcValueProfData, DstData); + SrcValueProfData = + SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize; } + + return 0; } + +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c index 2d67a55b985cf..407272806ba3c 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingNameVar.c @@ -14,4 +14,4 @@ * user has not specified one. Set this up by moving the runtime's copy of this * symbol to an object file within the archive. */ -COMPILER_RT_WEAK char INSTR_PROF_PROFILE_NAME_VAR[1] = {0}; +COMPILER_RT_WEAK COMPILER_RT_VISIBILITY char INSTR_PROF_PROFILE_NAME_VAR[1] = {0}; diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c new file mode 100644 index 0000000000000..b9d51b698b414 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c @@ -0,0 +1,221 @@ +/*===- InstrProfilingPlatformAIX.c - Profile data AIX platform ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#if defined(_AIX) + +#ifdef __64BIT__ +#define __XCOFF64__ +#endif +#include +#include +#include +#include +#include + +#include "InstrProfiling.h" +#include "InstrProfilingInternal.h" + +#define BIN_ID_PREFIX "xcoff_binary_id:" + +// If found, write the build-id into the Result buffer. +static size_t FindBinaryId(char *Result, size_t Size) { + unsigned long EntryAddr = (unsigned long)__builtin_return_address(0); + + // Use loadquery to get information about loaded modules; loadquery writes + // its result into a buffer of unknown size. + char Buf[1024]; + size_t BufSize = sizeof(Buf); + char *BufPtr = Buf; + int RC = -1; + + errno = 0; + RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize); + if (RC == -1 && errno == ENOMEM) { + BufSize = 64000; // should be plenty for any program. + BufPtr = malloc(BufSize); + if (BufPtr != 0) + RC = loadquery(L_GETXINFO | L_IGNOREUNLOAD, BufPtr, (unsigned int)BufSize); + } + + if (RC == -1) + goto done; + + // Locate the ld_xinfo corresponding to this module. + struct ld_xinfo *CurInfo = (struct ld_xinfo *)BufPtr; + while (1) { + unsigned long CurTextStart = (uint64_t)CurInfo->ldinfo_textorg; + unsigned long CurTextEnd = CurTextStart + CurInfo->ldinfo_textsize; + if (CurTextStart <= EntryAddr && EntryAddr < CurTextEnd) { + // Found my slot. Now search for the build-id. + char *p = (char *)CurInfo->ldinfo_textorg; + + FILHDR *f = (FILHDR *)p; + AOUTHDR *a = (AOUTHDR *)(p + FILHSZ); + SCNHDR *s = + (SCNHDR *)(p + FILHSZ + f->f_opthdr + SCNHSZ * (a->o_snloader - 1)); + LDHDR *ldhdr = (LDHDR *)(p + s->s_scnptr); + // This is the loader string table + char *lstr = (char *)ldhdr + ldhdr->l_stoff; + + // If the build-id exists, it's the first entry. + // Each entry is comprised of a 2-byte size component, followed by the + // data. + size_t len = *(short *)lstr; + char *str = (char *)(lstr + 2); + size_t PrefixLen = sizeof(BIN_ID_PREFIX) - 1; + if (len > PrefixLen && (len - PrefixLen) <= Size && + strncmp(str, BIN_ID_PREFIX, PrefixLen) == 0) { + memcpy(Result, str + PrefixLen, len - PrefixLen); + RC = len - PrefixLen; + goto done; + } + break; + } + if (CurInfo->ldinfo_next == 0u) + break; + CurInfo = (struct ld_xinfo *)((char *)CurInfo + CurInfo->ldinfo_next); + } +done: + if (BufSize != sizeof(Buf) && BufPtr != 0) + free(BufPtr); + return RC; +} + +static int StrToHexError = 0; +static uint8_t StrToHex(char c) { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'a' && c <= 'f') + return c - 'a' + 0xa; + if (c >= 'A' && c <= 'F') + return c - 'A' + 0xa; + StrToHexError = 1; + return 0; +} + +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + // 200 bytes should be enough for the build-id hex string. + static char Buf[200]; + // Profile reading tools expect this to be 8-bytes long. + static int64_t BinaryIdLen = 0; + static uint8_t *BinaryIdData = 0; + + // -1 means we already checked for a BinaryId and didn't find one. + if (BinaryIdLen == -1) + return 0; + + // Are we being called for the first time? + if (BinaryIdLen == 0) { + if (getenv("LLVM_PROFILE_NO_BUILD_ID")) + goto fail; + + int BuildIdLen = FindBinaryId(Buf, sizeof(Buf)); + if (BuildIdLen <= 0) + goto fail; + + if (Buf[BuildIdLen - 1] == '\0') + BuildIdLen--; + + // assume even number of digits/chars, so 0xabc must be 0x0abc + if ((BuildIdLen % 2) != 0 || BuildIdLen == 0) + goto fail; + + // The numeric ID is represented as an ascii string in the loader section, + // so convert it to raw binary. + BinaryIdLen = BuildIdLen / 2; + BinaryIdData = (uint8_t *)Buf; + + // Skip "0x" prefix if it exists. + if (Buf[0] == '0' && Buf[1] == 'x') { + BinaryIdLen -= 1; + BinaryIdData += 2; + } + + StrToHexError = 0; + for (int i = 0; i < BinaryIdLen; i++) + BinaryIdData[i] = (StrToHex(BinaryIdData[2 * i]) << 4) + + StrToHex(BinaryIdData[2 * i + 1]); + + if (StrToHexError) + goto fail; + + if (getenv("LLVM_PROFILE_VERBOSE")) { + char *StrBuf = (char *)COMPILER_RT_ALLOCA(2 * BinaryIdLen + 1); + for (int i = 0; i < (int)BinaryIdLen; i++) + sprintf(&StrBuf[2 * i], "%02x", BinaryIdData[i]); + PROF_NOTE("Writing binary id: %s\n", StrBuf); + } + } + + uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen); + if (Writer && lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData, + BinaryIdPadding) == -1) + return -1; // Return -1 rather goto fail to match the NT_GNU_BUILD_ID path. + + return sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding; + +fail: + if (getenv("LLVM_PROFILE_VERBOSE")) + fprintf(stderr, "no or invalid binary id: %.*s\n", (int)sizeof(Buf), Buf); + BinaryIdLen = -1; + return 0; +} + +// Empty stubs to allow linking object files using the registration-based scheme +COMPILER_RT_VISIBILITY +void __llvm_profile_register_function(void *Data_) {} + +COMPILER_RT_VISIBILITY +void __llvm_profile_register_names_function(void *NamesStart, + uint64_t NamesSize) {} + +// The __start_SECNAME and __stop_SECNAME symbols (for SECNAME \in +// {"__llvm_prf_cnts", "__llvm_prf_data", "__llvm_prf_name", "__llvm_prf_vnds", +// "__llvm_prf_vns", "__llvm_prf_vtab"}) +// are always live when linking on AIX, regardless if the .o's being linked +// reference symbols from the profile library (for example when no files were +// compiled with -fprofile-generate). That's because these symbols are kept +// alive through references in constructor functions that are always live in the +// default linking model on AIX (-bcdtors:all). The __start_SECNAME and +// __stop_SECNAME symbols are only resolved by the linker when the SECNAME +// section exists. So for the scenario where the user objects have no such +// section (i.e. when they are compiled with -fno-profile-generate), we always +// define these zero length variables in each of the above 4 sections. +static int dummy_cnts[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_CNTS_SECT_NAME); +static int dummy_bits[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_BITS_SECT_NAME); +static int dummy_data[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_DATA_SECT_NAME); +static const int dummy_name[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_NAME_SECT_NAME); +static int dummy_vnds[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); +static int dummy_orderfile[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_ORDERFILE_SECT_NAME); +static int dummy_vname[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VNAME_SECT_NAME); +static int dummy_vtab[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_VTAB_SECT_NAME); + +// To avoid GC'ing of the dummy variables by the linker, reference them in an +// array and reference the array in the runtime registration code +// (InstrProfilingRuntime.cpp) +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#endif +COMPILER_RT_VISIBILITY +void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, + (void *)&dummy_data, (void *)&dummy_name, + (void *)&dummy_vnds, (void *)&dummy_orderfile, + (void *)&dummy_vname, (void *)&dummy_vtab}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif +#endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c index 23bdb7f37179d..6adc7f328cbf7 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformDarwin.c @@ -6,7 +6,11 @@ |* \*===----------------------------------------------------------------------===*/ +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" #if defined(__APPLE__) /* Use linker magic to find the bounds of the Data section. */ @@ -22,11 +26,26 @@ extern char COMPILER_RT_VISIBILITY extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY -extern uint64_t +extern char CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY -extern uint64_t - CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +extern char CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + BitmapStart __asm("section$start$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char BitmapEnd __asm("section$end$__DATA$" INSTR_PROF_BITS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfStart __asm("section$start$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern VTableProfData + VTableProfEnd __asm("section$end$__DATA$" INSTR_PROF_VTAB_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char + VNameStart __asm("section$start$__DATA$" INSTR_PROF_VNAME_SECT_NAME); +COMPILER_RT_VISIBILITY +extern char VNameEnd __asm("section$end$__DATA$" INSTR_PROF_VNAME_SECT_NAME); COMPILER_RT_VISIBILITY extern uint32_t OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); @@ -49,9 +68,25 @@ const char *__llvm_profile_begin_names(void) { return &NamesStart; } COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &NamesEnd; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } +char *__llvm_profile_begin_counters(void) { return &CountersStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_counters(void) { return &CountersEnd; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } +COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_begin_vtables(void) { + return &VTableProfStart; +} +COMPILER_RT_VISIBILITY +const VTableProfData *__llvm_profile_end_vtables(void) { + return &VTableProfEnd; +} +COMPILER_RT_VISIBILITY +const char *__llvm_profile_begin_vtabnames(void) { return &VNameStart; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return &VNameEnd; } COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } @@ -64,4 +99,9 @@ ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &VNodesStart; COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &VNodesEnd; + +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c index 23b7efbe672b9..fdcb82e4d72ba 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -34,16 +34,13 @@ #include "InstrProfilingInternal.h" #include "InstrProfilingUtil.h" -/* VMO that contains the coverage data shared across all modules. This symbol - * has default visibility and is exported in each module (executable or DSO) - * that statically links in the profiling runtime. - */ -zx_handle_t __llvm_profile_vmo; -/* Current offset within the VMO where data should be written next. This symbol - * has default visibility and is exported in each module (executable or DSO) - * that statically links in the profiling runtime. - */ -uint64_t __llvm_profile_offset; +/* This variable is an external reference to symbol defined by the compiler. */ +COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; + +COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) { + return 1; +} +COMPILER_RT_VISIBILITY void lprofSetProfileDumped(unsigned Value) {} static const char ProfileSinkName[] = "llvm-profile"; @@ -55,68 +52,27 @@ static inline void lprofWrite(const char *fmt, ...) { int ret = vsnprintf(s, sizeof(s), fmt, ap); va_end(ap); - __sanitizer_log_write(s, ret + 1); + __sanitizer_log_write(s, ret); } -static void createVMO() { - /* Don't create VMO if it has been alread created. */ - if (__llvm_profile_vmo != ZX_HANDLE_INVALID) - return; - - /* Get information about the current process. */ - zx_info_handle_basic_t Info; - zx_status_t Status = - _zx_object_get_info(_zx_process_self(), ZX_INFO_HANDLE_BASIC, &Info, - sizeof(Info), NULL, NULL); - if (Status != ZX_OK) { - lprofWrite("LLVM Profile: cannot get info about current process: %s\n", - _zx_status_get_string(Status)); - return; - } - - /* Create VMO to hold the profile data. */ - Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &__llvm_profile_vmo); - if (Status != ZX_OK) { - lprofWrite("LLVM Profile: cannot create VMO: %s\n", - _zx_status_get_string(Status)); - return; - } - - /* Give the VMO a name including our process KOID so it's easy to spot. */ - char VmoName[ZX_MAX_NAME_LEN]; - snprintf(VmoName, sizeof(VmoName), "%s.%" PRIu64, ProfileSinkName, Info.koid); - _zx_object_set_property(__llvm_profile_vmo, ZX_PROP_NAME, VmoName, - strlen(VmoName)); - - /* Duplicate the handle since __sanitizer_publish_data consumes it. */ - zx_handle_t Handle; - Status = - _zx_handle_duplicate(__llvm_profile_vmo, ZX_RIGHT_SAME_RIGHTS, &Handle); - if (Status != ZX_OK) { - lprofWrite("LLVM Profile: cannot duplicate VMO handle: %s\n", - _zx_status_get_string(Status)); - _zx_handle_close(__llvm_profile_vmo); - __llvm_profile_vmo = ZX_HANDLE_INVALID; - return; - } - - /* Publish the VMO which contains profile data to the system. */ - __sanitizer_publish_data(ProfileSinkName, Handle); - - /* Use the dumpfile symbolizer markup element to write the name of VMO. */ - lprofWrite("LLVM Profile: {{{dumpfile:%s:%s}}}\n", ProfileSinkName, VmoName); -} +struct lprofVMOWriterCtx { + /* VMO that contains the profile data for this module. */ + zx_handle_t Vmo; + /* Current offset within the VMO where data should be written next. */ + uint64_t Offset; +}; static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, uint32_t NumIOVecs) { + struct lprofVMOWriterCtx *Ctx = (struct lprofVMOWriterCtx *)This->WriterCtx; + /* Compute the total length of data to be written. */ size_t Length = 0; for (uint32_t I = 0; I < NumIOVecs; I++) Length += IOVecs[I].ElmSize * IOVecs[I].NumElm; /* Resize the VMO to ensure there's sufficient space for the data. */ - zx_status_t Status = - _zx_vmo_set_size(__llvm_profile_vmo, __llvm_profile_offset + Length); + zx_status_t Status = _zx_vmo_set_size(Ctx->Vmo, Ctx->Offset + Length); if (Status != ZX_OK) return -1; @@ -124,74 +80,111 @@ static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, for (uint32_t I = 0; I < NumIOVecs; I++) { size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm; if (IOVecs[I].Data) { - Status = _zx_vmo_write(__llvm_profile_vmo, IOVecs[I].Data, - __llvm_profile_offset, Length); + Status = _zx_vmo_write(Ctx->Vmo, IOVecs[I].Data, Ctx->Offset, Length); if (Status != ZX_OK) return -1; } else if (IOVecs[I].UseZeroPadding) { /* Resizing the VMO should zero fill. */ } - __llvm_profile_offset += Length; + Ctx->Offset += Length; } + /* Record the profile size as a property of the VMO. */ + _zx_object_set_property(Ctx->Vmo, ZX_PROP_VMO_CONTENT_SIZE, &Ctx->Offset, + sizeof(Ctx->Offset)); + return 0; } -static void initVMOWriter(ProfDataWriter *This) { +static void initVMOWriter(ProfDataWriter *This, struct lprofVMOWriterCtx *Ctx) { This->Write = lprofVMOWriter; - This->WriterCtx = NULL; + This->WriterCtx = Ctx; } -static int dump(void) { - if (lprofProfileDumped()) { - lprofWrite("LLVM Profile: data not published: already written.\n"); - return 0; - } - +/* This method is invoked by the runtime initialization hook + * InstrProfilingRuntime.o if it is linked in. */ +COMPILER_RT_VISIBILITY +void __llvm_profile_initialize(void) { /* Check if there is llvm/runtime version mismatch. */ if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { lprofWrite("LLVM Profile: runtime and instrumentation version mismatch: " "expected %d, but got %d\n", INSTR_PROF_RAW_VERSION, (int)GET_VERSION(__llvm_profile_get_version())); - return -1; + return; } - /* Write the profile data into the mapped region. */ - ProfDataWriter VMOWriter; - initVMOWriter(&VMOWriter); - if (lprofWriteData(&VMOWriter, lprofGetVPDataReader(), 0) != 0) - return -1; + const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); + const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t CountersOffset = + sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + + /* Don't publish a VMO if there are no counters. */ + if (!CountersSize) + return; - return 0; -} + zx_status_t Status; -COMPILER_RT_VISIBILITY -int __llvm_profile_dump(void) { - int rc = dump(); - lprofSetProfileDumped(); - return rc; -} + /* Create a VMO to hold the profile data. */ + zx_handle_t Vmo = ZX_HANDLE_INVALID; + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &Vmo); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: cannot create VMO: %s\n", + _zx_status_get_string(Status)); + return; + } -static void dumpWithoutReturn(void) { dump(); } + /* Give the VMO a name that includes the module signature. */ + char VmoName[ZX_MAX_NAME_LEN]; + snprintf(VmoName, sizeof(VmoName), "%" PRIu64 ".profraw", + lprofGetLoadModuleSignature()); + _zx_object_set_property(Vmo, ZX_PROP_NAME, VmoName, strlen(VmoName)); -/* This method is invoked by the runtime initialization hook - * InstrProfilingRuntime.o if it is linked in. - */ -COMPILER_RT_VISIBILITY -void __llvm_profile_initialize_file(void) { createVMO(); } + /* Write the profile data into the mapped region. */ + ProfDataWriter VMOWriter; + struct lprofVMOWriterCtx Ctx = {.Vmo = Vmo, .Offset = 0}; + initVMOWriter(&VMOWriter, &Ctx); + if (lprofWriteData(&VMOWriter, 0, 0) != 0) { + lprofWrite("LLVM Profile: failed to write data\n"); + _zx_handle_close(Vmo); + return; + } -COMPILER_RT_VISIBILITY -int __llvm_profile_register_write_file_atexit(void) { - static bool HasBeenRegistered = false; + uint64_t Len = 0; + Status = _zx_vmo_get_size(Vmo, &Len); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: failed to get the VMO size: %s\n", + _zx_status_get_string(Status)); + _zx_handle_close(Vmo); + return; + } + + uintptr_t Mapping; + Status = + _zx_vmar_map(_zx_vmar_root_self(), ZX_VM_PERM_READ | ZX_VM_PERM_WRITE, 0, + Vmo, 0, Len, &Mapping); + if (Status != ZX_OK) { + lprofWrite("LLVM Profile: failed to map the VMO: %s\n", + _zx_status_get_string(Status)); + _zx_handle_close(Vmo); + return; + } - if (HasBeenRegistered) - return 0; + /* Publish the VMO which contains profile data to the system. Note that this + * also consumes the VMO handle. */ + __sanitizer_publish_data(ProfileSinkName, Vmo); - lprofSetupValueProfiler(); + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_COUNTER_BIAS_VAR = + (intptr_t)Mapping - (uintptr_t)CountersBegin + CountersOffset; - HasBeenRegistered = true; - return atexit(dumpWithoutReturn); + /* Return the memory allocated for counters to OS. */ + lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); } #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index becfe1fd9f5ad..b766436497b74 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -7,18 +7,31 @@ \*===----------------------------------------------------------------------===*/ #if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ - (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \ + defined(_AIX) +#if !defined(_AIX) +#include +#include +#endif #include +#include #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" #define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) #define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) #define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) #define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_VNAME_START INSTR_PROF_SECT_START(INSTR_PROF_VNAME_COMMON) +#define PROF_VNAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNAME_COMMON) #define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) #define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_VTABLE_START INSTR_PROF_SECT_START(INSTR_PROF_VTAB_COMMON) +#define PROF_VTABLE_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VTAB_COMMON) +#define PROF_BITS_START INSTR_PROF_SECT_START(INSTR_PROF_BITS_COMMON) +#define PROF_BITS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_BITS_COMMON) #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) @@ -26,25 +39,23 @@ /* Declare section start and stop symbols for various sections * generated by compiler instrumentation. */ -extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; -extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; -extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; -extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; -extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY; -extern char PROF_NAME_START COMPILER_RT_VISIBILITY; -extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; -extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY; -extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY; - -/* Add dummy data to ensure the section is always created. */ -__llvm_profile_data - __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME); -uint64_t - __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME); -uint32_t - __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME); -char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME); -ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME); +extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; +extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; +extern char PROF_CNTS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_CNTS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern VTableProfData PROF_VTABLE_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_VNAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_BITS_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; COMPILER_RT_VISIBILITY const __llvm_profile_data * __llvm_profile_begin_data(void) { @@ -60,12 +71,31 @@ COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return &PROF_NAME_STOP; } -COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { +COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_vtabnames(void) { + return &PROF_VNAME_START; +} +COMPILER_RT_VISIBILITY const char *__llvm_profile_end_vtabnames(void) { + return &PROF_VNAME_STOP; +} +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return &PROF_VTABLE_START; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return &PROF_VTABLE_STOP; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_counters(void) { return &PROF_CNTS_START; } -COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { +COMPILER_RT_VISIBILITY char *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY char *__llvm_profile_begin_bitmap(void) { + return &PROF_BITS_START; +} +COMPILER_RT_VISIBILITY char *__llvm_profile_end_bitmap(void) { + return &PROF_BITS_STOP; +} COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return &PROF_ORDERFILE_START; } @@ -80,4 +110,127 @@ COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) { COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START; COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP; +#ifdef NT_GNU_BUILD_ID +static size_t RoundUp(size_t size, size_t align) { + return (size + align - 1) & ~(align - 1); +} + +/* + * Look for the note that has the name "GNU\0" and type NT_GNU_BUILD_ID + * that contains build id. If build id exists, write binary id. + * + * Each note in notes section starts with a struct which includes + * n_namesz, n_descsz, and n_type members. It is followed by the name + * (whose length is defined in n_namesz) and then by the descriptor + * (whose length is defined in n_descsz). + * + * Note sections like .note.ABI-tag and .note.gnu.build-id are aligned + * to 4 bytes, so round n_namesz and n_descsz to the nearest 4 bytes. + */ +static int WriteBinaryIdForNote(ProfDataWriter *Writer, + const ElfW(Nhdr) * Note) { + int BinaryIdSize = 0; + const char *NoteName = (const char *)Note + sizeof(ElfW(Nhdr)); + if (Note->n_type == NT_GNU_BUILD_ID && Note->n_namesz == 4 && + memcmp(NoteName, "GNU\0", 4) == 0) { + uint64_t BinaryIdLen = Note->n_descsz; + const uint8_t *BinaryIdData = + (const uint8_t *)(NoteName + RoundUp(Note->n_namesz, 4)); + uint8_t BinaryIdPadding = __llvm_profile_get_num_padding_bytes(BinaryIdLen); + if (Writer != NULL && + lprofWriteOneBinaryId(Writer, BinaryIdLen, BinaryIdData, + BinaryIdPadding) == -1) + return -1; + + BinaryIdSize = sizeof(BinaryIdLen) + BinaryIdLen + BinaryIdPadding; + } + + return BinaryIdSize; +} + +/* + * Helper function that iterates through notes section and find build ids. + * If writer is given, write binary ids into profiles. + * If an error happens while writing, return -1. + */ +static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note, + const ElfW(Nhdr) * NotesEnd) { + int BinaryIdsSize = 0; + while (Note < NotesEnd) { + int OneBinaryIdSize = WriteBinaryIdForNote(Writer, Note); + if (OneBinaryIdSize == -1) + return -1; + BinaryIdsSize += OneBinaryIdSize; + + /* Calculate the offset of the next note in notes section. */ + size_t NoteOffset = sizeof(ElfW(Nhdr)) + RoundUp(Note->n_namesz, 4) + + RoundUp(Note->n_descsz, 4); + Note = (const ElfW(Nhdr) *)((const char *)(Note) + NoteOffset); + } + + return BinaryIdsSize; +} + +/* + * Write binary ids into profiles if writer is given. + * Return the total size of binary ids. + * If an error happens while writing, return -1. + */ +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + extern const ElfW(Ehdr) __ehdr_start __attribute__((visibility("hidden"))); + const ElfW(Ehdr) *ElfHeader = &__ehdr_start; + const ElfW(Phdr) *ProgramHeader = + (const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff); + + int TotalBinaryIdsSize = 0; + uint32_t I; + /* Iterate through entries in the program header. */ + for (I = 0; I < ElfHeader->e_phnum; I++) { + /* Look for the notes segment in program header entries. */ + if (ProgramHeader[I].p_type != PT_NOTE) + continue; + + /* There can be multiple notes segment, and examine each of them. */ + const ElfW(Nhdr) * Note; + const ElfW(Nhdr) * NotesEnd; + /* + * When examining notes in file, use p_offset, which is the offset within + * the elf file, to find the start of notes. + */ + if (ProgramHeader[I].p_memsz == 0 || + ProgramHeader[I].p_memsz == ProgramHeader[I].p_filesz) { + Note = (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + + ProgramHeader[I].p_offset); + NotesEnd = (const ElfW(Nhdr) *)((const char *)(Note) + + ProgramHeader[I].p_filesz); + } else { + /* + * When examining notes in memory, use p_vaddr, which is the address of + * section after loaded to memory, to find the start of notes. + */ + Note = + (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_vaddr); + NotesEnd = + (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz); + } + + int BinaryIdsSize = WriteBinaryIds(Writer, Note, NotesEnd); + if (TotalBinaryIdsSize == -1) + return -1; + + TotalBinaryIdsSize += BinaryIdsSize; + } + + return TotalBinaryIdsSize; +} +#elif !defined(_AIX) /* !NT_GNU_BUILD_ID */ +/* + * Fallback implementation for targets that don't support the GNU + * extensions NT_GNU_BUILD_ID and __ehdr_start. + */ +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} +#endif + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index 56c5d8378c29d..aa79a5641ceca 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -7,20 +7,25 @@ \*===----------------------------------------------------------------------===*/ #if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \ - !defined(_WIN32) + !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) && \ + !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) #include #include #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" static const __llvm_profile_data *DataFirst = NULL; static const __llvm_profile_data *DataLast = NULL; +static const VTableProfData *VTableProfDataFirst = NULL; +static const VTableProfData *VTableProfDataLast = NULL; static const char *NamesFirst = NULL; static const char *NamesLast = NULL; -static uint64_t *CountersFirst = NULL; -static uint64_t *CountersLast = NULL; +static const char *VNamesFirst = NULL; +static const char *VNamesLast = NULL; +static char *CountersFirst = NULL; +static char *CountersLast = NULL; static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { @@ -45,17 +50,21 @@ void __llvm_profile_register_function(void *Data_) { if (!DataFirst) { DataFirst = Data; DataLast = Data + 1; - CountersFirst = Data->CounterPtr; - CountersLast = (uint64_t *)Data->CounterPtr + Data->NumCounters; + CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr); + CountersLast = + CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size(); return; } DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); - CountersFirst = (uint64_t *)getMinAddr(CountersFirst, Data->CounterPtr); + CountersFirst = (char *)getMinAddr( + CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr)); DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); - CountersLast = (uint64_t *)getMaxAddr( - CountersLast, (uint64_t *)Data->CounterPtr + Data->NumCounters); + CountersLast = (char *)getMaxAddr( + CountersLast, + (char *)((uintptr_t)Data_ + Data->CounterPtr) + + Data->NumCounters * __llvm_profile_counter_entry_size()); } COMPILER_RT_VISIBILITY @@ -75,14 +84,29 @@ COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { return DataFirst; } COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_end_data(void) { return DataLast; } +COMPILER_RT_VISIBILITY const VTableProfData * +__llvm_profile_begin_vtables(void) { + return VTableProfDataFirst; +} +COMPILER_RT_VISIBILITY const VTableProfData *__llvm_profile_end_vtables(void) { + return VTableProfDataLast; +} COMPILER_RT_VISIBILITY const char *__llvm_profile_begin_names(void) { return NamesFirst; } COMPILER_RT_VISIBILITY const char *__llvm_profile_end_names(void) { return NamesLast; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } +const char *__llvm_profile_begin_vtabnames(void) { return VNamesFirst; } +COMPILER_RT_VISIBILITY +const char *__llvm_profile_end_vtabnames(void) { return VNamesLast; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_begin_counters(void) { return CountersFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_counters(void) { return CountersLast; } COMPILER_RT_VISIBILITY -uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +char *__llvm_profile_begin_bitmap(void) { return BitmapFirst; } +COMPILER_RT_VISIBILITY +char *__llvm_profile_end_bitmap(void) { return BitmapLast; } /* TODO: correctly set up OrderFileFirst. */ COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } @@ -97,4 +121,8 @@ ValueProfNode *__llvm_profile_end_vnodes(void) { return 0; } COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = 0; COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = 0; +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + return 0; +} + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c index 81b708bb2a38f..741b01faada4e 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformWindows.c @@ -6,18 +6,23 @@ |* \*===----------------------------------------------------------------------===*/ +#include + #include "InstrProfiling.h" +#include "InstrProfilingInternal.h" #if defined(_WIN32) #if defined(_MSC_VER) /* Merge read-write sections into .data. */ -#pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfb=.data") #pragma comment(linker, "/MERGE:.lprfd=.data") #pragma comment(linker, "/MERGE:.lprfv=.data") #pragma comment(linker, "/MERGE:.lprfnd=.data") /* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find * after the fact. + * Do *NOT* merge .lprfc .rdata. When binary profile correlation is enabled, + * llvm-cov must be able to find after the fact. */ /* Allocate read-only section bounds. */ @@ -29,6 +34,8 @@ #pragma section(".lprfd$Z", read, write) #pragma section(".lprfc$A", read, write) #pragma section(".lprfc$Z", read, write) +#pragma section(".lprfb$A", read, write) +#pragma section(".lprfb$Z", read, write) #pragma section(".lorderfile$A", read, write) #pragma section(".lprfnd$A", read, write) #pragma section(".lprfnd$Z", read, write) @@ -40,8 +47,10 @@ __llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; -uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; -uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfc$A") CountersStart; +char COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +char COMPILER_RT_SECTION(".lprfb$A") BitmapStart; +char COMPILER_RT_SECTION(".lprfb$Z") BitmapEnd; uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; @@ -52,11 +61,30 @@ const __llvm_profile_data *__llvm_profile_begin_data(void) { } const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } +// Type profiling isn't implemented under MSVC ABI, so return NULL (rather than +// implementing linker magic on Windows) to make it more explicit. To elaborate, +// the current type profiling implementation maps a profiled vtable address to a +// vtable variable through vtables mangled name. Under MSVC ABI, the variable +// name for vtables might not be the mangled name (see +// MicrosoftCXXABI::getAddrOfVTable in MicrosoftCXXABI.cpp for more details on +// how a vtable name is computed). Note the mangled name is still in the vtable +// IR (just not variable name) for mapping purpose, but more implementation work +// is required. +const VTableProfData *__llvm_profile_begin_vtables(void) { return NULL; } +const VTableProfData *__llvm_profile_end_vtables(void) { return NULL; } + const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } const char *__llvm_profile_end_names(void) { return &NamesEnd; } -uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } -uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +// Type profiling isn't supported on Windows, so return NULl to make it more +// explicit. +const char *__llvm_profile_begin_vtabnames(void) { return NULL; } +const char *__llvm_profile_end_vtabnames(void) { return NULL; } + +char *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +char *__llvm_profile_end_counters(void) { return &CountersEnd; } +char *__llvm_profile_begin_bitmap(void) { return &BitmapStart + 1; } +char *__llvm_profile_end_bitmap(void) { return &BitmapEnd; } uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } @@ -65,4 +93,19 @@ ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } ValueProfNode *CurrentVNode = &VNodesStart + 1; ValueProfNode *EndVNode = &VNodesEnd; +/* lld-link provides __buildid symbol which points to the 16 bytes build id when + * using /build-id flag. https://lld.llvm.org/windows_support.html#lld-flags */ +#define BUILD_ID_LEN 16 +COMPILER_RT_WEAK uint8_t __buildid[BUILD_ID_LEN] = {0}; +COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { + static const uint8_t zeros[BUILD_ID_LEN] = {0}; + if (memcmp(__buildid, zeros, BUILD_ID_LEN) != 0) { + if (Writer && + lprofWriteOneBinaryId(Writer, BUILD_ID_LEN, __buildid, 0) == -1) + return -1; + return sizeof(uint64_t) + BUILD_ID_LEN; + } + return 0; +} + #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h index 20cf5d660c6ae..ed0905cc5f202 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h @@ -23,14 +23,22 @@ #define COMPILER_RT_FTRUNCATE(f,l) _chsize(_fileno(f),l) #define COMPILER_RT_ALWAYS_INLINE __forceinline #define COMPILER_RT_CLEANUP(x) +#define COMPILER_RT_USED #elif __GNUC__ -#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) +#ifdef _WIN32 +#define COMPILER_RT_FTRUNCATE(f, l) _chsize(fileno(f), l) +#define COMPILER_RT_VISIBILITY +#define COMPILER_RT_WEAK __attribute__((selectany)) +#else +#define COMPILER_RT_FTRUNCATE(f, l) ftruncate(fileno(f), l) #define COMPILER_RT_VISIBILITY __attribute__((visibility("hidden"))) #define COMPILER_RT_WEAK __attribute__((weak)) +#endif +#define COMPILER_RT_ALIGNAS(x) __attribute__((aligned(x))) #define COMPILER_RT_ALLOCA __builtin_alloca -#define COMPILER_RT_FTRUNCATE(f,l) ftruncate(fileno(f),l) #define COMPILER_RT_ALWAYS_INLINE inline __attribute((always_inline)) #define COMPILER_RT_CLEANUP(x) __attribute__((cleanup(x))) +#define COMPILER_RT_USED __attribute__((used)) #endif #if defined(__APPLE__) @@ -53,9 +61,9 @@ #endif #if COMPILER_RT_HAS_ATOMICS == 1 -#ifdef _MSC_VER +#ifdef _WIN32 #include -#if _MSC_VER < 1900 +#if defined(_MSC_VER) && _MSC_VER < 1900 #define snprintf _snprintf #endif #if defined(_WIN64) @@ -73,7 +81,7 @@ (DomType *)InterlockedExchangeAdd((LONG volatile *)&PtrVar, \ (LONG)sizeof(DomType) * PtrIncr) #endif -#else /* !defined(_MSC_VER) */ +#else /* !defined(_WIN32) */ #define COMPILER_RT_BOOL_CMPXCHG(Ptr, OldV, NewV) \ __sync_bool_compare_and_swap(Ptr, OldV, NewV) #define COMPILER_RT_PTR_FETCH_ADD(DomType, PtrVar, PtrIncr) \ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp index 5dff09d706329..6b2ce97001735 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingRuntime.cpp @@ -10,21 +10,15 @@ extern "C" { #include "InstrProfiling.h" -/* int __llvm_profile_runtime */ -COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR; +static int RegisterRuntime() { + __llvm_profile_initialize(); +#ifdef _AIX + extern COMPILER_RT_VISIBILITY void *__llvm_profile_keep[]; + (void)*(void *volatile *)__llvm_profile_keep; +#endif + return 0; } -namespace { - -class RegisterRuntime { -public: - RegisterRuntime() { - __llvm_profile_initialize_file(); - if (!__llvm_profile_is_continuous_mode_enabled()) - __llvm_profile_register_write_file_atexit(); - } -}; - -RegisterRuntime Registration; - +/* int __llvm_profile_runtime */ +COMPILER_RT_VISIBILITY int INSTR_PROF_PROFILE_RUNTIME_VAR = RegisterRuntime(); } diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c index bf5a9670fe18c..cd18cba3e268f 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -12,12 +12,13 @@ #include #include "WindowsMMap.h" #else +#include +#include #include +#include #include #include #include -#include -#include #endif #ifdef COMPILER_RT_HAS_UNAME @@ -32,10 +33,20 @@ #include #endif +#if defined(__Fuchsia__) +#include +#include +#endif + +#if defined(__FreeBSD__) +#include +#include +#endif + #include "InstrProfiling.h" #include "InstrProfilingUtil.h" -COMPILER_RT_WEAK unsigned lprofDirMode = 0755; +COMPILER_RT_VISIBILITY unsigned lprofDirMode = 0755; COMPILER_RT_VISIBILITY void __llvm_profile_recursive_mkdir(char *path) { @@ -313,20 +324,52 @@ COMPILER_RT_VISIBILITY const char *lprofFindLastDirSeparator(const char *Path) { return Sep; } -COMPILER_RT_VISIBILITY int lprofSuspendSigKill() { +COMPILER_RT_VISIBILITY int lprofSuspendSigKill(void) { #if defined(__linux__) int PDeachSig = 0; /* Temporarily suspend getting SIGKILL upon exit of the parent process. */ if (prctl(PR_GET_PDEATHSIG, &PDeachSig) == 0 && PDeachSig == SIGKILL) prctl(PR_SET_PDEATHSIG, 0); return (PDeachSig == SIGKILL); +#elif defined(__FreeBSD__) + int PDeachSig = 0, PDisableSig = 0; + if (procctl(P_PID, 0, PROC_PDEATHSIG_STATUS, &PDeachSig) == 0 && + PDeachSig == SIGKILL) + procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PDisableSig); + return (PDeachSig == SIGKILL); #else return 0; #endif } -COMPILER_RT_VISIBILITY void lprofRestoreSigKill() { +COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) { #if defined(__linux__) prctl(PR_SET_PDEATHSIG, SIGKILL); +#elif defined(__FreeBSD__) + int PEnableSig = SIGKILL; + procctl(P_PID, 0, PROC_PDEATHSIG_CTL, &PEnableSig); +#endif +} + +COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin, + uintptr_t End) { +#if defined(__ve__) + // VE doesn't support madvise. + return 0; +#else + size_t PageSize = getpagesize(); + uintptr_t BeginAligned = lprofRoundUpTo((uintptr_t)Begin, PageSize); + uintptr_t EndAligned = lprofRoundDownTo((uintptr_t)End, PageSize); + if (BeginAligned < EndAligned) { +#if defined(__Fuchsia__) + return _zx_vmar_op_range(_zx_vmar_root_self(), ZX_VMAR_OP_DECOMMIT, + (zx_vaddr_t)BeginAligned, + EndAligned - BeginAligned, NULL, 0); +#else + return madvise((void *)BeginAligned, EndAligned - BeginAligned, + MADV_DONTNEED); +#endif + } + return 0; #endif } diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h index f0e29a8803a05..4a88a03580941 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -9,6 +9,7 @@ #ifndef PROFILE_INSTRPROFILINGUTIL_H #define PROFILE_INSTRPROFILINGUTIL_H +#include #include #include @@ -30,11 +31,13 @@ int lprofUnlockFileHandle(FILE *F); * lock for exclusive access. The caller will block * if the lock is already held by another process. */ FILE *lprofOpenFileEx(const char *Filename); -/* PS4 doesn't have setenv/getenv. Define a shim. */ +/* PS4 doesn't have setenv/getenv/fork. Define a shim. */ #if __ORBIS__ +#include static inline char *getenv(const char *name) { return NULL; } static inline int setenv(const char *name, const char *value, int overwrite) { return 0; } +static pid_t fork() { return -1; } #endif /* #if __ORBIS__ */ /* GCOV_PREFIX and GCOV_PREFIX_STRIP support */ @@ -71,4 +74,14 @@ int lprofSuspendSigKill(); /* Restore previously suspended SIGKILL. */ void lprofRestoreSigKill(); +static inline size_t lprofRoundUpTo(size_t x, size_t boundary) { + return (x + boundary - 1) & ~(boundary - 1); +} + +static inline size_t lprofRoundDownTo(size_t x, size_t boundary) { + return x & ~(boundary - 1); +} + +int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End); + #endif /* PROFILE_INSTRPROFILINGUTIL_H */ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c index fd53cac3dff31..a608d41d39e77 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingValue.c @@ -6,6 +6,7 @@ |* \*===----------------------------------------------------------------------===*/ +#include #include #include #include @@ -17,13 +18,14 @@ #define INSTR_PROF_VALUE_PROF_DATA #define INSTR_PROF_COMMON_API_IMPL +#define INSTR_PROF_VALUE_PROF_MEMOP_API #include "profile/InstrProfData.inc" static int hasStaticCounters = 1; static int OutOfNodesWarnings = 0; static int hasNonDefaultValsPerSite = 0; #define INSTR_PROF_MAX_VP_WARNS 10 -#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 16 +#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 24 #define INSTR_PROF_VNODE_POOL_SIZE 1024 #ifndef _MSC_VER @@ -37,7 +39,7 @@ COMPILER_RT_VISIBILITY ValueProfNode COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE; -COMPILER_RT_VISIBILITY void lprofSetupValueProfiler() { +COMPILER_RT_VISIBILITY void lprofSetupValueProfiler(void) { const char *Str = 0; Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE"); if (Str && Str[0]) { @@ -57,7 +59,19 @@ COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) { COMPILER_RT_VISIBILITY void __llvm_profile_set_num_value_sites(__llvm_profile_data *Data, uint32_t ValueKind, uint16_t NumValueSites) { +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-qual" +#elif defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wcast-qual" +#endif *((uint16_t *)&Data->NumValueSites[ValueKind]) = NumValueSites; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#elif defined(__clang__) +#pragma clang diagnostic pop +#endif } /* This method is only used in value profiler mock testing. */ @@ -93,6 +107,8 @@ static int allocateValueProfileCounters(__llvm_profile_data *Data) { for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI) NumVSites += Data->NumValueSites[VKI]; + // If NumVSites = 0, calloc is allowed to return a non-null pointer. + assert(NumVSites > 0 && "NumVSites can't be zero"); ValueProfNode **Mem = (ValueProfNode **)calloc(NumVSites, sizeof(ValueProfNode *)); if (!Mem) @@ -235,38 +251,21 @@ __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, } /* - * The target values are partitioned into multiple regions/ranges. There is one - * contiguous region which is precise -- every value in the range is tracked - * individually. A value outside the precise region will be collapsed into one - * value depending on the region it falls in. - * - * There are three regions: - * 1. (-inf, PreciseRangeStart) and (PreciseRangeLast, LargeRangeValue) belong - * to one region -- all values here should be mapped to one value of - * "PreciseRangeLast + 1". - * 2. [PreciseRangeStart, PreciseRangeLast] - * 3. Large values: [LargeValue, +inf) maps to one value of LargeValue. - * - * The range for large values is optional. The default value of INT64_MIN - * indicates it is not specified. + * The target values are partitioned into multiple ranges. The range spec is + * defined in InstrProfData.inc. */ -COMPILER_RT_VISIBILITY void __llvm_profile_instrument_range( - uint64_t TargetValue, void *Data, uint32_t CounterIndex, - int64_t PreciseRangeStart, int64_t PreciseRangeLast, int64_t LargeValue) { - - if (LargeValue != INT64_MIN && (int64_t)TargetValue >= LargeValue) - TargetValue = LargeValue; - else if ((int64_t)TargetValue < PreciseRangeStart || - (int64_t)TargetValue > PreciseRangeLast) - TargetValue = PreciseRangeLast + 1; - - __llvm_profile_instrument_target(TargetValue, Data, CounterIndex); +COMPILER_RT_VISIBILITY void +__llvm_profile_instrument_memop(uint64_t TargetValue, void *Data, + uint32_t CounterIndex) { + // Map the target value to the representative value of its range. + uint64_t RepValue = InstrProfGetRangeRepValue(TargetValue); + __llvm_profile_instrument_target(RepValue, Data, CounterIndex); } /* * A wrapper struct that represents value profile runtime data. * Like InstrProfRecord class which is used by profiling host tools, - * ValueProfRuntimeRecord also implements the abstract intefaces defined in + * ValueProfRuntimeRecord also implements the abstract interfaces defined in * ValueProfRecordClosure so that the runtime data can be serialized using * shared C implementation. */ @@ -366,6 +365,6 @@ static VPDataReaderType TheVPDataReader = { getFirstValueProfRecord, getNumValueDataForSiteWrapper, getValueProfDataSizeWrapper, getNextNValueData}; -COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader() { +COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader(void) { return &TheVPDataReader; } diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c b/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c new file mode 100644 index 0000000000000..21400bfb2caa0 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingVersionVar.c @@ -0,0 +1,18 @@ +/*===- InstrProfilingVersionVar.c - profile version variable setup -------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +/* uint64 __llvm_profile_raw_version + * + * The runtime should only provide its own definition of this symbol when the + * user has not specified one. Set this up by moving the runtime's copy of this + * symbol to an object file within the archive. + */ +COMPILER_RT_VISIBILITY COMPILER_RT_WEAK uint64_t INSTR_PROF_RAW_VERSION_VAR = + INSTR_PROF_RAW_VERSION; diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c index c34e110a69590..8816a71155511 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingWriter.c @@ -6,6 +6,9 @@ |* \*===----------------------------------------------------------------------===*/ +// Note: This is linked into the Darwin kernel, and must remain compatible +// with freestanding compilation. See `darwin_add_builtin_libraries`. + #ifdef _MSC_VER /* For _alloca */ #include @@ -29,7 +32,7 @@ static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray); COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0; COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0; -/* The buffer writer is reponsponsible in keeping writer state +/* The buffer writer is responsible in keeping writer state * across the call. */ COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This, @@ -241,60 +244,136 @@ COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer, /* Match logic in __llvm_profile_write_buffer(). */ const __llvm_profile_data *DataBegin = __llvm_profile_begin_data(); const __llvm_profile_data *DataEnd = __llvm_profile_end_data(); - const uint64_t *CountersBegin = __llvm_profile_begin_counters(); - const uint64_t *CountersEnd = __llvm_profile_end_counters(); + const char *CountersBegin = __llvm_profile_begin_counters(); + const char *CountersEnd = __llvm_profile_end_counters(); + const char *BitmapBegin = __llvm_profile_begin_bitmap(); + const char *BitmapEnd = __llvm_profile_end_bitmap(); const char *NamesBegin = __llvm_profile_begin_names(); const char *NamesEnd = __llvm_profile_end_names(); + const VTableProfData *VTableBegin = __llvm_profile_begin_vtables(); + const VTableProfData *VTableEnd = __llvm_profile_end_vtables(); + const char *VNamesBegin = __llvm_profile_begin_vtabnames(); + const char *VNamesEnd = __llvm_profile_end_vtabnames(); return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin, - CountersEnd, VPDataReader, NamesBegin, NamesEnd, - SkipNameDataWrite); + CountersEnd, BitmapBegin, BitmapEnd, VPDataReader, + NamesBegin, NamesEnd, VTableBegin, VTableEnd, + VNamesBegin, VNamesEnd, SkipNameDataWrite); } COMPILER_RT_VISIBILITY int lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd, - const uint64_t *CountersBegin, const uint64_t *CountersEnd, + const char *CountersBegin, const char *CountersEnd, + const char *BitmapBegin, const char *BitmapEnd, VPDataReaderType *VPDataReader, const char *NamesBegin, - const char *NamesEnd, int SkipNameDataWrite) { - + const char *NamesEnd, const VTableProfData *VTableBegin, + const VTableProfData *VTableEnd, const char *VNamesBegin, + const char *VNamesEnd, int SkipNameDataWrite) { /* Calculate size of sections. */ - const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); - const uint64_t CountersSize = CountersEnd - CountersBegin; - const uint64_t NamesSize = NamesEnd - NamesBegin; + const uint64_t DataSectionSize = + __llvm_profile_get_data_size(DataBegin, DataEnd); + const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd); + const uint64_t CountersSectionSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + const uint64_t NumCounters = + __llvm_profile_get_num_counters(CountersBegin, CountersEnd); + const uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); + const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd); + const uint64_t NumVTables = + __llvm_profile_get_num_vtable(VTableBegin, VTableEnd); + const uint64_t VTableSectionSize = + __llvm_profile_get_vtable_section_size(VTableBegin, VTableEnd); + const uint64_t VNamesSize = + __llvm_profile_get_name_size(VNamesBegin, VNamesEnd); /* Create the header. */ __llvm_profile_header Header; - if (!DataSize) - return 0; - /* Determine how much padding is needed before/after the counters and after * the names. */ uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters, - PaddingBytesAfterNames; - __llvm_profile_get_padding_sizes_for_counters( - DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters, - &PaddingBytesAfterCounters, &PaddingBytesAfterNames); + PaddingBytesAfterBitmapBytes, PaddingBytesAfterNames, + PaddingBytesAfterVTable, PaddingBytesAfterVNames; + if (__llvm_profile_get_padding_sizes_for_counters( + DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize, + VTableSectionSize, VNamesSize, &PaddingBytesBeforeCounters, + &PaddingBytesAfterCounters, &PaddingBytesAfterBitmapBytes, + &PaddingBytesAfterNames, &PaddingBytesAfterVTable, + &PaddingBytesAfterVNames) == -1) + return -1; + { /* Initialize header structure. */ #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init; #include "profile/InstrProfData.inc" + } + + /* On WIN64, label differences are truncated 32-bit values. Truncate + * CountersDelta to match. */ +#ifdef _WIN64 + Header.CountersDelta = (uint32_t)Header.CountersDelta; + Header.BitmapDelta = (uint32_t)Header.BitmapDelta; +#endif + + /* The data and names sections are omitted in lightweight mode. */ + if (NumData == 0 && NamesSize == 0) { + Header.CountersDelta = 0; + Header.NamesDelta = 0; + } + + /* Write the profile header. */ + ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}}; + if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) + return -1; + + /* Write the binary id lengths and data. */ + if (__llvm_write_binary_ids(Writer) == -1) + return -1; - /* Write the data. */ - ProfDataIOVec IOVec[] = { - {&Header, sizeof(__llvm_profile_header), 1, 0}, - {DataBegin, sizeof(__llvm_profile_data), DataSize, 0}, + /* Write the profile data. */ + ProfDataIOVec IOVecData[] = { + {DataBegin, sizeof(uint8_t), DataSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1}, - {CountersBegin, sizeof(uint64_t), CountersSize, 0}, + {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0}, {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1}, + {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1}, {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0}, - {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}}; - if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec))) + {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}, + {VTableBegin, sizeof(uint8_t), VTableSectionSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVTable, 1}, + {SkipNameDataWrite ? NULL : VNamesBegin, sizeof(uint8_t), VNamesSize, 0}, + {NULL, sizeof(uint8_t), PaddingBytesAfterVNames, 1}}; + if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData))) return -1; - /* Value profiling is not yet supported in continuous mode. */ - if (__llvm_profile_is_continuous_mode_enabled()) + /* Value profiling is not yet supported in continuous mode and profile + * correlation mode. */ + if (__llvm_profile_is_continuous_mode_enabled() || + (NumData == 0 && NamesSize == 0)) return 0; return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd); } + +/* + * Write binary id length and then its data, because binary id does not + * have a fixed length. + */ +COMPILER_RT_VISIBILITY +int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen, + const uint8_t *BinaryIdData, + uint64_t BinaryIdPadding) { + ProfDataIOVec BinaryIdIOVec[] = { + {&BinaryIdLen, sizeof(uint64_t), 1, 0}, + {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0}, + {NULL, sizeof(uint8_t), BinaryIdPadding, 1}, + }; + if (Writer->Write(Writer, BinaryIdIOVec, + sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec))) + return -1; + + /* Successfully wrote binary id, report success. */ + return 0; +} diff --git a/system/lib/compiler-rt/lib/profile/WindowsMMap.c b/system/lib/compiler-rt/lib/profile/WindowsMMap.c new file mode 100644 index 0000000000000..9d7da835b1ed3 --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/WindowsMMap.c @@ -0,0 +1,187 @@ +/* + * This code is derived from uClibc (original license follows). + * https://git.uclibc.org/uClibc/tree/utils/mmap-windows.c + */ + /* mmap() replacement for Windows + * + * Author: Mike Frysinger + * Placed into the public domain + */ + +/* References: + * CreateFileMapping: http://msdn.microsoft.com/en-us/library/aa366537(VS.85).aspx + * CloseHandle: http://msdn.microsoft.com/en-us/library/ms724211(VS.85).aspx + * MapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366761(VS.85).aspx + * UnmapViewOfFile: http://msdn.microsoft.com/en-us/library/aa366882(VS.85).aspx + */ + +#if defined(_WIN32) + +#include "WindowsMMap.h" + +#define WIN32_LEAN_AND_MEAN +#include + +#include "InstrProfiling.h" + +COMPILER_RT_VISIBILITY +void *mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset) +{ + if (prot & ~(PROT_READ | PROT_WRITE | PROT_EXEC)) + return MAP_FAILED; + if (fd == -1) { + if (!(flags & MAP_ANON) || offset) + return MAP_FAILED; + } else if (flags & MAP_ANON) + return MAP_FAILED; + + DWORD flProtect; + if (prot & PROT_WRITE) { + if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE_READWRITE; + else + flProtect = PAGE_READWRITE; + } else if (prot & PROT_EXEC) { + if (prot & PROT_READ) + flProtect = PAGE_EXECUTE_READ; + else if (prot & PROT_EXEC) + flProtect = PAGE_EXECUTE; + } else + flProtect = PAGE_READONLY; + + off_t end = length + offset; + HANDLE mmap_fd, h; + if (fd == -1) + mmap_fd = INVALID_HANDLE_VALUE; + else + mmap_fd = (HANDLE)_get_osfhandle(fd); + h = CreateFileMapping(mmap_fd, NULL, flProtect, DWORD_HI(end), DWORD_LO(end), NULL); + if (h == NULL) + return MAP_FAILED; + + DWORD dwDesiredAccess; + if (prot & PROT_WRITE) + dwDesiredAccess = FILE_MAP_WRITE; + else + dwDesiredAccess = FILE_MAP_READ; + if (prot & PROT_EXEC) + dwDesiredAccess |= FILE_MAP_EXECUTE; + if (flags & MAP_PRIVATE) + dwDesiredAccess |= FILE_MAP_COPY; + void *ret = MapViewOfFile(h, dwDesiredAccess, DWORD_HI(offset), DWORD_LO(offset), length); + if (ret == NULL) { + CloseHandle(h); + ret = MAP_FAILED; + } + return ret; +} + +COMPILER_RT_VISIBILITY +void munmap(void *addr, size_t length) +{ + UnmapViewOfFile(addr); + /* ruh-ro, we leaked handle from CreateFileMapping() ... */ +} + +COMPILER_RT_VISIBILITY +int msync(void *addr, size_t length, int flags) +{ + if (flags & MS_INVALIDATE) + return -1; /* Not supported. */ + + /* Exactly one of MS_ASYNC or MS_SYNC must be specified. */ + switch (flags & (MS_ASYNC | MS_SYNC)) { + case MS_SYNC: + case MS_ASYNC: + break; + default: + return -1; + } + + if (!FlushViewOfFile(addr, length)) + return -1; + + if (flags & MS_SYNC) { + /* FIXME: No longer have access to handle from CreateFileMapping(). */ + /* + * if (!FlushFileBuffers(h)) + * return -1; + */ + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int madvise(void *addr, size_t length, int advice) +{ + if (advice != MADV_DONTNEED) + return -1; /* Not supported. */ + + if (!VirtualUnlock(addr, length)) + return -1; + + return 0; +} + +static int lock(HANDLE handle, DWORD lockType, BOOL blocking) { + DWORD flags = lockType; + if (!blocking) + flags |= LOCKFILE_FAIL_IMMEDIATELY; + + OVERLAPPED overlapped; + ZeroMemory(&overlapped, sizeof(OVERLAPPED)); + overlapped.hEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + BOOL result = LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped); + if (!result) { + DWORD dw = GetLastError(); + + // In non-blocking mode, return an error if the file is locked. + if (!blocking && dw == ERROR_LOCK_VIOLATION) + return -1; // EWOULDBLOCK + + // If the error is ERROR_IO_PENDING, we need to wait until the operation + // finishes. Otherwise, we return an error. + if (dw != ERROR_IO_PENDING) + return -1; + + DWORD dwNumBytes; + if (!GetOverlappedResult(handle, &overlapped, &dwNumBytes, TRUE)) + return -1; + } + + return 0; +} + +COMPILER_RT_VISIBILITY +int flock(int fd, int operation) { + HANDLE handle = (HANDLE)_get_osfhandle(fd); + if (handle == INVALID_HANDLE_VALUE) + return -1; + + BOOL blocking = (operation & LOCK_NB) == 0; + int op = operation & ~LOCK_NB; + + switch (op) { + case LOCK_EX: + return lock(handle, LOCKFILE_EXCLUSIVE_LOCK, blocking); + + case LOCK_SH: + return lock(handle, 0, blocking); + + case LOCK_UN: + if (!UnlockFile(handle, 0, 0, MAXDWORD, MAXDWORD)) + return -1; + break; + + default: + return -1; + } + + return 0; +} + +#undef DWORD_HI +#undef DWORD_LO + +#endif /* _WIN32 */ diff --git a/system/lib/compiler-rt/lib/profile/WindowsMMap.h b/system/lib/compiler-rt/lib/profile/WindowsMMap.h new file mode 100644 index 0000000000000..1df1a0be0b02b --- /dev/null +++ b/system/lib/compiler-rt/lib/profile/WindowsMMap.h @@ -0,0 +1,82 @@ +/*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#ifndef PROFILE_INSTRPROFILING_WINDOWS_MMAP_H +#define PROFILE_INSTRPROFILING_WINDOWS_MMAP_H + +#if defined(_WIN32) + +#include +#include +#include + +/* + * mmap() flags + */ +#define PROT_READ 0x1 +#define PROT_WRITE 0x2 +#define PROT_EXEC 0x0 + +#define MAP_FILE 0x00 +#define MAP_SHARED 0x01 +#define MAP_PRIVATE 0x02 +#define MAP_ANONYMOUS 0x20 +#define MAP_ANON MAP_ANONYMOUS +#define MAP_FAILED ((void *) -1) + +/* + * msync() flags + */ +#define MS_ASYNC 0x0001 /* return immediately */ +#define MS_INVALIDATE 0x0002 /* invalidate all cached data */ +#define MS_SYNC 0x0010 /* msync synchronously */ + +/* + * madvise() flags + */ + +#define MADV_NORMAL 0 /* no special treatment */ +#define MADV_WILLNEED 3 /* expect access in the near future */ +#define MADV_DONTNEED 4 /* do not expect access in the near future */ + +/* + * flock() operations + */ +#define LOCK_SH 1 /* shared lock */ +#define LOCK_EX 2 /* exclusive lock */ +#define LOCK_NB 4 /* don't block when locking */ +#define LOCK_UN 8 /* unlock */ + +#ifdef __USE_FILE_OFFSET64 +# define DWORD_HI(x) (x >> 32) +# define DWORD_LO(x) ((x) & 0xffffffff) +#else +# define DWORD_HI(x) (0) +# define DWORD_LO(x) (x) +#endif + +#define mmap __llvm_profile_mmap +#define munmap __llvm_profile_munmap +#define msync __llvm_profile_msync +#define madvise __llvm_profile_madvise +#define flock __llvm_profile_flock + +void *mmap(void *start, size_t length, int prot, int flags, int fd, + off_t offset); + +void munmap(void *addr, size_t length); + +int msync(void *addr, size_t length, int flags); + +int madvise(void *addr, size_t length, int advice); + +int flock(int fd, int operation); + +#endif /* _WIN32 */ + +#endif /* PROFILE_INSTRPROFILING_WINDOWS_MMAP_H */ From af4b5ddc831a9e81d0905fb8a4eee1b8de6472f9 Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Wed, 16 Apr 2025 12:45:36 -0400 Subject: [PATCH 04/11] bitmaps --- .../lib/profile/InstrProfilingPlatformOther.c | 9 +++++++++ tools/system_libs.py | 12 ++++++++++++ 2 files changed, 21 insertions(+) diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index aa79a5641ceca..073309d85a535 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -26,6 +26,8 @@ static const char *VNamesFirst = NULL; static const char *VNamesLast = NULL; static char *CountersFirst = NULL; static char *CountersLast = NULL; +static char *BitmapFirst = NULL; +static char *BitmapLast = NULL; static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { @@ -53,18 +55,25 @@ void __llvm_profile_register_function(void *Data_) { CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr); CountersLast = CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size(); + BitmapFirst = (char *)((uintptr_t)Data_ + Data->BitmapPtr); + BitmapLast = BitmapFirst + Data->NumBitmapBytes; return; } DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); CountersFirst = (char *)getMinAddr( CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr)); + BitmapFirst = (char *)getMinAddr( + BitmapFirst, (char *)((uintptr_t)Data_ + Data->BitmapPtr)); DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); CountersLast = (char *)getMaxAddr( CountersLast, (char *)((uintptr_t)Data_ + Data->CounterPtr) + Data->NumCounters * __llvm_profile_counter_entry_size()); + BitmapLast = (char *)getMaxAddr( + BitmapLast, + (char *)((uintptr_t)Data_ + Data->CounterPtr) + Data->NumBitmapBytes); } COMPILER_RT_VISIBILITY diff --git a/tools/system_libs.py b/tools/system_libs.py index ae3a3168de21f..1688fc5c02317 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2083,6 +2083,16 @@ class CompilerRTLibrary(Library): force_object_files = True +class libcompiler_rt_profile(CompilerRTLibrary, MTLibrary): + name = 'libcompiler_rt_profile' + + + cflags = ['-DCOMPILER_RT_HAS_UNAME=1'] + includes = ['system/lib/libc'] + src_dir = 'system/lib/compiler-rt/lib/profile' + src_files = glob_in_path(src_dir, '*.c') + src_files += glob_in_path(src_dir, '*.cpp') + class libubsan_minimal_rt(CompilerRTLibrary, MTLibrary): name = 'libubsan_minimal_rt' never_force = True @@ -2357,6 +2367,7 @@ def add_sanitizer_libs(): if only_forced: add_library('libcompiler_rt') + add_library('libcompiler_rt_profile') add_sanitizer_libs() add_forced_libs() return libs_to_link @@ -2394,6 +2405,7 @@ def add_sanitizer_libs(): elif settings.MALLOC != 'none': add_library('libmalloc') add_library('libcompiler_rt') + add_library('libcompiler_rt_profile') if settings.LINK_AS_CXX: add_library('libc++') if settings.LINK_AS_CXX or sanitize: From 12d5e88485a7d4fbcaa7346dfc08a6a00ecae8a3 Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Thu, 17 Apr 2025 10:09:55 -0400 Subject: [PATCH 05/11] update --- .../lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index 073309d85a535..e329dc2dbe2e2 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -73,7 +73,7 @@ void __llvm_profile_register_function(void *Data_) { Data->NumCounters * __llvm_profile_counter_entry_size()); BitmapLast = (char *)getMaxAddr( BitmapLast, - (char *)((uintptr_t)Data_ + Data->CounterPtr) + Data->NumBitmapBytes); + (char *)((uintptr_t)Data_ + Data->BitmapPtr) + Data->NumBitmapBytes); } COMPILER_RT_VISIBILITY From a0f30c6aac09c36ee7747a026e87eeab6569de3c Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Fri, 18 Apr 2025 10:11:31 -0400 Subject: [PATCH 06/11] update port --- .../include/profile/InstrProfData.inc | 29 ++++++- .../compiler-rt/lib/profile/GCDAProfiling.c | 25 +++++- .../compiler-rt/lib/profile/InstrProfiling.h | 27 ++++-- .../lib/profile/InstrProfilingFile.c | 54 ++++++++++-- .../lib/profile/InstrProfilingInternal.h | 6 +- .../lib/profile/InstrProfilingMerge.c | 55 ++++++------ .../lib/profile/InstrProfilingPlatformAIX.c | 5 +- .../profile/InstrProfilingPlatformFuchsia.c | 2 +- .../lib/profile/InstrProfilingPlatformLinux.c | 64 +++++++------- .../lib/profile/InstrProfilingPlatformOther.c | 12 +-- .../lib/profile/InstrProfilingPort.h | 4 +- .../lib/profile/InstrProfilingUtil.c | 83 ++++++++++++++++++- .../lib/profile/InstrProfilingUtil.h | 9 +- tools/system_libs.py | 2 - 14 files changed, 275 insertions(+), 102 deletions(-) diff --git a/system/lib/compiler-rt/include/profile/InstrProfData.inc b/system/lib/compiler-rt/include/profile/InstrProfData.inc index 847e53cfa7432..39613da81ecb4 100644 --- a/system/lib/compiler-rt/include/profile/InstrProfData.inc +++ b/system/lib/compiler-rt/include/profile/InstrProfData.inc @@ -303,6 +303,18 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #undef COVMAP_HEADER /* COVMAP_HEADER end. */ +/* COVINIT_FUNC start */ +#ifndef COVINIT_FUNC +#define COVINIT_FUNC(Type, LLVMType, Name, Initializer) +#else +#define INSTR_PROF_DATA_DEFINED +#endif +COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), WriteoutFunction, \ + WriteoutF) +COVINIT_FUNC(IntPtrT, llvm::PointerType::getUnqual(Ctx), ResetFunction, \ + ResetF) +#undef COVINIT_FUNC +/* COVINIT_FUNC end */ #ifdef INSTR_PROF_SECT_ENTRY #define INSTR_PROF_DATA_DEFINED @@ -345,6 +357,9 @@ INSTR_PROF_SECT_ENTRY(IPSK_covdata, \ INSTR_PROF_SECT_ENTRY(IPSK_covname, \ INSTR_PROF_QUOTE(INSTR_PROF_COVNAME_COMMON), \ INSTR_PROF_COVNAME_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_covinit, \ + INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON), \ + INSTR_PROF_COVINIT_COFF, "__LLVM_COV,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -715,10 +730,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVMAP_VERSION 6 /* Profile version is always of type uint64_t. Reserve the upper 32 bits in the - * version for other variants of profile. We set the 8th most significant bit + * version for other variants of profile. We set the 8th most significant bit * (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentation * generated profile, and 0 if this is a Clang FE generated profile. * 1 in bit 57 indicates there are context-sensitive records in the profile. + * The 54th bit indicates whether to always instrument loop entry blocks. + * The 58th bit indicates whether to always instrument function entry blocks. * The 59th bit indicates whether to use debug info to correlate profiles. * The 60th bit indicates single byte coverage instrumentation. * The 61st bit indicates function entry instrumentation only. @@ -727,6 +744,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, */ #define VARIANT_MASKS_ALL 0xffffffff00000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) +#define VARIANT_MASK_INSTR_LOOP_ENTRIES (0x1ULL << 55) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) #define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define VARIANT_MASK_INSTR_ENTRY (0x1ULL << 58) @@ -738,6 +756,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime #define INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __llvm_profile_counter_bias +#define INSTR_PROF_PROFILE_BITMAP_BIAS_VAR __llvm_profile_bitmap_bias #define INSTR_PROF_PROFILE_SET_TIMESTAMP __llvm_profile_set_timestamp #define INSTR_PROF_PROFILE_SAMPLING_VAR __llvm_profile_sampling @@ -760,6 +779,8 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVDATA_COMMON __llvm_covdata #define INSTR_PROF_COVNAME_COMMON __llvm_covnames #define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +#define INSTR_PROF_COVINIT_COMMON __llvm_covinit + /* Windows section names. Because these section names contain dollar characters, * they must be quoted. */ @@ -780,6 +801,10 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVNAME_COFF ".lcovn" #define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" +// FIXME: Placeholder for Windows. Windows currently does not initialize +// the GCOV functions in the runtime. +#define INSTR_PROF_COVINIT_COFF ".lcovd$M" + #ifdef _WIN32 /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COFF @@ -799,6 +824,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_COVDATA_SECT_NAME INSTR_PROF_COVDATA_COFF #define INSTR_PROF_COVNAME_SECT_NAME INSTR_PROF_COVNAME_COFF #define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF +#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_COVINIT_COFF #else /* Runtime section names and name strings. */ #define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) @@ -820,6 +846,7 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Order file instrumentation. */ #define INSTR_PROF_ORDERFILE_SECT_NAME \ INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) +#define INSTR_PROF_COVINIT_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVINIT_COMMON) #endif #define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer diff --git a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c index 4f46fd2839b90..ac01805e70adc 100644 --- a/system/lib/compiler-rt/lib/profile/GCDAProfiling.c +++ b/system/lib/compiler-rt/lib/profile/GCDAProfiling.c @@ -584,7 +584,7 @@ void llvm_reset_counters(void) { } } -#if !defined(_WIN32) +#if !defined(_WIN32) && !defined(__wasm__) COMPILER_RT_VISIBILITY pid_t __gcov_fork() { pid_t parent_pid = getpid(); @@ -617,13 +617,32 @@ void llvm_gcov_init(fn_ptr wfn, fn_ptr rfn) { atexit_ran = 1; /* Make sure we write out the data and delete the data structures. */ - atexit(llvm_delete_reset_function_list); + lprofAtExit(llvm_delete_reset_function_list); #ifdef _WIN32 - atexit(llvm_writeout_and_clear); + lprofAtExit(llvm_writeout_and_clear); #endif } } +#if defined(_AIX) +COMPILER_RT_VISIBILITY __attribute__((constructor)) void +__llvm_profile_gcov_initialize() { + const __llvm_gcov_init_func_struct *InitFuncStart = + __llvm_profile_begin_covinit(); + const __llvm_gcov_init_func_struct *InitFuncEnd = + __llvm_profile_end_covinit(); + + for (const __llvm_gcov_init_func_struct *Ptr = InitFuncStart; + Ptr != InitFuncEnd; ++Ptr) { + fn_ptr wfn = (fn_ptr)Ptr->WriteoutFunction; + fn_ptr rfn = (fn_ptr)Ptr->ResetFunction; + if (!(wfn && rfn)) + continue; + llvm_gcov_init(wfn, rfn); + } +} +#endif + void __gcov_dump(void) { for (struct fn_node *f = writeout_fn_list.head; f; f = f->next) f->fn(); diff --git a/system/lib/compiler-rt/lib/profile/InstrProfiling.h b/system/lib/compiler-rt/lib/profile/InstrProfiling.h index 6906d52eacaf1..7f0c0c194dc91 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfiling.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfiling.h @@ -54,6 +54,12 @@ typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) VTableProfData { #include "profile/InstrProfData.inc" } VTableProfData; +typedef struct COMPILER_RT_ALIGNAS(INSTR_PROF_DATA_ALIGNMENT) + __llvm_gcov_init_func_struct { +#define COVINIT_FUNC(Type, LLVMType, Name, Initializer) Type Name; +#include "profile/InstrProfData.inc" +} __llvm_gcov_init_func_struct; + /*! * \brief Return 1 if profile counters are continuously synced to the raw * profile via an mmap(). This is in contrast to the default mode, in which @@ -114,11 +120,11 @@ char *__llvm_profile_begin_counters(void); char *__llvm_profile_end_counters(void); char *__llvm_profile_begin_bitmap(void); char *__llvm_profile_end_bitmap(void); -ValueProfNode *__llvm_profile_begin_vnodes(); -ValueProfNode *__llvm_profile_end_vnodes(); -const VTableProfData *__llvm_profile_begin_vtables(); -const VTableProfData *__llvm_profile_end_vtables(); -uint32_t *__llvm_profile_begin_orderfile(); +ValueProfNode *__llvm_profile_begin_vnodes(void); +ValueProfNode *__llvm_profile_end_vnodes(void); +const VTableProfData *__llvm_profile_begin_vtables(void); +const VTableProfData *__llvm_profile_end_vtables(void); +uint32_t *__llvm_profile_begin_orderfile(void); /*! * \brief Merge profile data from buffer. @@ -208,6 +214,9 @@ void __llvm_profile_initialize_file(void); /*! \brief Initialize the profile runtime. */ void __llvm_profile_initialize(void); +/*! \brief Initialize the gcov profile runtime. */ +void __llvm_profile_gcov_initialize(void); + /*! * \brief Return path prefix (excluding the base filename) of the profile data. * This is useful for users using \c -fprofile-generate=./path_prefix who do @@ -216,7 +225,7 @@ void __llvm_profile_initialize(void); * merge mode is turned on for instrumented programs with shared libs). * Side-effect: this API call will invoke malloc with dynamic memory allocation. */ -const char *__llvm_profile_get_path_prefix(); +const char *__llvm_profile_get_path_prefix(void); /*! * \brief Return filename (including path) of the profile data. Note that if the @@ -229,7 +238,7 @@ const char *__llvm_profile_get_path_prefix(); * instrumented image/DSO). This API only retrieves the filename from the copy * of the runtime available to the calling image. */ -const char *__llvm_profile_get_filename(); +const char *__llvm_profile_get_filename(void); /*! \brief Get the magic token for the file format. */ uint64_t __llvm_profile_get_magic(void); @@ -293,7 +302,7 @@ int __llvm_profile_get_padding_sizes_for_counters( * certain processes in case the processes don't have permission to write to * the disks, and trying to do so would result in side effects such as crashes. */ -void __llvm_profile_set_dumped(); +void __llvm_profile_set_dumped(void); /*! * This variable is defined in InstrProfilingRuntime.cpp as a hidden @@ -324,4 +333,6 @@ COMPILER_RT_VISIBILITY extern uint64_t */ extern char INSTR_PROF_PROFILE_NAME_VAR[1]; /* __llvm_profile_filename. */ +const __llvm_gcov_init_func_struct *__llvm_profile_begin_covinit(); +const __llvm_gcov_init_func_struct *__llvm_profile_end_covinit(); #endif /* PROFILE_INSTRPROFILING_H_ */ diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c index 1c58584d2d4f7..bad4cc71801ec 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingFile.c @@ -101,6 +101,8 @@ static const int UseBiasVar = 0; static const char *FileOpenMode = "a+b"; static void *BiasAddr = NULL; static void *BiasDefaultAddr = NULL; +static void *BitmapBiasAddr = NULL; +static void *BitmapBiasDefaultAddr = NULL; static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Get the sizes of various profile data sections. Taken from * __llvm_profile_get_size_for_buffer(). */ @@ -194,16 +196,20 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { } return 0; } -#elif defined(__ELF__) || defined(_WIN32) +#elif defined(__ELF__) || defined(_WIN32) || defined(_AIX) #define INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR \ INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_COUNTER_BIAS_VAR, _default) -COMPILER_RT_VISIBILITY intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; +COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR = 0; +#define INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR \ + INSTR_PROF_CONCAT(INSTR_PROF_PROFILE_BITMAP_BIAS_VAR, _default) +COMPILER_RT_VISIBILITY int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR = 0; /* This variable is a weak external reference which could be used to detect * whether or not the compiler defined this symbol. */ #if defined(_MSC_VER) -COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR; #if defined(_M_IX86) || defined(__i386__) #define WIN_SYM_PREFIX "_" #else @@ -213,10 +219,17 @@ COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ INSTR_PROF_PROFILE_COUNTER_BIAS_VAR) "=" WIN_SYM_PREFIX \ INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)) +#pragma comment( \ + linker, "/alternatename:" WIN_SYM_PREFIX INSTR_PROF_QUOTE( \ + INSTR_PROF_PROFILE_BITMAP_BIAS_VAR) "=" WIN_SYM_PREFIX \ + INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR)) #else -COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR +COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR __attribute__((weak, alias(INSTR_PROF_QUOTE( INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR)))); +COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_BITMAP_BIAS_VAR + __attribute__((weak, alias(INSTR_PROF_QUOTE( + INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR)))); #endif static const int ContinuousModeSupported = 1; static const int UseBiasVar = 1; @@ -227,6 +240,9 @@ static const char *FileOpenMode = "w+b"; * used and runtime provides a weak alias so we can check if it's defined. */ static void *BiasAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; static void *BiasDefaultAddr = &INSTR_PROF_PROFILE_COUNTER_BIAS_DEFAULT_VAR; +static void *BitmapBiasAddr = &INSTR_PROF_PROFILE_BITMAP_BIAS_VAR; +static void *BitmapBiasDefaultAddr = + &INSTR_PROF_PROFILE_BITMAP_BIAS_DEFAULT_VAR; static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Get the sizes of various profile data sections. Taken from * __llvm_profile_get_size_for_buffer(). */ @@ -237,12 +253,18 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { const char *BitmapBegin = __llvm_profile_begin_bitmap(); const char *BitmapEnd = __llvm_profile_end_bitmap(); uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd); + uint64_t CountersSize = + __llvm_profile_get_counters_size(CountersBegin, CountersEnd); + uint64_t NumBitmapBytes = + __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd); /* Get the file size. */ uint64_t FileSize = 0; if (getProfileFileSizeForMerging(File, &FileSize)) return 1; int Fileno = fileno(File); + uint64_t PaddingBytesAfterCounters = + __llvm_profile_get_num_padding_bytes(CountersSize); uint64_t FileOffsetToCounters = sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) + DataSize; @@ -260,7 +282,17 @@ static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)CountersBegin, (uintptr_t)CountersEnd); - /* BIAS MODE not supported yet for Bitmap (MCDC). */ + /* Also mmap MCDC bitmap bytes. If there aren't any bitmap bytes, mmap() + * will fail with EINVAL. */ + if (NumBitmapBytes == 0) + return 0; + + /* Update profbm_bias. */ + uint64_t FileOffsetToBitmap = + FileOffsetToCounters + CountersSize + PaddingBytesAfterCounters; + /* Update the profile fields based on the current mapping. */ + INSTR_PROF_PROFILE_BITMAP_BIAS_VAR = + (uintptr_t)Profile - (uintptr_t)BitmapBegin + FileOffsetToBitmap; /* Return the memory allocated for counters to OS. */ lprofReleaseMemoryPagesToOS((uintptr_t)BitmapBegin, (uintptr_t)BitmapEnd); @@ -272,6 +304,8 @@ static const int UseBiasVar = 0; static const char *FileOpenMode = "a+b"; static void *BiasAddr = NULL; static void *BiasDefaultAddr = NULL; +static void *BitmapBiasAddr = NULL; +static void *BitmapBiasDefaultAddr = NULL; static int mmapForContinuousMode(uint64_t CurrentFileOffset, FILE *File) { return 0; } @@ -619,8 +653,10 @@ static void initializeProfileForContinuousMode(void) { PROF_ERR("%s\n", "continuous mode is unsupported on this platform"); return; } - if (UseBiasVar && BiasAddr == BiasDefaultAddr) { - PROF_ERR("%s\n", "__llvm_profile_counter_bias is undefined"); + if (UseBiasVar && BiasAddr == BiasDefaultAddr && + BitmapBiasAddr == BitmapBiasDefaultAddr) { + PROF_ERR("%s\n", "Neither __llvm_profile_counter_bias nor " + "__llvm_profile_bitmap_bias is defined"); return; } @@ -826,7 +862,7 @@ static int parseFilenamePattern(const char *FilenamePat, __llvm_profile_disable_continuous_mode(); return -1; } -#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) +#if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) || defined(_AIX) __llvm_profile_set_page_size(getpagesize()); __llvm_profile_enable_continuous_mode(); #else @@ -1223,7 +1259,7 @@ int __llvm_profile_register_write_file_atexit(void) { lprofSetupValueProfiler(); HasBeenRegistered = 1; - return atexit(writeFileWithoutReturn); + return lprofAtExit(writeFileWithoutReturn); } COMPILER_RT_VISIBILITY int __llvm_profile_set_file_object(FILE *File, diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h index d5bd0e41fb129..b100343ca04f9 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingInternal.h @@ -167,20 +167,20 @@ int lprofWriteDataImpl(ProfDataWriter *Writer, void lprofMergeValueProfData(struct ValueProfData *SrcValueProfData, __llvm_profile_data *DstData); -VPDataReaderType *lprofGetVPDataReader(); +VPDataReaderType *lprofGetVPDataReader(void); /* Internal interface used by test to reset the max number of * tracked values per value site to be \p MaxVals. */ void lprofSetMaxValsPerSite(uint32_t MaxVals); -void lprofSetupValueProfiler(); +void lprofSetupValueProfiler(void); /* Return the profile header 'signature' value associated with the current * executable or shared library. The signature value can be used to for * a profile name that is unique to this load module so that it does not * collide with profiles from other binaries. It also allows shared libraries * to dump merged profile data into its own profile file. */ -uint64_t lprofGetLoadModuleSignature(); +uint64_t lprofGetLoadModuleSignature(void); /* * Return non zero value if the profile data has already been diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c index c0706b73e1668..92721c4fd55ea 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingMerge.c @@ -137,26 +137,23 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, return 1; } - __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData; __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData; - char *SrcCountersStart, *DstCounter; - const char *SrcCountersEnd, *SrcCounter; - const char *SrcBitmapStart; - const char *SrcNameStart; - const char *SrcValueProfDataStart, *SrcValueProfData; uintptr_t CountersDelta = Header->CountersDelta; uintptr_t BitmapDelta = Header->BitmapDelta; - SrcDataStart = + __llvm_profile_data *SrcDataStart = (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) + Header->BinaryIdsSize); - SrcDataEnd = SrcDataStart + Header->NumData; - SrcCountersStart = (char *)SrcDataEnd; - SrcCountersEnd = SrcCountersStart + - Header->NumCounters * __llvm_profile_counter_entry_size(); - SrcBitmapStart = SrcCountersEnd; - SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; - SrcValueProfDataStart = + __llvm_profile_data *SrcDataEnd = SrcDataStart + Header->NumData; + uintptr_t SrcCountersStart = (uintptr_t)SrcDataEnd; + uintptr_t SrcCountersEnd = + SrcCountersStart + + Header->NumCounters * __llvm_profile_counter_entry_size(); + uintptr_t SrcBitmapStart = + SrcCountersEnd + + __llvm_profile_get_num_padding_bytes(SrcCountersEnd - SrcCountersStart); + uintptr_t SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes; + uintptr_t SrcValueProfDataStart = SrcNameStart + getDistanceFromCounterToValueProf(Header); if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart) return 1; @@ -164,13 +161,13 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // Merge counters by iterating the entire counter section when data section is // empty due to correlation. if (Header->NumData == 0) { - for (SrcCounter = SrcCountersStart, - DstCounter = __llvm_profile_begin_counters(); + for (uintptr_t SrcCounter = SrcCountersStart, + DstCounter = (uintptr_t)__llvm_profile_begin_counters(); SrcCounter < SrcCountersEnd;) { if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { - *DstCounter &= *SrcCounter; + *(char *)DstCounter &= *(const char *)SrcCounter; } else { - *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter; + *(uint64_t *)DstCounter += *(const uint64_t *)SrcCounter; } SrcCounter += __llvm_profile_counter_entry_size(); DstCounter += __llvm_profile_counter_entry_size(); @@ -178,6 +175,8 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, return 0; } + __llvm_profile_data *SrcData, *DstData; + uintptr_t SrcValueProfData; for (SrcData = SrcDataStart, DstData = (__llvm_profile_data *)__llvm_profile_begin_data(), SrcValueProfData = SrcValueProfDataStart; @@ -186,10 +185,10 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // address of the data to the start address of the counter. On WIN64, // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign // extend CounterPtr to get the original value. - char *DstCounters = - (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr)); - char *DstBitmap = - (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr)); + uintptr_t DstCounters = + (uintptr_t)DstData + signextIfWin64(DstData->CounterPtr); + uintptr_t DstBitmap = + (uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr); unsigned NVK = 0; // SrcData is a serialized representation of the memory image. We need to @@ -199,7 +198,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, // CountersDelta computes the offset into the in-buffer counter section. // // On WIN64, CountersDelta is truncated as well, so no need for signext. - char *SrcCounters = + uintptr_t SrcCounters = SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta); // CountersDelta needs to be decreased as we advance to the next data // record. @@ -213,13 +212,13 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, for (unsigned I = 0; I < NC; I++) { if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) { // A value of zero signifies the function is covered. - DstCounters[I] &= SrcCounters[I]; + ((char *)DstCounters)[I] &= ((const char *)SrcCounters)[I]; } else { - ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I]; + ((uint64_t *)DstCounters)[I] += ((const uint64_t *)SrcCounters)[I]; } } - const char *SrcBitmap = + uintptr_t SrcBitmap = SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta); // BitmapDelta also needs to be decreased as we advance to the next data // record. @@ -231,7 +230,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, return 1; // Merge Src and Dst Bitmap bytes by simply ORing them together. for (unsigned I = 0; I < NB; I++) - DstBitmap[I] |= SrcBitmap[I]; + ((char *)DstBitmap)[I] |= ((const char *)SrcBitmap)[I]; } /* Now merge value profile data. */ @@ -244,7 +243,7 @@ int __llvm_profile_merge_from_buffer(const char *ProfileData, if (!NVK) continue; - if (SrcValueProfData >= ProfileData + ProfileSize) + if (SrcValueProfData >= (uintptr_t)ProfileData + ProfileSize) return 1; VPMergeHook((ValueProfData *)SrcValueProfData, DstData); SrcValueProfData = diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c index b9d51b698b414..651f8785d0b94 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformAIX.c @@ -202,6 +202,8 @@ static int dummy_vname[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_VNAME_SECT_NAME); static int dummy_vtab[0] COMPILER_RT_SECTION( COMPILER_RT_SEG INSTR_PROF_VTAB_SECT_NAME); +static int dummy_covinit_funcs[0] COMPILER_RT_SECTION( + COMPILER_RT_SEG INSTR_PROF_COVINIT_SECT_NAME); // To avoid GC'ing of the dummy variables by the linker, reference them in an // array and reference the array in the runtime registration code @@ -214,7 +216,8 @@ COMPILER_RT_VISIBILITY void *__llvm_profile_keep[] = {(void *)&dummy_cnts, (void *)&dummy_bits, (void *)&dummy_data, (void *)&dummy_name, (void *)&dummy_vnds, (void *)&dummy_orderfile, - (void *)&dummy_vname, (void *)&dummy_vtab}; + (void *)&dummy_vname, (void *)&dummy_vtab, + (void *)&dummy_covinit_funcs}; #ifdef __GNUC__ #pragma GCC diagnostic pop #endif diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c index fdcb82e4d72ba..65b7bdaf403da 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformFuchsia.c @@ -35,7 +35,7 @@ #include "InstrProfilingUtil.h" /* This variable is an external reference to symbol defined by the compiler. */ -COMPILER_RT_VISIBILITY extern intptr_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; +COMPILER_RT_VISIBILITY extern int64_t INSTR_PROF_PROFILE_COUNTER_BIAS_VAR; COMPILER_RT_VISIBILITY unsigned lprofProfileDumped(void) { return 1; diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c index b766436497b74..5b230c1b20062 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformLinux.c @@ -6,11 +6,11 @@ |* \*===----------------------------------------------------------------------===*/ -#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ - (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \ - defined(_AIX) +#if defined(__linux__) || defined(__FreeBSD__) || defined(__Fuchsia__) || \ + (defined(__sun__) && defined(__svr4__)) || defined(__NetBSD__) || \ + defined(_AIX) || defined(__wasm__) || defined(__HAIKU__) -#if !defined(_AIX) +#if !defined(_AIX) && !defined(__wasm__) #include #include #endif @@ -35,6 +35,8 @@ #define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) #define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) #define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) +#define PROF_COVINIT_START INSTR_PROF_SECT_START(INSTR_PROF_COVINIT_COMMON) +#define PROF_COVINIT_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_COVINIT_COMMON) /* Declare section start and stop symbols for various sections * generated by compiler instrumentation. @@ -56,6 +58,10 @@ extern char PROF_NAME_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY COMPILER_RT_WEAK; +extern __llvm_gcov_init_func_struct PROF_COVINIT_START COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; +extern __llvm_gcov_init_func_struct PROF_COVINIT_STOP COMPILER_RT_VISIBILITY + COMPILER_RT_WEAK; COMPILER_RT_VISIBILITY const __llvm_profile_data * __llvm_profile_begin_data(void) { @@ -110,6 +116,16 @@ COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_end_vnodes(void) { COMPILER_RT_VISIBILITY ValueProfNode *CurrentVNode = &PROF_VNODES_START; COMPILER_RT_VISIBILITY ValueProfNode *EndVNode = &PROF_VNODES_STOP; +COMPILER_RT_VISIBILITY const __llvm_gcov_init_func_struct * +__llvm_profile_begin_covinit() { + return &PROF_COVINIT_START; +} + +COMPILER_RT_VISIBILITY const __llvm_gcov_init_func_struct * +__llvm_profile_end_covinit() { + return &PROF_COVINIT_STOP; +} + #ifdef NT_GNU_BUILD_ID static size_t RoundUp(size_t size, size_t align) { return (size + align - 1) & ~(align - 1); @@ -178,41 +194,33 @@ static int WriteBinaryIds(ProfDataWriter *Writer, const ElfW(Nhdr) * Note, */ COMPILER_RT_VISIBILITY int __llvm_write_binary_ids(ProfDataWriter *Writer) { extern const ElfW(Ehdr) __ehdr_start __attribute__((visibility("hidden"))); + extern ElfW(Dyn) _DYNAMIC[] __attribute__((weak, visibility("hidden"))); + const ElfW(Ehdr) *ElfHeader = &__ehdr_start; const ElfW(Phdr) *ProgramHeader = (const ElfW(Phdr) *)((uintptr_t)ElfHeader + ElfHeader->e_phoff); + /* Compute the added base address in case of position-independent code. */ + uintptr_t Base = 0; + for (uint32_t I = 0; I < ElfHeader->e_phnum; I++) { + if (ProgramHeader[I].p_type == PT_PHDR) + Base = (uintptr_t)ProgramHeader - ProgramHeader[I].p_vaddr; + if (ProgramHeader[I].p_type == PT_DYNAMIC && _DYNAMIC) + Base = (uintptr_t)_DYNAMIC - ProgramHeader[I].p_vaddr; + } + int TotalBinaryIdsSize = 0; - uint32_t I; /* Iterate through entries in the program header. */ - for (I = 0; I < ElfHeader->e_phnum; I++) { + for (uint32_t I = 0; I < ElfHeader->e_phnum; I++) { /* Look for the notes segment in program header entries. */ if (ProgramHeader[I].p_type != PT_NOTE) continue; /* There can be multiple notes segment, and examine each of them. */ - const ElfW(Nhdr) * Note; - const ElfW(Nhdr) * NotesEnd; - /* - * When examining notes in file, use p_offset, which is the offset within - * the elf file, to find the start of notes. - */ - if (ProgramHeader[I].p_memsz == 0 || - ProgramHeader[I].p_memsz == ProgramHeader[I].p_filesz) { - Note = (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + - ProgramHeader[I].p_offset); - NotesEnd = (const ElfW(Nhdr) *)((const char *)(Note) + - ProgramHeader[I].p_filesz); - } else { - /* - * When examining notes in memory, use p_vaddr, which is the address of - * section after loaded to memory, to find the start of notes. - */ - Note = - (const ElfW(Nhdr) *)((uintptr_t)ElfHeader + ProgramHeader[I].p_vaddr); - NotesEnd = - (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz); - } + const ElfW(Nhdr) *Note = + (const ElfW(Nhdr) *)(Base + ProgramHeader[I].p_vaddr); + const ElfW(Nhdr) *NotesEnd = + (const ElfW(Nhdr) *)((const char *)(Note) + ProgramHeader[I].p_memsz); int BinaryIdsSize = WriteBinaryIds(Writer, Note, NotesEnd); if (TotalBinaryIdsSize == -1) diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c index e329dc2dbe2e2..29e570b9fba92 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPlatformOther.c @@ -8,7 +8,8 @@ #if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ !defined(__Fuchsia__) && !(defined(__sun__) && defined(__svr4__)) && \ - !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) + !defined(__NetBSD__) && !defined(_WIN32) && !defined(_AIX) && \ + !defined(__wasm__) && !defined(__HAIKU__) #include #include @@ -26,8 +27,6 @@ static const char *VNamesFirst = NULL; static const char *VNamesLast = NULL; static char *CountersFirst = NULL; static char *CountersLast = NULL; -static char *BitmapFirst = NULL; -static char *BitmapLast = NULL; static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { @@ -55,25 +54,18 @@ void __llvm_profile_register_function(void *Data_) { CountersFirst = (char *)((uintptr_t)Data_ + Data->CounterPtr); CountersLast = CountersFirst + Data->NumCounters * __llvm_profile_counter_entry_size(); - BitmapFirst = (char *)((uintptr_t)Data_ + Data->BitmapPtr); - BitmapLast = BitmapFirst + Data->NumBitmapBytes; return; } DataFirst = (const __llvm_profile_data *)getMinAddr(DataFirst, Data); CountersFirst = (char *)getMinAddr( CountersFirst, (char *)((uintptr_t)Data_ + Data->CounterPtr)); - BitmapFirst = (char *)getMinAddr( - BitmapFirst, (char *)((uintptr_t)Data_ + Data->BitmapPtr)); DataLast = (const __llvm_profile_data *)getMaxAddr(DataLast, Data + 1); CountersLast = (char *)getMaxAddr( CountersLast, (char *)((uintptr_t)Data_ + Data->CounterPtr) + Data->NumCounters * __llvm_profile_counter_entry_size()); - BitmapLast = (char *)getMaxAddr( - BitmapLast, - (char *)((uintptr_t)Data_ + Data->BitmapPtr) + Data->NumBitmapBytes); } COMPILER_RT_VISIBILITY diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h index ed0905cc5f202..66d697885eaee 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingPort.h @@ -54,7 +54,7 @@ #endif #define COMPILER_RT_MAX_HOSTLEN 128 -#ifdef __ORBIS__ +#if defined(__ORBIS__) || defined(__wasi__) #define COMPILER_RT_GETHOSTNAME(Name, Len) ((void)(Name), (void)(Len), (-1)) #else #define COMPILER_RT_GETHOSTNAME(Name, Len) lprofGetHostName(Name, Len) @@ -111,7 +111,7 @@ #if defined(_WIN32) #include -static inline size_t getpagesize() { +static inline size_t getpagesize(void) { SYSTEM_INFO S; GetNativeSystemInfo(&S); return S.dwPageSize; diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c index cd18cba3e268f..c637b9d0b893c 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.c @@ -152,9 +152,12 @@ COMPILER_RT_VISIBILITY int lprofLockFd(int fd) { } } return 0; -#else +#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32) + // Windows doesn't have flock but WindowsMMap.h provides a shim flock(fd, LOCK_EX); return 0; +#else + return 0; #endif } @@ -177,9 +180,12 @@ COMPILER_RT_VISIBILITY int lprofUnlockFd(int fd) { } } return 0; -#else +#elif defined(COMPILER_RT_HAS_FLOCK) || defined(_WIN32) + // Windows doesn't have flock but WindowsMMap.h provides a shim flock(fd, LOCK_UN); return 0; +#else + return 0; #endif } @@ -353,8 +359,8 @@ COMPILER_RT_VISIBILITY void lprofRestoreSigKill(void) { COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End) { -#if defined(__ve__) - // VE doesn't support madvise. +#if defined(__ve__) || defined(__wasi__) + // VE and WASI doesn't support madvise. return 0; #else size_t PageSize = getpagesize(); @@ -373,3 +379,72 @@ COMPILER_RT_VISIBILITY int lprofReleaseMemoryPagesToOS(uintptr_t Begin, return 0; #endif } + +#ifdef _AIX +typedef struct fn_node { + AtExit_Fn_ptr func; + struct fn_node *next; +} fn_node; +typedef struct { + fn_node *top; +} fn_stack; + +static void fn_stack_push(fn_stack *, AtExit_Fn_ptr); +static AtExit_Fn_ptr fn_stack_pop(fn_stack *); +/* return 1 if stack is empty, 0 otherwise */ +static int fn_stack_is_empty(fn_stack *); + +static fn_stack AtExit_stack = {0}; +#define ATEXIT_STACK (&AtExit_stack) + +/* On AIX, atexit() functions registered by a shared library do not get called + * when the library is dlclose'd, causing a crash when they are eventually + * called at main program exit. However, a destructor does get called. So we + * collect all atexit functions registered by profile-rt and at program + * termination time (normal exit, shared library unload, or dlclose) we walk + * the list and execute any function that is still sitting in the atexit system + * queue. + */ +__attribute__((__destructor__)) static void cleanup() { + while (!fn_stack_is_empty(ATEXIT_STACK)) { + AtExit_Fn_ptr func = fn_stack_pop(ATEXIT_STACK); + if (func && unatexit(func) == 0) + func(); + } +} + +static void fn_stack_push(fn_stack *st, AtExit_Fn_ptr func) { + fn_node *old_top, *n = (fn_node *)malloc(sizeof(fn_node)); + n->func = func; + + while (1) { + old_top = st->top; + n->next = old_top; + if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, n)) + return; + } +} +static AtExit_Fn_ptr fn_stack_pop(fn_stack *st) { + fn_node *old_top, *new_top; + while (1) { + old_top = st->top; + if (old_top == 0) + return 0; + new_top = old_top->next; + if (COMPILER_RT_BOOL_CMPXCHG(&st->top, old_top, new_top)) { + AtExit_Fn_ptr func = old_top->func; + free(old_top); + return func; + } + } +} + +static int fn_stack_is_empty(fn_stack *st) { return st->top == 0; } +#endif + +COMPILER_RT_VISIBILITY int lprofAtExit(AtExit_Fn_ptr func) { +#ifdef _AIX + fn_stack_push(ATEXIT_STACK, func); +#endif + return atexit(func); +} diff --git a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h index 4a88a03580941..227c2aa0a7cae 100644 --- a/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h +++ b/system/lib/compiler-rt/lib/profile/InstrProfilingUtil.h @@ -69,10 +69,10 @@ void *lprofPtrFetchAdd(void **Mem, long ByteIncr); /* Temporarily suspend SIGKILL. Return value of 1 means a restore is needed. * Other return values mean no restore is needed. */ -int lprofSuspendSigKill(); +int lprofSuspendSigKill(void); /* Restore previously suspended SIGKILL. */ -void lprofRestoreSigKill(); +void lprofRestoreSigKill(void); static inline size_t lprofRoundUpTo(size_t x, size_t boundary) { return (x + boundary - 1) & ~(boundary - 1); @@ -84,4 +84,9 @@ static inline size_t lprofRoundDownTo(size_t x, size_t boundary) { int lprofReleaseMemoryPagesToOS(uintptr_t Begin, uintptr_t End); +typedef void (*AtExit_Fn_ptr)(void); + +/* Call atexit and perform other platform-specific bookkeeping. */ +int lprofAtExit(AtExit_Fn_ptr); + #endif /* PROFILE_INSTRPROFILINGUTIL_H */ diff --git a/tools/system_libs.py b/tools/system_libs.py index 1688fc5c02317..0213bfb263322 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2085,9 +2085,7 @@ class CompilerRTLibrary(Library): class libcompiler_rt_profile(CompilerRTLibrary, MTLibrary): name = 'libcompiler_rt_profile' - - cflags = ['-DCOMPILER_RT_HAS_UNAME=1'] includes = ['system/lib/libc'] src_dir = 'system/lib/compiler-rt/lib/profile' src_files = glob_in_path(src_dir, '*.c') From 90db88205e1f418301cc1c96ed90bd6f410b282c Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Fri, 18 Apr 2025 10:15:37 -0400 Subject: [PATCH 07/11] fix incldue --- tools/system_libs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 0213bfb263322..06a734efaacb4 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2085,8 +2085,8 @@ class CompilerRTLibrary(Library): class libcompiler_rt_profile(CompilerRTLibrary, MTLibrary): name = 'libcompiler_rt_profile' - - includes = ['system/lib/libc'] + + includes = ['system/lib/libc', 'system/lib/compiler-rt/include'] src_dir = 'system/lib/compiler-rt/lib/profile' src_files = glob_in_path(src_dir, '*.c') src_files += glob_in_path(src_dir, '*.cpp') From 8207cb19d199596a64fa3aeb135caf420cb2d6e7 Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Fri, 18 Apr 2025 10:19:37 -0400 Subject: [PATCH 08/11] update --- tools/system_libs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/system_libs.py b/tools/system_libs.py index 06a734efaacb4..900cb78ef3ce3 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2086,6 +2086,7 @@ class CompilerRTLibrary(Library): class libcompiler_rt_profile(CompilerRTLibrary, MTLibrary): name = 'libcompiler_rt_profile' + cflags = ['-fno-builtin', '-DCOMPILER_RT_HAS_UNAME=1'] includes = ['system/lib/libc', 'system/lib/compiler-rt/include'] src_dir = 'system/lib/compiler-rt/lib/profile' src_files = glob_in_path(src_dir, '*.c') From 57521f84acd536d7222ad1867b4f90dfd76cc32e Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Mon, 21 Apr 2025 14:31:05 -0400 Subject: [PATCH 09/11] non rt --- tools/system_libs.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 900cb78ef3ce3..7651d90976079 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -2082,8 +2082,7 @@ class CompilerRTLibrary(Library): # restriction soon: https://reviews.llvm.org/D71738 force_object_files = True - -class libcompiler_rt_profile(CompilerRTLibrary, MTLibrary): +class libcompiler_rt_profile(Library): name = 'libcompiler_rt_profile' cflags = ['-fno-builtin', '-DCOMPILER_RT_HAS_UNAME=1'] @@ -2366,7 +2365,7 @@ def add_sanitizer_libs(): if only_forced: add_library('libcompiler_rt') - add_library('libcompiler_rt_profile') + add_library('libcompiler_rt_profile') add_sanitizer_libs() add_forced_libs() return libs_to_link From e1a9928a6ca84a5597d6ebf1e38d438a46d40bfd Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Mon, 21 Apr 2025 15:48:54 -0400 Subject: [PATCH 10/11] test --- test/test_core.py | 28 +++++++++++++++++++++++++++- tools/shared.py | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/test/test_core.py b/test/test_core.py index 7addc63783af4..192659c340194 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -18,7 +18,7 @@ raise Exception('do not run this file directly; do something like: test/runner') from tools.shared import PIPE -from tools.shared import EMCC, EMAR, FILE_PACKAGER +from tools.shared import EMCC, EMAR, FILE_PACKAGER, LLVM_PROFDATA, LLVM_COV from tools.utils import WINDOWS, MACOS, LINUX, write_file, delete_file from tools import shared, building, config, utils, webassembly import common @@ -9452,6 +9452,32 @@ def test_abort_on_exceptions_main(self): self.assertNotContained('unhandled exception', output) self.assertNotContained('Aborted', output) + def test_fcoverage_mapping(self): + expected = ''' 1| |/* + 2| | * Copyright 2016 The Emscripten Authors. All rights reserved. + 3| | * Emscripten is available under two separate licenses, the MIT license and the + 4| | * University of Illinois/NCSA Open Source License. Both these licenses can be + 5| | * found in the LICENSE file. + 6| | */ + 7| | + 8| |#include + 9| 1|int main() { + 10| 1| printf("hello, world!\\n"); + 11| 1| return 0; + 12| 1|} + +''' + self.emcc_args.append('-fprofile-instr-generate') + self.emcc_args.append('-fcoverage-mapping') + self.emcc_args.append('-g') + self.set_setting('NODERAWFS') + self.set_setting('EXIT_RUNTIME') + self.do_core_test('test_hello_world.c') + self.assertExists('default.profraw') + self.run_process([LLVM_PROFDATA, 'merge', '-sparse', 'default.profraw', '-o', 'out.profdata']) + self.assertExists('out.profdata') + self.assertEquals(expected, self.run_process([LLVM_COV, 'show', 'test_hello_world.wasm', '-instr-profile=out.profdata'], stdout=PIPE).stdout) + @node_pthreads @flaky('https://github.com/emscripten-core/emscripten/issues/20067') def test_abort_on_exceptions_pthreads(self): diff --git a/tools/shared.py b/tools/shared.py index d34e2a56177d0..e4fe8552e5794 100644 --- a/tools/shared.py +++ b/tools/shared.py @@ -754,6 +754,8 @@ class OFormat(Enum): LLVM_OBJCOPY = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-objcopy'))) LLVM_STRIP = os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-strip'))) WASM_LD = os.path.expanduser(build_llvm_tool_path(exe_suffix('wasm-ld'))) +LLVM_PROFDATA = 'llvm-profdata-20'#os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-profdata'))) +LLVM_COV = 'llvm-cov-20'#os.path.expanduser(build_llvm_tool_path(exe_suffix('llvm-cov'))) EMCC = bat_suffix(path_from_root('emcc')) EMXX = bat_suffix(path_from_root('em++')) From 10bbd61d472aa8f048a131ad53f75e39cbf2cb42 Mon Sep 17 00:00:00 2001 From: Andrew Snyder Date: Mon, 21 Apr 2025 16:29:38 -0400 Subject: [PATCH 11/11] move to compiler-rt --- tools/system_libs.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/tools/system_libs.py b/tools/system_libs.py index 7651d90976079..4f591bed25ffa 100644 --- a/tools/system_libs.py +++ b/tools/system_libs.py @@ -940,9 +940,10 @@ class libcompiler_rt(MTLibrary, SjLjLibrary): # restriction soon: https://reviews.llvm.org/D71738 force_object_files = True - cflags = ['-fno-builtin', '-DNDEBUG'] + cflags = ['-fno-builtin', '-DNDEBUG', '-DCOMPILER_RT_HAS_UNAME=1'] src_dir = 'system/lib/compiler-rt/lib/builtins' - includes = ['system/lib/libc'] + profile_src_dir = 'system/lib/compiler-rt/lib/profile' + includes = ['system/lib/libc', 'system/lib/compiler-rt/include'] excludes = [ # gcc_personality_v0.c depends on libunwind, which don't include by default. 'gcc_personality_v0.c', @@ -970,6 +971,8 @@ class libcompiler_rt(MTLibrary, SjLjLibrary): 'trunctfxf2.c', ] src_files = glob_in_path(src_dir, '*.c', excludes=excludes) + src_files += glob_in_path(profile_src_dir, '*.c') + src_files += glob_in_path(profile_src_dir, '*.cpp') src_files += files_in_path( path='system/lib/compiler-rt', filenames=[ @@ -2082,15 +2085,6 @@ class CompilerRTLibrary(Library): # restriction soon: https://reviews.llvm.org/D71738 force_object_files = True -class libcompiler_rt_profile(Library): - name = 'libcompiler_rt_profile' - - cflags = ['-fno-builtin', '-DCOMPILER_RT_HAS_UNAME=1'] - includes = ['system/lib/libc', 'system/lib/compiler-rt/include'] - src_dir = 'system/lib/compiler-rt/lib/profile' - src_files = glob_in_path(src_dir, '*.c') - src_files += glob_in_path(src_dir, '*.cpp') - class libubsan_minimal_rt(CompilerRTLibrary, MTLibrary): name = 'libubsan_minimal_rt' never_force = True @@ -2365,7 +2359,6 @@ def add_sanitizer_libs(): if only_forced: add_library('libcompiler_rt') - add_library('libcompiler_rt_profile') add_sanitizer_libs() add_forced_libs() return libs_to_link @@ -2403,7 +2396,6 @@ def add_sanitizer_libs(): elif settings.MALLOC != 'none': add_library('libmalloc') add_library('libcompiler_rt') - add_library('libcompiler_rt_profile') if settings.LINK_AS_CXX: add_library('libc++') if settings.LINK_AS_CXX or sanitize: